Flowfile 0.4.1__py3-none-any.whl → 0.5.3__py3-none-any.whl

build_backends/main.py

@@ -1,12 +1,11 @@
 import os
-import subprocess
 import platform
 import shutil
+import subprocess
 from pathlib import Path
-from typing import List
 
 
-def merge_directories(directories: List[str], target_dir: str, cleanup_after_merge: bool = True):
+def merge_directories(directories: list[str], target_dir: str, cleanup_after_merge: bool = True):
     """
     Merge all files from two folders into a new target directory.
     After successful merge, removes the original folders.
@@ -16,9 +15,9 @@ def merge_directories(directories: List[str], target_dir: str, cleanup_after_mer
     for directory in directories:
         if os.path.exists(directory):
             shutil.copytree(directory, target_dir, dirs_exist_ok=True)
-    print('Merged directories:', directories, 'into', target_dir)
+    print("Merged directories:", directories, "into", target_dir)
     if cleanup_after_merge:
-        print('Cleaning up directories:', directories)
+        print("Cleaning up directories:", directories)
         for directory in directories:
             if os.path.exists(directory):
                 shutil.rmtree(directory)
@@ -197,25 +196,28 @@ coll = COLLECT(
     name='{output_name}'
 )
 '''
-    spec_path = f'{output_name}.spec'
-    with open(spec_path, 'w') as f:
+    spec_path = f"{output_name}.spec"
+    with open(spec_path, "w") as f:
         f.write(spec_content)
     return spec_path
 
+
 def build_backend(directory, script_name, output_name, hidden_imports=None):
     try:
         spec_path = create_spec_file(directory, script_name, output_name, hidden_imports)
 
         env = os.environ.copy()
-        env['PYTHONOPTIMIZE'] = "1"
+        env["PYTHONOPTIMIZE"] = "1"
 
         command = [
             "pyinstaller",
             "--clean",
             "-y",
-            "--dist", "./services_dist",
-            "--workpath", "/tmp" if platform.system() != "Windows" else os.path.join(os.getenv('TEMP'), 'pyinstaller'),
-            spec_path
+            "--dist",
+            "./services_dist",
+            "--workpath",
+            "/tmp" if platform.system() != "Windows" else os.path.join(os.getenv("TEMP"), "pyinstaller"),
+            spec_path,
         ]
 
         print(f"Building {output_name}...")
@@ -247,7 +249,7 @@ def combine_packages():
             exe_name = project + ".exe" if platform.system() == "Windows" else project
             src_exe = os.path.join(src_dir, exe_name)
             temp_target_exe = os.path.join(dist_dir, "_" + exe_name)
-            target_exe = os.path.join(dist_dir,  exe_name)
+            target_exe = os.path.join(dist_dir, exe_name)
             if os.path.exists(src_exe) and os.path.isfile(src_exe):
                 # Instead of removing, overwrite the target
                 shutil.move(src_exe, temp_target_exe)
@@ -257,9 +259,10 @@ def combine_packages():
             if platform.system() == "Windows" and os.path.exists(os.path.join(dist_dir, project)):
                 shutil.rmtree(os.path.join(dist_dir, project))
 
+
 def main():
     # Clean previous builds
-    for dir_name in ['services_dist']:
+    for dir_name in ["services_dist"]:
         if os.path.exists(dir_name):
             shutil.rmtree(dir_name)
 
@@ -286,20 +289,20 @@ def main():
     # Build flowfile_worker
 
     if not build_backend(
-            directory=os.path.join("flowfile_worker", "flowfile_worker"),
-            script_name="main.py",
-            output_name="flowfile_worker",
-            hidden_imports=common_imports
+        directory=os.path.join("flowfile_worker", "flowfile_worker"),
+        script_name="main.py",
+        output_name="flowfile_worker",
+        hidden_imports=common_imports,
     ):
         builds_successful = False
 
     # Build flowfile_core
 
     if not build_backend(
-            directory=os.path.join("flowfile_core", "flowfile_core"),
-            script_name="main.py",
-            output_name="flowfile_core",
-            hidden_imports=common_imports
+        directory=os.path.join("flowfile_core", "flowfile_core"),
+        script_name="main.py",
+        output_name="flowfile_core",
+        hidden_imports=common_imports,
     ):
         builds_successful = False
 
@@ -310,4 +313,4 @@ def main():
 
 
 # if __name__ == "__main__":
-#     main()
+#     main()

build_backends/main_prd.py

@@ -1,14 +1,9 @@
-import os
-import subprocess
-import platform
-from concurrent.futures import ProcessPoolExecutor, wait
-
 import subprocess
 import time
-import requests
-from statistics import mean, stdev
-import sys
 from datetime import datetime
+from statistics import mean, stdev
+
+import requests
 
 
 def wait_for_endpoint(url, timeout=60):
@@ -27,11 +22,7 @@ def wait_for_endpoint(url, timeout=60):
 def shutdown_service():
     """Shutdown the service gracefully using the shutdown endpoint."""
     try:
-        response = requests.post(
-            'http://0.0.0.0:63578/shutdown',
-            headers={'accept': 'application/json'},
-            data=''
-        )
+        response = requests.post("http://0.0.0.0:63578/shutdown", headers={"accept": "application/json"}, data="")
         print("Shutdown request sent, waiting for service to stop...")
         time.sleep(1)  # Wait 10 seconds to ensure the service is fully stopped
         return True
@@ -97,14 +88,14 @@ def run_comparison_test(old_exe, new_exe, num_runs=3):
     print("\nResults:")
     print("-" * 50)
     if old_times:
-        print(f"Old executable:")
+        print("Old executable:")
         print(f"  Average: {mean(old_times):.3f} seconds")
         print(f"  Std Dev: {stdev(old_times):.3f} seconds" if len(old_times) > 1 else "  Std Dev: N/A")
         print(f"  Min: {min(old_times):.3f} seconds")
         print(f"  Max: {max(old_times):.3f} seconds")
 
     if new_times:
-        print(f"\nNew executable:")
+        print("\nNew executable:")
         print(f"  Average: {mean(new_times):.3f} seconds")
         print(f"  Std Dev: {stdev(new_times):.3f} seconds" if len(new_times) > 1 else "  Std Dev: N/A")
         print(f"  Min: {min(new_times):.3f} seconds")
@@ -112,13 +103,13 @@ def run_comparison_test(old_exe, new_exe, num_runs=3):
 
     if old_times and new_times:
         improvement = (mean(old_times) - mean(new_times)) / mean(old_times) * 100
-        print(f"\nPerformance difference:")
+        print("\nPerformance difference:")
         print(f"  {improvement:.1f}% {'faster' if improvement > 0 else 'slower'} than old version")
 
 
 if __name__ == "__main__":
-    old_exe = '/Users/edwardvanechoud/personal_dev/Flowfile/dist/flowfile_core/flowfile_core'
-    new_exe = '/Users/edwardvanechoud/personal_dev/Flowfile/dist_flowfile_core/flowfile_core'
+    old_exe = "/Users/edwardvanechoud/personal_dev/Flowfile/dist/flowfile_core/flowfile_core"
+    new_exe = "/Users/edwardvanechoud/personal_dev/Flowfile/dist_flowfile_core/flowfile_core"
 
     run_comparison_test(old_exe, old_exe)
 
@@ -199,4 +190,4 @@ if __name__ == "__main__":
 #
 #
 # if __name__ == "__main__":
-#     main()
+#     main()

flowfile/__init__.py

@@ -7,100 +7,206 @@ This package ties together the FlowFile ecosystem components:
 - flowfile_worker: Computation engine
 """
 
-__version__ = "0.4.1"
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("Flowfile")
+except PackageNotFoundError:
+    __version__ = "0.5.0"
 
-import os
 import logging
+import os
 
-os.environ['FLOWFILE_WORKER_PORT'] = "63578"
-os.environ['FLOWFILE_SINGLE_FILE_MODE'] = "1"
+os.environ["FLOWFILE_WORKER_PORT"] = "63578"
+os.environ["FLOWFILE_SINGLE_FILE_MODE"] = "1"
 
-from flowfile.web import start_server as start_web_ui
-from flowfile.api import (open_graph_in_editor)
-from flowfile_frame.flow_frame import (
-    FlowFrame
+from polars.datatypes import (
+    Array,
+    Binary,
+    Boolean,
+    Categorical,
+    DataType,
+    DataTypeClass,
+    Date,
+    Datetime,
+    Decimal,
+    Duration,
+    Enum,
+    Field,
+    Float32,
+    Float64,
+    Int8,
+    Int16,
+    Int32,
+    Int64,
+    Int128,
+    List,
+    Null,
+    Object,
+    String,
+    Struct,
+    Time,
+    UInt8,
+    UInt16,
+    UInt32,
+    UInt64,
+    Unknown,
+    Utf8,
 )
-from flowfile_core.schemas.cloud_storage_schemas import FullCloudStorageConnection
-from flowfile_core.flowfile.flow_graph import FlowGraph
+
+from flowfile.api import open_graph_in_editor
+from flowfile.web import start_server as start_web_ui
+from flowfile_core.flowfile import node_designer
 from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEngine
-from flowfile_core.flowfile.flow_node.flow_node import FlowNode
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn
+from flowfile_core.flowfile.flow_graph import FlowGraph
+from flowfile_core.flowfile.flow_node.flow_node import FlowNode
 from flowfile_core.schemas import input_schema as node_interface
-from flowfile_core.schemas.schemas import FlowSettings, FlowInformation
 from flowfile_core.schemas import transform_schema
-from flowfile_frame import (read_csv, read_parquet, from_dict, concat, scan_csv, scan_parquet,
-                            scan_delta,
-                            scan_parquet_from_cloud_storage,
-                            scan_json_from_cloud_storage,
-                            scan_csv_from_cloud_storage,
-                            get_all_available_cloud_storage_connections,
-                            create_cloud_storage_connection,
-                            del_cloud_storage_connection,
-                            create_cloud_storage_connection_if_not_exists,
-                            FuzzyMapping)
-from flowfile_frame.expr import (
-    col, lit, column, cum_count, len,
-    sum, min, max, mean, count, when
+from flowfile_core.schemas.cloud_storage_schemas import FullCloudStorageConnection
+from flowfile_core.schemas.schemas import FlowInformation, FlowSettings
+from flowfile_frame import (
+    FuzzyMapping,
+    concat,
+    create_cloud_storage_connection,
+    create_cloud_storage_connection_if_not_exists,
+    del_cloud_storage_connection,
+    from_dict,
+    get_all_available_cloud_storage_connections,
+    read_csv,
+    read_parquet,
+    scan_csv,
+    scan_csv_from_cloud_storage,
+    scan_delta,
+    scan_json_from_cloud_storage,
+    scan_parquet,
+    scan_parquet_from_cloud_storage,
 )
+from flowfile_frame.expr import col, column, count, cum_count, len, lit, max, mean, min, sum, when
+from flowfile_frame.flow_frame import FlowFrame
 from flowfile_frame.group_frame import GroupByFrame
-from flowfile_frame.utils import create_flow_graph
 from flowfile_frame.selectors import (
-    numeric, float_, integer, string, temporal,
-    datetime, date, time, duration, boolean,
-    categorical, object_, list_, struct, all_,
-    by_dtype, contains, starts_with, ends_with, matches
-)
-
-from polars.datatypes import (
-    Int8, Int16, Int32, Int64, Int128,
-    UInt8, UInt16, UInt32, UInt64,
-    Float32, Float64,
-    Boolean, String, Utf8, Binary, Null,
-    List, Array, Struct, Object,
-    Date, Time, Datetime, Duration,
-    Categorical, Decimal, Enum, Unknown,
-    DataType, DataTypeClass, Field
+    all_,
+    boolean,
+    by_dtype,
+    categorical,
+    contains,
+    date,
+    datetime,
+    duration,
+    ends_with,
+    float_,
+    integer,
+    list_,
+    matches,
+    numeric,
+    object_,
+    starts_with,
+    string,
+    struct,
+    temporal,
+    time,
 )
-
-from flowfile_core.flowfile import node_designer
-
+from flowfile_frame.utils import create_flow_graph
 
 __all__ = [
     # Core FlowFrame classes
-    'FlowFrame', 'GroupByFrame', 'FullCloudStorageConnection',
-
+    "FlowFrame",
+    "GroupByFrame",
+    "FullCloudStorageConnection",
     # Main creation functions
-    'read_csv', 'read_parquet', 'from_dict', 'concat', 'scan_csv', 'scan_parquet', 'scan_delta',
-    'scan_parquet_from_cloud_storage', 'scan_json_from_cloud_storage',
-    'scan_csv_from_cloud_storage', 'get_all_available_cloud_storage_connections', 'create_cloud_storage_connection',
-    'del_cloud_storage_connection', 'create_cloud_storage_connection_if_not_exists',
-    'FlowGraph', 'FlowDataEngine', 'node_interface', 'FlowSettings', 'transform_schema',
-    'FlowNode', 'FlowfileColumn', 'FlowInformation', "FuzzyMapping",
-
+    "read_csv",
+    "read_parquet",
+    "from_dict",
+    "concat",
+    "scan_csv",
+    "scan_parquet",
+    "scan_delta",
+    "scan_parquet_from_cloud_storage",
+    "scan_json_from_cloud_storage",
+    "scan_csv_from_cloud_storage",
+    "get_all_available_cloud_storage_connections",
+    "create_cloud_storage_connection",
+    "del_cloud_storage_connection",
+    "create_cloud_storage_connection_if_not_exists",
+    "FlowGraph",
+    "FlowDataEngine",
+    "node_interface",
+    "FlowSettings",
+    "transform_schema",
+    "FlowNode",
+    "FlowfileColumn",
+    "FlowInformation",
+    "FuzzyMapping",
     # Expression API
-    'col', 'lit', 'column', 'cum_count', 'len',
-    'sum', 'min', 'max', 'mean', 'count', 'when',
-
+    "col",
+    "lit",
+    "column",
+    "cum_count",
+    "len",
+    "sum",
+    "min",
+    "max",
+    "mean",
+    "count",
+    "when",
     # Selector utilities
-    'numeric', 'float_', 'integer', 'string', 'temporal',
-    'datetime', 'date', 'time', 'duration', 'boolean',
-    'categorical', 'object_', 'list_', 'struct', 'all_',
-    'by_dtype', 'contains', 'starts_with', 'ends_with', 'matches',
-
+    "numeric",
+    "float_",
+    "integer",
+    "string",
+    "temporal",
+    "datetime",
+    "date",
+    "time",
+    "duration",
+    "boolean",
+    "categorical",
+    "object_",
+    "list_",
+    "struct",
+    "all_",
+    "by_dtype",
+    "contains",
+    "starts_with",
+    "ends_with",
+    "matches",
     "node_designer",
-
     # Utilities
-    'create_flow_graph', 'open_graph_in_editor',
-
+    "create_flow_graph",
+    "open_graph_in_editor",
     # Data types from Polars
-    'Int8', 'Int16', 'Int32', 'Int64', 'Int128',
-    'UInt8', 'UInt16', 'UInt32', 'UInt64',
-    'Float32', 'Float64',
-    'Boolean', 'String', 'Utf8', 'Binary', 'Null',
-    'List', 'Array', 'Struct', 'Object',
-    'Date', 'Time', 'Datetime', 'Duration',
-    'Categorical', 'Decimal', 'Enum', 'Unknown',
-    'DataType', 'DataTypeClass', 'Field',
-    'start_web_ui'
+    "Int8",
+    "Int16",
+    "Int32",
+    "Int64",
+    "Int128",
+    "UInt8",
+    "UInt16",
+    "UInt32",
+    "UInt64",
+    "Float32",
+    "Float64",
+    "Boolean",
+    "String",
+    "Utf8",
+    "Binary",
+    "Null",
+    "List",
+    "Array",
+    "Struct",
+    "Object",
+    "Date",
+    "Time",
+    "Datetime",
+    "Duration",
+    "Categorical",
+    "Decimal",
+    "Enum",
+    "Unknown",
+    "DataType",
+    "DataTypeClass",
+    "Field",
+    "start_web_ui",
 ]
 logging.getLogger("PipelineHandler").setLevel(logging.WARNING)

flowfile/__main__.py

@@ -1,16 +1,19 @@
 #  flowfile/__main__.py
 
+
 def main():
     """
     Display information about FlowFile when run directly as a module.
     """
-    import flowfile
     import argparse
 
+    import flowfile
+
     parser = argparse.ArgumentParser(description="FlowFile: A visual ETL tool with a Polars-like API")
     parser.add_argument("command", nargs="?", choices=["run"], help="Command to execute")
-    parser.add_argument("component", nargs="?", choices=["ui", "core", "worker"],
-                        help="Component to run (ui, core, or worker)")
+    parser.add_argument(
+        "component", nargs="?", choices=["ui", "core", "worker"], help="Component to run (ui, core, or worker)"
+    )
     parser.add_argument("--host", default="127.0.0.1", help="Host to bind the server to")
     parser.add_argument("--port", type=int, default=63578, help="Port to bind the server to")
     parser.add_argument("--no-browser", action="store_true", help="Don't open a browser window")
@@ -21,18 +24,18 @@ def main():
     if args.command == "run" and args.component:
         if args.component == "ui":
             try:
-                flowfile.start_web_ui(
-                    open_browser=not args.no_browser
-                )
+                flowfile.start_web_ui(open_browser=not args.no_browser)
             except KeyboardInterrupt:
                 print("\nFlowFile service stopped.")
         elif args.component == "core":
             # Only for direct core service usage
             from flowfile_core.main import run as run_core
+
             run_core(host=args.host, port=args.port)
         elif args.component == "worker":
             # Only for direct worker service usage
             from flowfile_worker.main import run as run_worker
+
             run_worker(host=args.host, port=args.port)
     else:
         # Default action - show info
@@ -54,4 +57,4 @@ def main():
         print("  import flowfile as ff")
         print("  df = ff.read_csv('data.csv')")
         print("  result = df.filter(ff.col('value') > 10)")
-        print("  ff.open_graph_in_editor(result)")
+        print("  ff.open_graph_in_editor(result)")