Flowfile 0.2.2__py3-none-any.whl

build_backends/main.py

@@ -0,0 +1,313 @@
+import os
+import subprocess
+import platform
+import shutil
+from pathlib import Path
+from typing import List
+
+
+def merge_directories(directories: List[str], target_dir: str, cleanup_after_merge: bool = True):
+    """
+    Merge all files from two folders into a new target directory.
+    After successful merge, removes the original folders.
+    """
+    # Create target directory
+    Path(target_dir).mkdir(parents=True, exist_ok=True)
+    for directory in directories:
+        if os.path.exists(directory):
+            shutil.copytree(directory, target_dir, dirs_exist_ok=True)
+    print('Merged directories:', directories, 'into', target_dir)
+    if cleanup_after_merge:
+        print('Cleaning up directories:', directories)
+        for directory in directories:
+            if os.path.exists(directory):
+                shutil.rmtree(directory)
+
+
+def create_spec_file(directory, script_name, output_name, hidden_imports):
+    """Create an optimized spec file for faster startup"""
+    spec_content = f'''
+import sys
+import os
+from PyInstaller.utils.hooks import collect_submodules, collect_data_files
+
+# Add hook to fix connectorx metadata
+def get_connectorx_metadata():
+    print("Collecting connectorx metadata...")
+    try:
+        import site
+        import connectorx
+
+        # Find the site-packages directory
+        site_packages = site.getsitepackages()[0]
+        print(f"Site-packages directory: {{site_packages}}")
+
+        # Try both common metadata formats with glob to catch any version
+        import glob
+        metadata_locations = []
+
+        # Look for dist-info directories
+        dist_info_pattern = os.path.join(site_packages, 'connectorx*.dist-info')
+        for dist_info in glob.glob(dist_info_pattern):
+            metadata_locations.append(dist_info)
+
+        # Look for egg-info directories    
+        egg_info_pattern = os.path.join(site_packages, 'connectorx*.egg-info')
+        for egg_info in glob.glob(egg_info_pattern):
+            metadata_locations.append(egg_info)
+
+        # Also try looking in the parent directory of the connectorx package
+        connectorx_dir = os.path.dirname(connectorx.__file__)
+        parent_dir = os.path.dirname(connectorx_dir)
+
+        dist_info_pattern = os.path.join(parent_dir, 'connectorx*.dist-info')
+        for dist_info in glob.glob(dist_info_pattern):
+            metadata_locations.append(dist_info)
+
+        egg_info_pattern = os.path.join(parent_dir, 'connectorx*.egg-info')
+        for egg_info in glob.glob(egg_info_pattern):
+            metadata_locations.append(egg_info)
+
+        found_metadata = []
+        for loc in metadata_locations:
+            if os.path.exists(loc):
+                dest_name = os.path.basename(loc)
+                found_metadata.append((loc, dest_name))
+                print(f"Found metadata at {{loc}}")
+
+        if found_metadata:
+            return found_metadata
+
+        # If we can't find the metadata, create a fake one
+        print("No connectorx metadata found, creating manual metadata...")
+        import tempfile
+        temp_dir = tempfile.mkdtemp()
+        fake_meta_dir = os.path.join(temp_dir, 'connectorx-0.4.3.dist-info')
+        os.makedirs(fake_meta_dir, exist_ok=True)
+
+        # Create minimal METADATA file
+        with open(os.path.join(fake_meta_dir, 'METADATA'), 'w') as f:
+            f.write("""Metadata-Version: 2.1
+Name: connectorx
+Version: 0.4.3
+Summary: ConnectorX: Fast and Reliable Data Loading
+""")
+
+        # Return the fake metadata directory
+        print(f"Created fake metadata at {{fake_meta_dir}}")
+        return [(fake_meta_dir, 'connectorx-0.4.3.dist-info')]
+    except Exception as e:
+        print(f"Error collecting connectorx metadata: {{e}}")
+        return []
+
+# Add runtime hook to handle connectorx metadata issues
+def create_runtime_hook():
+    return """
+# Runtime hook to handle connectorx metadata issues
+import sys
+import importlib.metadata
+
+# Store original version function
+original_version = importlib.metadata.version
+
+# Create patched version function
+def patched_version(distribution_name):
+    try:
+        return original_version(distribution_name)
+    except (importlib.metadata.PackageNotFoundError, StopIteration):
+        # Handle specific packages
+        if distribution_name == 'connectorx':
+            return '0.4.3'  # Hardcode the version
+        # Let other package errors propagate normally
+        raise
+
+# Apply the patch
+importlib.metadata.version = patched_version
+print("Applied connectorx metadata patch")
+"""
+
+# Collect minimal snowflake dependencies
+snowflake_imports = collect_submodules('snowflake.connector', 
+    filter=lambda name: any(x in name for x in [
+        'connection',
+        'errors',
+        'snow_logging',
+        'auth',
+        'network'
+    ])
+)
+
+# Collect numpy and pyarrow data files
+numpy_datas = collect_data_files('numpy')
+pyarrow_datas = collect_data_files('pyarrow')
+connectorx_datas = get_connectorx_metadata()
+
+# Create runtime hook file
+with open('connectorx_hook.py', 'w') as f:
+    f.write(create_runtime_hook())
+
+a = Analysis(
+    [r'{os.path.join(directory, script_name)}'],
+    binaries=[],
+    datas=numpy_datas + pyarrow_datas + connectorx_datas,
+    hiddenimports={hidden_imports} + snowflake_imports + [
+        'numpy',
+        'numpy.core._dtype_ctypes',
+        'numpy.core._methods',
+        'numpy._pyarray_api',
+        'pyarrow',
+        'pyarrow.lib',
+        'fastexcel',
+        'importlib.metadata',
+    ],
+    excludes=[
+        'tkinter',
+        'PIL',
+        'pytest',
+        'unittest'
+    ],
+    runtime_hooks=['connectorx_hook.py'],
+    noarchive=False,
+)
+
+pyz = PYZ(a.pure, compress_level=9)
+
+exe = EXE(
+    pyz,
+    a.scripts,
+    [],
+    exclude_binaries=True,
+    name='{output_name}',
+    debug=False,
+    bootloader_ignore_signals=False,
+    strip=False,
+    upx=False,
+    console=True,
+    optimize=1
+)
+
+coll = COLLECT(
+    exe,
+    a.binaries,
+    a.zipfiles,
+    a.datas,
+    strip=False,
+    upx=False,
+    upx_exclude=[],
+    name='{output_name}'
+)
+'''
+    spec_path = f'{output_name}.spec'
+    with open(spec_path, 'w') as f:
+        f.write(spec_content)
+    return spec_path
+
+def build_backend(directory, script_name, output_name, hidden_imports=None):
+    try:
+        spec_path = create_spec_file(directory, script_name, output_name, hidden_imports)
+
+        env = os.environ.copy()
+        env['PYTHONOPTIMIZE'] = "1"
+
+        command = [
+            "pyinstaller",
+            "--clean",
+            "-y",
+            "--dist", "./services_dist",
+            "--workpath", "/tmp" if platform.system() != "Windows" else os.path.join(os.getenv('TEMP'), 'pyinstaller'),
+            spec_path
+        ]
+
+        print(f"Building {output_name}...")
+        subprocess.run(command, check=True, env=env)
+        os.remove(spec_path)
+
+        return True
+
+    except subprocess.CalledProcessError as e:
+        print(f"Error while building {script_name}: {e}")
+        return False
+    except Exception as e:
+        print(f"Unexpected error: {e}")
+        return False
+
+
+def combine_packages():
+    """Reorganize the services_dist directory to have shared dependencies"""
+    dist_dir = "services_dist"
+    shared_internal = os.path.join(dist_dir, "_internal")
+    core_internal = os.path.join(dist_dir, "flowfile_core", "_internal")
+    worker_internal = os.path.join(dist_dir, "flowfile_worker", "_internal")
+    merge_directories([core_internal, worker_internal], shared_internal, False)
+
+    for project in ["flowfile_worker", "flowfile_core"]:
+        src_dir = os.path.join(dist_dir, project)
+        if os.path.exists(src_dir) and os.path.isdir(src_dir):
+            # Move executable
+            exe_name = project + ".exe" if platform.system() == "Windows" else project
+            src_exe = os.path.join(src_dir, exe_name)
+            temp_target_exe = os.path.join(dist_dir, "_" + exe_name)
+            target_exe = os.path.join(dist_dir,  exe_name)
+            if os.path.exists(src_exe) and os.path.isfile(src_exe):
+                # Instead of removing, overwrite the target
+                shutil.move(src_exe, temp_target_exe)
+            if os.path.exists(target_exe) and os.path.isdir(target_exe):
+                shutil.rmtree(target_exe)
+            shutil.move(temp_target_exe, target_exe)
+            if platform.system() == "Windows" and os.path.exists(os.path.join(dist_dir, project)):
+                shutil.rmtree(os.path.join(dist_dir, project))
+
+def main():
+    # Clean previous builds
+    for dir_name in ['services_dist']:
+        if os.path.exists(dir_name):
+            shutil.rmtree(dir_name)
+
+    # Common imports for both projects
+    common_imports = [
+        "fastexcel",
+        "polars",
+        "numpy",
+        "numpy.core._methods",
+        "pyarrow",
+        "snowflake.connector",
+        "snowflake.connector.snow_logging",
+        "snowflake.connector.errors",
+        "multiprocessing",
+        "uvicorn.protocols.http",
+        "uvicorn.protocols.websockets",
+        "passlib.handlers.bcrypt",
+        "connectorx",
+    ]
+
+    # Build both projects
+    builds_successful = True
+
+    # Build flowfile_worker
+
+    if not build_backend(
+            directory=os.path.join("flowfile_worker", "flowfile_worker"),
+            script_name="main.py",
+            output_name="flowfile_worker",
+            hidden_imports=common_imports
+    ):
+        builds_successful = False
+
+    # Build flowfile_core
+
+    if not build_backend(
+            directory=os.path.join("flowfile_core", "flowfile_core"),
+            script_name="main.py",
+            output_name="flowfile_core",
+            hidden_imports=common_imports
+    ):
+        builds_successful = False
+
+    if builds_successful:
+        print("Reorganizing services_dist directory...")
+        combine_packages()
+        print("Build complete! Final structure created in services_dist/")
+
+
+# if __name__ == "__main__":
+#     main()

build_backends/main_prd.py

@@ -0,0 +1,202 @@
+import os
+import subprocess
+import platform
+from concurrent.futures import ProcessPoolExecutor, wait
+
+import subprocess
+import time
+import requests
+from statistics import mean, stdev
+import sys
+from datetime import datetime
+
+
+def wait_for_endpoint(url, timeout=60):
+    """Wait for the endpoint to become available."""
+    start_time = time.time()
+    while time.time() - start_time < timeout:
+        try:
+            response = requests.get(url)
+            if response.status_code == 200:
+                return True
+        except requests.exceptions.ConnectionError:
+            time.sleep(0.1)
+    return False
+
+
+def shutdown_service():
+    """Shutdown the service gracefully using the shutdown endpoint."""
+    try:
+        response = requests.post(
+            'http://0.0.0.0:63578/shutdown',
+            headers={'accept': 'application/json'},
+            data=''
+        )
+        print("Shutdown request sent, waiting for service to stop...")
+        time.sleep(1)  # Wait 10 seconds to ensure the service is fully stopped
+        return True
+    except requests.exceptions.RequestException as e:
+        print(f"Error shutting down service: {e}")
+        return False
+
+
+def measure_startup_time(executable_path):
+    """Measure the startup time of the executable."""
+    start_time = time.time()
+
+    # Start the process
+    process = subprocess.Popen([executable_path])
+
+    # Wait for the endpoint to become available
+    endpoint_url = "http://0.0.0.0:63578/docs"
+    if not wait_for_endpoint(endpoint_url):
+        print(f"Error: Endpoint did not become available for {executable_path}")
+        process.kill()
+        return None
+
+    elapsed_time = time.time() - start_time
+
+    # Gracefully shutdown the service
+    if not shutdown_service():
+        print("Failed to shutdown service gracefully, killing process...")
+        process.kill()
+        time.sleep(1)
+
+    return elapsed_time
+
+
+def run_comparison_test(old_exe, new_exe, num_runs=3):
+    """Run multiple comparison tests and print statistics."""
+    print(f"\nStarting comparison test at {datetime.now()}")
+    print(f"Number of runs: {num_runs}")
+    print("\nExecutables being tested:")
+    print(f"Old: {old_exe}")
+    print(f"New: {new_exe}")
+
+    old_times = []
+    new_times = []
+
+    for i in range(num_runs):
+        print(f"\nRun {i + 1}/{num_runs}")
+
+        # Test old executable
+        print("Testing old executable...")
+        old_time = measure_startup_time(old_exe)
+        if old_time is not None:
+            old_times.append(old_time)
+            print(f"Old startup time: {old_time:.3f} seconds")
+
+        # Test new executable
+        print("Testing new executable...")
+        new_time = measure_startup_time(new_exe)
+        if new_time is not None:
+            new_times.append(new_time)
+            print(f"New startup time: {new_time:.3f} seconds")
+
+    # Print results
+    print("\nResults:")
+    print("-" * 50)
+    if old_times:
+        print(f"Old executable:")
+        print(f"  Average: {mean(old_times):.3f} seconds")
+        print(f"  Std Dev: {stdev(old_times):.3f} seconds" if len(old_times) > 1 else "  Std Dev: N/A")
+        print(f"  Min: {min(old_times):.3f} seconds")
+        print(f"  Max: {max(old_times):.3f} seconds")
+
+    if new_times:
+        print(f"\nNew executable:")
+        print(f"  Average: {mean(new_times):.3f} seconds")
+        print(f"  Std Dev: {stdev(new_times):.3f} seconds" if len(new_times) > 1 else "  Std Dev: N/A")
+        print(f"  Min: {min(new_times):.3f} seconds")
+        print(f"  Max: {max(new_times):.3f} seconds")
+
+    if old_times and new_times:
+        improvement = (mean(old_times) - mean(new_times)) / mean(old_times) * 100
+        print(f"\nPerformance difference:")
+        print(f"  {improvement:.1f}% {'faster' if improvement > 0 else 'slower'} than old version")
+
+
+if __name__ == "__main__":
+    old_exe = '/Users/edwardvanechoud/personal_dev/Flowfile/dist/flowfile_core/flowfile_core'
+    new_exe = '/Users/edwardvanechoud/personal_dev/Flowfile/dist_flowfile_core/flowfile_core'
+
+    run_comparison_test(old_exe, old_exe)
+
+# def build_backend(directory, script_name, output_name, hidden_imports=None):
+#     try:
+#         script_path = os.path.join(directory, script_name)
+#         command = [
+#             "python", "-m", "nuitka",
+#             "--onefile",
+#             "--standalone",
+#             "--assume-yes-for-downloads",
+#             "--include-package=tempfile",
+#             "--include-package=polars",
+#             "--include-package=fastexcel",
+#             "--include-package=snowflake.connector"
+#         ]
+#
+#         if hidden_imports:
+#             for imp in hidden_imports:
+#                 if '.' not in imp:
+#                     command.extend(["--include-package=" + imp])
+#                 else:
+#                     command.extend(["--include-module=" + imp])
+#
+#         dist_folder = f"dist_{output_name}"
+#         os.makedirs(dist_folder, exist_ok=True)
+#         ext = ".exe" if platform.system() == "Windows" else ""
+#         command.extend([
+#             f"--output-dir={dist_folder}",
+#             f"--output-filename={output_name}{ext}",
+#             script_path
+#         ])
+#
+#         print(f"Starting build for {output_name}")
+#         result = subprocess.run(command, check=True)
+#         print(f"Build completed for {output_name} with exit code {result.returncode}")
+#         return result.returncode
+#
+#     except subprocess.CalledProcessError as e:
+#         print(f"Error while building {script_name}: {e}")
+#         return 1
+#
+#
+# def main():
+#     common_imports = [
+#         "fastexcel",
+#         "polars",
+#         "snowflake.connector",
+#         "snowflake.connector.snow_logging",
+#         "snowflake.connector.errors"
+#     ]
+#
+#     builds = [
+#         {
+#             "directory": os.path.join("flowfile_worker", "flowfile_worker"),
+#             "script_name": "main.py",
+#             "output_name": "flowfile_worker",
+#             "hidden_imports": ["multiprocessing", "multiprocessing.resource_tracker",
+#                                "multiprocessing.sharedctypes", "uvicorn",
+#                                "uvicorn.logging", "uvicorn.protocols.http",
+#                                "uvicorn.protocols.websockets"] + common_imports
+#         },
+#         {
+#             "directory": os.path.join("flowfile_core", "flowfile_core"),
+#             "script_name": "main.py",
+#             "output_name": "flowfile_core",
+#             "hidden_imports": ["passlib.handlers.bcrypt"] + common_imports
+#         }
+#     ]
+#
+#     with ProcessPoolExecutor(max_workers=2) as executor:
+#         futures = [executor.submit(build_backend, **build) for build in builds]
+#         wait(futures)
+#
+#         for future in futures:
+#             if future.result() != 0:
+#                 raise Exception("One or more builds failed")
+#
+#
+# if __name__ == "__main__":
+#     main()

flowfile/__init__.py

@@ -0,0 +1,71 @@
+"""
+FlowFile: A framework combining visual ETL with a Polars-like API.
+
+This package ties together the FlowFile ecosystem components:
+- flowfile_core: Core ETL functionality
+- flowfile_frame: Polars-like DataFrame API
+- flowfile_worker: Computation engine
+"""
+
+__version__ = "0.2.1"
+
+# Import the key components from flowfile_frame
+from flowfile_frame.flow_frame import (
+    FlowFrame, read_csv, read_parquet, from_dict, concat
+)
+from flowfile_frame.expr import (
+    col, lit, column, cum_count, len,
+    sum, min, max, mean, count, when
+)
+from flowfile_frame.group_frame import GroupByFrame
+from flowfile_frame.utils import create_etl_graph, open_graph_in_editor
+from flowfile_frame.selectors import (
+    numeric, float_, integer, string, temporal,
+    datetime, date, time, duration, boolean,
+    categorical, object_, list_, struct, all_,
+    by_dtype, contains, starts_with, ends_with, matches
+)
+
+# Import Polars data types for convenience
+from polars.datatypes import (
+    Int8, Int16, Int32, Int64, Int128,
+    UInt8, UInt16, UInt32, UInt64,
+    Float32, Float64,
+    Boolean, String, Utf8, Binary, Null,
+    List, Array, Struct, Object,
+    Date, Time, Datetime, Duration,
+    Categorical, Decimal, Enum, Unknown,
+    DataType, DataTypeClass, Field
+)
+
+# Define what's publicly available from the package
+__all__ = [
+    # Core FlowFrame classes
+    'FlowFrame', 'GroupByFrame',
+
+    # Main creation functions
+    'read_csv', 'read_parquet', 'from_dict', 'concat',
+
+    # Expression API
+    'col', 'lit', 'column', 'cum_count', 'len',
+    'sum', 'min', 'max', 'mean', 'count', 'when',
+
+    # Selector utilities
+    'numeric', 'float_', 'integer', 'string', 'temporal',
+    'datetime', 'date', 'time', 'duration', 'boolean',
+    'categorical', 'object_', 'list_', 'struct', 'all_',
+    'by_dtype', 'contains', 'starts_with', 'ends_with', 'matches',
+
+    # Utilities
+    'create_etl_graph', 'open_graph_in_editor',
+
+    # Data types from Polars
+    'Int8', 'Int16', 'Int32', 'Int64', 'Int128',
+    'UInt8', 'UInt16', 'UInt32', 'UInt64',
+    'Float32', 'Float64',
+    'Boolean', 'String', 'Utf8', 'Binary', 'Null',
+    'List', 'Array', 'Struct', 'Object',
+    'Date', 'Time', 'Datetime', 'Duration',
+    'Categorical', 'Decimal', 'Enum', 'Unknown',
+    'DataType', 'DataTypeClass', 'Field',
+]

flowfile/__main__.py

@@ -0,0 +1,24 @@
+"""
+Main entry point for the FlowFile package.
+"""
+
+
+def main():
+    """
+    Display information about FlowFile when run directly as a module.
+    """
+    import flowfile
+
+    print(f"FlowFile v{flowfile.__version__}")
+    print("A framework combining visual ETL with a Polars-like API")
+    print("\nUsage examples:")
+    print("  import flowfile as ff")
+    print("  df = ff.read_csv('data.csv')")
+    print("  result = df.filter(ff.col('value') > 10)")
+    print("  result.write_csv('output.csv')")
+    print("\nFor visual ETL:")
+    print("  ff.open_graph_in_editor(result.to_graph())")
+
+
+if __name__ == "__main__":
+    main()

flowfile-0.2.2.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Edward van Eechoud
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.