Flowfile 0.3.0__py3-none-any.whl → 0.3.0.1__py3-none-any.whl

flowfile/__init__.py

@@ -7,9 +7,16 @@ This package ties together the FlowFile ecosystem components:
 - flowfile_worker: Computation engine
 """
 
-__version__ = "0.2.1"
+__version__ = "0.3.1"
 
-# Import the key components from flowfile_frame
+import os
+import logging
+
+os.environ['WORKER_PORT'] = "63578"
+os.environ['SINGLE_FILE_MODE'] = "1"
+
+from flowfile.web import start_server as start_web_ui
+from flowfile.api import open_graph_in_editor
 from flowfile_frame.flow_frame import (
     FlowFrame, read_csv, read_parquet, from_dict, concat
 )
@@ -18,7 +25,7 @@ from flowfile_frame.expr import (
     sum, min, max, mean, count, when
 )
 from flowfile_frame.group_frame import GroupByFrame
-from flowfile_frame.utils import create_flow_graph, open_graph_in_editor
+from flowfile_frame.utils import create_flow_graph
 from flowfile_frame.selectors import (
     numeric, float_, integer, string, temporal,
     datetime, date, time, duration, boolean,
@@ -26,7 +33,6 @@ from flowfile_frame.selectors import (
     by_dtype, contains, starts_with, ends_with, matches
 )
 
-# Import Polars data types for convenience
 from polars.datatypes import (
     Int8, Int16, Int32, Int64, Int128,
     UInt8, UInt16, UInt32, UInt64,
@@ -38,7 +44,6 @@ from polars.datatypes import (
     DataType, DataTypeClass, Field
 )
 
-# Define what's publicly available from the package
 __all__ = [
     # Core FlowFrame classes
     'FlowFrame', 'GroupByFrame',
@@ -68,4 +73,6 @@ __all__ = [
     'Date', 'Time', 'Datetime', 'Duration',
     'Categorical', 'Decimal', 'Enum', 'Unknown',
     'DataType', 'DataTypeClass', 'Field',
-]
+    'start_web_ui'
+]
+logging.getLogger("PipelineHandler").setLevel(logging.WARNING)

flowfile/__main__.py

@@ -1,24 +1,60 @@
-"""
-Main entry point for the FlowFile package.
-"""
-
+#  flowfile/__main__.py
 
 def main():
     """
     Display information about FlowFile when run directly as a module.
     """
     import flowfile
+    import argparse
 
-    print(f"FlowFile v{flowfile.__version__}")
-    print("A framework combining visual ETL with a Polars-like API")
-    print("\nUsage examples:")
-    print("  import flowfile as ff")
-    print("  df = ff.read_csv('data.csv')")
-    print("  result = df.filter(ff.col('value') > 10)")
-    print("  result.write_csv('output.csv')")
-    print("\nFor visual ETL:")
-    print("  ff.open_graph_in_editor(result.to_graph())")
+    parser = argparse.ArgumentParser(description="FlowFile: A visual ETL tool with a Polars-like API")
+    parser.add_argument("command", nargs="?", choices=["run"], help="Command to execute")
+    parser.add_argument("component", nargs="?", choices=["web", "core", "worker"],
+                        help="Component to run (web, core, or worker)")
+    parser.add_argument("--host", default="127.0.0.1", help="Host to bind the server to")
+    parser.add_argument("--port", type=int, default=63578, help="Port to bind the server to")
+    parser.add_argument("--no-browser", action="store_true", help="Don't open a browser window")
 
+    # Parse arguments
+    args = parser.parse_args()
 
-if __name__ == "__main__":
-    main()
+    if args.command == "run" and args.component:
+        if args.component == "web":
+            try:
+                flowfile.start_web_ui(
+                    host=args.host,
+                    port=args.port,
+                    open_browser=not args.no_browser
+                )
+            except KeyboardInterrupt:
+                print("\nFlowFile service stopped.")
+        elif args.component == "core":
+            # Only for direct core service usage
+            from flowfile_core.main import run as run_core
+            run_core(host=args.host, port=args.port)
+        elif args.component == "worker":
+            # Only for direct worker service usage
+            from flowfile_worker.main import run as run_worker
+            run_worker(host=args.host, port=args.port)
+    else:
+        # Default action - show info
+        print(f"FlowFile v{flowfile.__version__}")
+        print("A framework combining visual ETL with a Polars-like API")
+        print("\nUsage:")
+        print("  # Start the FlowFile web UI with integrated services")
+        print("  flowfile run web")
+        print("")
+        print("  # Advanced: Run individual components")
+        print("  flowfile run core  # Start only the core service")
+        print("  flowfile run worker  # Start only the worker service")
+        print("")
+        print("  # Options")
+        print("  flowfile run web --host 0.0.0.0 --port 8080  # Custom host/port")
+        print("  flowfile run web --no-browser  # Don't open browser")
+        print("")
+        print("  # Python API usage examples")
+        print("  import flowfile as ff")
+        print("  df = ff.read_csv('data.csv')")
+        print("  result = df.filter(ff.col('value') > 10)")
+        print("  ff.open_graph_in_editor(result)")
+        print("  ff.start_web_ui()")

flowfile/api.py

@@ -0,0 +1,379 @@
+# flowfile/api.py
+import uuid
+import time
+import os
+import requests
+import subprocess
+import sys
+import atexit
+import logging
+import webbrowser
+import shutil
+from pathlib import Path
+from typing import Optional, Dict, Any, Union, Tuple, List
+from subprocess import Popen
+from flowfile_core.flowfile.FlowfileFlow import FlowGraph
+from tempfile import TemporaryDirectory
+
+# Configuration
+FLOWFILE_HOST: str = os.environ.get("FLOWFILE_HOST", "127.0.0.1")
+FLOWFILE_PORT: int = int(os.environ.get("FLOWFILE_PORT", 63578))
+FLOWFILE_BASE_URL: str = f"http://{FLOWFILE_HOST}:{FLOWFILE_PORT}"
+DEFAULT_MODULE_NAME: str = os.environ.get("FLOWFILE_MODULE_NAME", "flowfile")
+FORCE_POETRY: bool = os.environ.get("FORCE_POETRY", "").lower() in ("true", "1", "yes")
+POETRY_PATH: str = os.environ.get("POETRY_PATH", "poetry")
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+# Global variable to track the managed server process
+_server_process: Optional[Popen] = None
+
+
+def is_flowfile_running() -> bool:
+    """Check if the Flowfile core API endpoint is responsive."""
+    try:
+        response: requests.Response = requests.get(f"{FLOWFILE_BASE_URL}/docs", timeout=1)
+        return 200 <= response.status_code < 300
+    except (requests.ConnectionError, requests.Timeout):
+        return False
+    except Exception as e:
+        logger.error(f"Unexpected error checking Flowfile status: {e}")
+        return False
+
+
+def stop_flowfile_server_process() -> None:
+    """Stop the Flowfile server process if it was started by this module."""
+    global _server_process
+    if _server_process and _server_process.poll() is None:
+        logger.info(f"Stopping managed Flowfile server process (PID: {_server_process.pid})...")
+        _server_process.terminate()
+        try:
+            _server_process.wait(timeout=5)
+            logger.info("Server process terminated gracefully.")
+        except subprocess.TimeoutExpired:
+            logger.warning("Server process did not terminate gracefully, killing...")
+            _server_process.kill()
+            _server_process.wait()
+            logger.info("Server process killed.")
+        except Exception as e:
+            logger.error(f"Error during server process termination: {e}")
+        finally:
+            _server_process = None
+
+
+def is_poetry_environment() -> bool:
+    """
+    Detect if we're running in a Poetry environment by checking:
+    1. If pyproject.toml exists up the directory tree
+    2. If VIRTUAL_ENV points to a poetry environment
+    3. If POETRY_ACTIVE environment variable is set
+    """
+    # Check if explicitly set via env variable
+    if FORCE_POETRY:
+        return True
+
+    # Check if POETRY_ACTIVE is set
+    if os.environ.get("POETRY_ACTIVE", "").lower() in ("true", "1", "yes"):
+        return True
+
+    # Check if we're in a poetry virtual environment
+    venv_path = os.environ.get("VIRTUAL_ENV", "")
+    if venv_path and (
+            "poetry" in venv_path.lower() or
+            Path(venv_path).joinpath(".poetry-venv").exists()
+    ):
+        return True
+
+    # Look for pyproject.toml with poetry section
+    cwd = Path.cwd()
+    for parent in [cwd, *cwd.parents]:
+        pyproject = parent / "pyproject.toml"
+        if pyproject.exists():
+            with open(pyproject, "r") as f:
+                content = f.read()
+                if "[tool.poetry]" in content:
+                    return True
+
+    return False
+
+
+def is_command_available(command: str) -> bool:
+    """Check if a command is available in the PATH."""
+    return shutil.which(command) is not None
+
+
+def build_server_command(module_name: str) -> List[str]:
+    """
+    Build the appropriate command to start the server based on environment detection.
+    Tries Poetry first if in a Poetry environment, falls back to direct module execution.
+    """
+    command: List[str] = []
+
+    # Case 1: Check if we're in a Poetry environment
+    if is_poetry_environment():
+        logger.info("Poetry environment detected.")
+        if is_command_available(POETRY_PATH):
+            logger.info(f"Using Poetry to run {module_name}")
+            command = [
+                POETRY_PATH,
+                "run",
+                module_name,
+                "run",
+                "web",
+                "--no-browser",
+                f"--port={FLOWFILE_PORT}",
+            ]
+            return command
+        else:
+            logger.warning(f"Poetry command not found at '{POETRY_PATH}'. Falling back to Python module.")
+
+    # Case 2: Try direct module execution
+    logger.info(f"Using Python module approach with {module_name}")
+    command = [
+        sys.executable,
+        "-m",
+        module_name,
+        "run",
+        "web",
+        "--no-browser",
+        f"--port={FLOWFILE_PORT}",
+    ]
+
+    return command
+
+
+def start_flowfile_server_process(module_name: str = DEFAULT_MODULE_NAME) -> bool:
+    """
+    Start the Flowfile server as a background process if it's not already running.
+    Automatically detects and uses Poetry if in a Poetry environment.
+
+    Parameters:
+        module_name: The module name to run. Defaults to the value from environment
+                    variable or "flowfile".
+    """
+    global _server_process
+    if is_flowfile_running():
+        return True
+
+    if _server_process and _server_process.poll() is None:
+        logger.warning("Server process object exists but API not responding. Attempting to restart.")
+        stop_flowfile_server_process()
+
+    # Build command automatically based on environment detection
+    command = build_server_command(module_name)
+    logger.info(f"Starting server with command: {' '.join(command)}")
+
+    try:
+        _server_process = Popen(
+            command,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.PIPE,
+        )
+        logger.info(f"Started server process with PID: {_server_process.pid}")
+
+        atexit.register(stop_flowfile_server_process)
+
+        logger.info("Waiting for server to initialize...")
+
+        for i in range(10):
+            time.sleep(1)
+            if is_flowfile_running():
+                logger.info("Server started successfully.")
+                return True
+        else:
+            logger.error("Failed to start server: API did not become responsive.")
+            if _server_process and _server_process.stderr:
+                try:
+                    stderr_output: str = _server_process.stderr.read().decode(errors='ignore')
+                    logger.error(f"Server process stderr:\n{stderr_output[:1000]}...")
+                except Exception as read_err:
+                    logger.error(f"Could not read stderr from server process: {read_err}")
+                    stop_flowfile_server_process()
+                    return False
+            else:
+                stop_flowfile_server_process()
+                return False
+
+    except FileNotFoundError:
+        logger.error(f"Error: Could not execute command: '{' '.join(command)}'.")
+        logger.error(f"Ensure '{module_name}' is installed correctly.")
+        _server_process = None
+        return False
+    except Exception as e:
+        logger.error(f"An unexpected error occurred while starting the server process: {e}")
+        if _server_process and _server_process.stderr:
+            try:
+                stderr_output = _server_process.stderr.read().decode(errors='ignore')
+                logger.error(f"Server process stderr:\n{stderr_output[:1000]}...")
+            except Exception as read_err:
+                logger.error(f"Could not read stderr from server process: {read_err}")
+        stop_flowfile_server_process()
+        _server_process = None
+        return False
+
+
+def get_auth_token() -> Optional[str]:
+    """Get an authentication token from the Flowfile API."""
+    try:
+        response: requests.Response = requests.post(
+            f"{FLOWFILE_BASE_URL}/auth/token",
+            json={},
+            timeout=5
+        )
+        response.raise_for_status()
+        token_data: Dict[str, Any] = response.json()
+        access_token: Optional[str] = token_data.get("access_token")
+        if not access_token:
+            logger.error("Auth token endpoint succeeded but 'access_token' was missing in response.")
+            return None
+        return access_token
+
+    except requests.exceptions.RequestException as e:
+        logger.error(f"Failed to get auth token: {e}")
+        return None
+    except Exception as e:
+        logger.error(f"An unexpected error occurred getting auth token: {e}")
+        return None
+
+
+def import_flow_to_editor(flow_path: Path, auth_token: str) -> Optional[int]:
+    """Import the flow into the Flowfile editor using the API endpoint."""
+    if not flow_path.is_file():
+        logger.error(f"Flow file not found: {flow_path}")
+        return None
+    if not auth_token:
+        logger.error("Cannot import flow without auth token.")
+        return None
+
+    try:
+        headers: Dict[str, str] = {"Authorization": f"Bearer {auth_token}"}
+        params: Dict[str, str] = {"flow_path": str(flow_path)}
+
+        response: requests.Response = requests.get(
+            f"{FLOWFILE_BASE_URL}/import_flow/",
+            params=params,
+            headers=headers,
+            timeout=10
+        )
+        response.raise_for_status()
+
+        flow_id_data: Union[int, Dict[str, Any], Any] = response.json()
+        flow_id: Optional[int] = None
+
+        if isinstance(flow_id_data, int):
+            flow_id = flow_id_data
+        elif isinstance(flow_id_data, dict) and "flow_id" in flow_id_data:
+            flow_id = int(flow_id_data["flow_id"])
+        else:
+            logger.error(f"Unexpected response format from import_flow endpoint: {flow_id_data}")
+            return None
+
+        logger.info(f"Flow '{flow_path.name}' imported successfully with ID: {flow_id}")
+        return flow_id
+
+    except requests.exceptions.RequestException as e:
+        logger.error(f"Failed to import flow: {e}")
+        if e.response is not None:
+            logger.error(f"Server response: {e.response.status_code} - {e.response.text[:500]}")
+        return None
+    except Exception as e:
+        logger.error(f"An unexpected error occurred importing flow: {e}")
+        return None
+
+
+def _save_flow_to_location(
+        flow_graph: FlowGraph, storage_location: Optional[str]
+) -> Tuple[Optional[Path], Optional[TemporaryDirectory]]:
+    """Handles graph saving, path resolution, and temporary directory creation."""
+    temp_dir_obj: Optional[TemporaryDirectory] = None
+    flow_file_path: Path
+    try:
+        if storage_location:
+            flow_file_path = Path(storage_location).resolve()
+            flow_file_path.parent.mkdir(parents=True, exist_ok=True)
+        else:
+            temp_dir_obj = TemporaryDirectory(prefix="flowfile_graph_")
+            flow_file_path = Path(temp_dir_obj.name) / f"temp_flow_{uuid.uuid4().hex[:8]}.flowfile"
+
+        logger.info(f"Applying layout and saving flow to: {flow_file_path}")
+        flow_graph.apply_layout()
+        flow_graph.save_flow(str(flow_file_path))
+        logger.info("Flow saved successfully.")
+        return flow_file_path, temp_dir_obj
+    except Exception as e:
+        logger.error(f"Failed to save flow graph to location '{storage_location}': {e}", exc_info=True)
+        if temp_dir_obj:
+            try:
+                temp_dir_obj.cleanup()
+                logger.info(f"Cleaned up temp dir {temp_dir_obj.name} after save failure.")
+            except Exception as cleanup_err:
+                logger.error(f"Error during immediate cleanup of temp dir: {cleanup_err}")
+        return None, None
+
+
+def _open_flow_in_browser(flow_id: int) -> None:
+    """Opens the specified flow ID in a browser tab if in unified mode."""
+    if os.environ.get("FLOWFILE_MODE") == "electron":
+        flow_url = f"http://{FLOWFILE_HOST}:{FLOWFILE_PORT}/web/flow/{flow_id}"
+        logger.info(f"Unified mode detected. Opening imported flow in browser: {flow_url}")
+        try:
+            time.sleep(0.5)
+            webbrowser.open_new_tab(flow_url)
+        except Exception as wb_err:
+            logger.warning(f"Could not automatically open browser tab: {wb_err}")
+    else:
+        logger.info("Not in unified mode ('electron'), browser will not be opened automatically.")
+
+
+def _cleanup_temporary_storage(temp_dir_obj: Optional[TemporaryDirectory]) -> None:
+    """Safely cleans up the temporary directory if one was created."""
+    if temp_dir_obj:
+        try:
+            temp_dir_obj.cleanup()
+            logger.info(f"Cleaned up temporary directory: {temp_dir_obj.name}")
+        except Exception as e:
+            logger.error(f"Error cleaning up temporary directory {temp_dir_obj.name}: {e}")
+
+
+def open_graph_in_editor(flow_graph: FlowGraph, storage_location: Optional[str] = None, module_name: str = DEFAULT_MODULE_NAME) -> bool:
+    """
+    Save the ETL graph, ensure the Flowfile server is running (starting it
+    if necessary), import the graph via API, and open it in a new browser
+    tab if running in unified mode.
+
+    Parameters:
+        flow_graph: The FlowGraph object to save and open.
+        storage_location: Optional path to save the .flowfile. If None,
+                          a temporary file is used.
+        module_name: The module name to run if server needs to be started.
+                    Use your Poetry package name if not using "flowfile".
+
+    Returns:
+        True if the graph was successfully imported, False otherwise.
+    """
+    temp_dir_obj: Optional[TemporaryDirectory] = None
+    try:
+        flow_file_path, temp_dir_obj = _save_flow_to_location(flow_graph, storage_location)
+        if not flow_file_path:
+            return False
+
+        if not start_flowfile_server_process(module_name):
+            return False
+
+        auth_token = get_auth_token()
+        if not auth_token:
+            return False
+
+        flow_id = import_flow_to_editor(flow_file_path, auth_token)
+
+        if flow_id is not None:
+            _open_flow_in_browser(flow_id)
+            return True
+        else:
+            return False
+
+    except Exception as e:
+        logger.error(f"An unexpected error occurred in open_graph_in_editor: {e}", exc_info=True)
+        return False
+    finally:
+        _cleanup_temporary_storage(temp_dir_obj)

flowfile/web/__init__.py

@@ -0,0 +1,155 @@
+"""
+flowfile/web/__init__.py
+Web interface for Flowfile.
+Extends the flowfile_core FastAPI app to serve the Vue.js frontend
+and includes worker functionality.
+"""
+import os
+import time
+from pathlib import Path
+import webbrowser
+import asyncio
+from fastapi import FastAPI, Response
+from fastapi.staticfiles import StaticFiles
+from fastapi.responses import FileResponse, RedirectResponse
+
+static_dir = Path(__file__).parent / "static"
+
+
+def extend_app(app: FastAPI):
+    """
+    Extend the flowfile_core FastAPI app with routes to serve the Vue.js frontend
+    and worker functionality.
+    """
+    # Serve static files if the directory exists
+    if static_dir.exists():
+        # Mount the assets directory
+        if (static_dir / "assets").exists():
+            app.mount("/assets", StaticFiles(directory=str(static_dir / "assets")), name="assets")
+
+        # Mount other common directories
+        for dir_name in ["css", "js", "img", "fonts", "icons", "images"]:
+            dir_path = static_dir / dir_name
+            if dir_path.exists() and dir_path.is_dir():
+                app.mount(f"/{dir_name}", StaticFiles(directory=str(dir_path)), name=dir_name)
+
+    @app.get("/favicon.ico", include_in_schema=False)
+    async def favicon():
+        """Serve the favicon.ico file"""
+        favicon_path = static_dir / "favicon.ico"
+        if favicon_path.exists():
+            return FileResponse(favicon_path)
+        return Response(status_code=404)
+
+    @app.get("/flowfile.svg", include_in_schema=False)
+    async def svg_logo():
+        """Serve the SVG logo file"""
+        svg_path = static_dir / "flowfile.svg"
+        if svg_path.exists():
+            return FileResponse(svg_path, media_type="image/svg+xml")
+        return Response(status_code=404)
+
+    @app.get("/test")
+    async def get_worker_host():
+        from flowfile_core.configs.settings import WORKER_URL
+        return WORKER_URL
+
+    @app.get("/web", include_in_schema=False)
+    async def web_ui_root():
+        """Serve the main index.html file for the web UI"""
+        index_path = static_dir / "index.html"
+        if index_path.exists():
+            return FileResponse(index_path)
+        return {"error": "Web UI not installed. Build the frontend and install it in the package."}
+
+    @app.get("/web/{path:path}", include_in_schema=False)
+    async def serve_vue_app(path: str):
+        """Serve static files or the index.html for client-side routing"""
+        # Try to serve the requested file
+        file_path = static_dir / path
+        if file_path.exists() and file_path.is_file():
+            return FileResponse(file_path)
+
+        # If it's a directory, redirect to add trailing slash
+        if (static_dir / path).exists() and (static_dir / path).is_dir():
+            return RedirectResponse(f"/web/{path}/")
+
+        # For client-side routing, serve the index.html
+        index_path = static_dir / "index.html"
+        if index_path.exists():
+            return FileResponse(index_path)
+
+        return {"error": f"File not found: {path}"}
+
+    # Include worker routes if simplified mode is enabled
+    include_worker_routes(app)
+
+    return app
+
+
+def include_worker_routes(app: FastAPI):
+    """
+    Include worker routes from flowfile_worker for simplified deployments.
+    This creates a unified API that serves both the web UI and processes the worker operations.
+    """
+    try:
+        # Import worker modules
+        from flowfile_worker.routes import router as worker_router
+        from flowfile_worker import mp_context, CACHE_DIR
+
+        # Add lifecycle event handler for worker cleanup
+        @app.on_event("shutdown")
+        async def shutdown_worker():
+            """Clean up worker resources on shutdown"""
+            print("Cleaning up worker resources...")
+            for p in mp_context.active_children():
+                try:
+                    p.terminate()
+                    p.join()
+                except Exception as e:
+                    print(f"Error cleaning up process: {e}")
+
+            try:
+                CACHE_DIR.cleanup()
+            except Exception as e:
+                print(f"Error cleaning up cache directory: {e}")
+
+            await asyncio.sleep(0.1)
+
+        # Include the worker router with a prefix
+        app.include_router(worker_router, prefix="/worker")
+
+        print("Worker functionality included in unified API")
+
+    except ImportError as e:
+        print(f"Worker module could not be imported, running without worker functionality: {e}")
+        print("This is normal for lightweight deployments that don't need data processing.")
+
+
+def start_server(host="127.0.0.1", port=63578, open_browser=True):
+    """
+    Start the flowfile_core FastAPI app with the web UI routes and worker functionality.
+    This function is a wrapper around flowfile_core.main.run().
+    """
+    # Set electron mode
+    os.environ["FLOWFILE_MODE"] = "electron"
+
+    # Import core app
+    from flowfile_core.main import run, app as core_app
+
+    # Extend the core app with web UI routes and worker functionality
+    extend_app(core_app)
+
+    # Open browser if requested
+    if open_browser:
+        time.sleep(2)
+        webbrowser.open_new_tab(f"http://{host}:{port}/web")
+
+    print("\n" + "=" * 60)
+    print("    FlowFile - Visual ETL Tool (Unified Mode)")
+    print(f"    Web UI: http://{host}:{port}/web")
+    print(f"    API Docs: http://{host}:{port}/docs")
+    print("=" * 60 + "\n")
+
+    # Run the core app
+    run(host=host, port=port)