Flowfile 0.3.0__py3-none-any.whl → 0.3.0.2__py3-none-any.whl

flowfile/__init__.py

@@ -7,9 +7,16 @@ This package ties together the FlowFile ecosystem components:
 - flowfile_worker: Computation engine
 """
 
-__version__ = "0.2.1"
+__version__ = "0.3.1"
 
-# Import the key components from flowfile_frame
+import os
+import logging
+
+os.environ['WORKER_PORT'] = "63578"
+os.environ['SINGLE_FILE_MODE'] = "1"
+
+from flowfile.web import start_server as start_web_ui
+from flowfile.api import open_graph_in_editor
 from flowfile_frame.flow_frame import (
     FlowFrame, read_csv, read_parquet, from_dict, concat
 )
@@ -18,7 +25,7 @@ from flowfile_frame.expr import (
     sum, min, max, mean, count, when
 )
 from flowfile_frame.group_frame import GroupByFrame
-from flowfile_frame.utils import create_flow_graph, open_graph_in_editor
+from flowfile_frame.utils import create_flow_graph
 from flowfile_frame.selectors import (
     numeric, float_, integer, string, temporal,
     datetime, date, time, duration, boolean,
@@ -26,7 +33,6 @@ from flowfile_frame.selectors import (
     by_dtype, contains, starts_with, ends_with, matches
 )
 
-# Import Polars data types for convenience
 from polars.datatypes import (
     Int8, Int16, Int32, Int64, Int128,
     UInt8, UInt16, UInt32, UInt64,
@@ -38,7 +44,6 @@ from polars.datatypes import (
     DataType, DataTypeClass, Field
 )
 
-# Define what's publicly available from the package
 __all__ = [
     # Core FlowFrame classes
     'FlowFrame', 'GroupByFrame',
@@ -68,4 +73,6 @@ __all__ = [
     'Date', 'Time', 'Datetime', 'Duration',
     'Categorical', 'Decimal', 'Enum', 'Unknown',
     'DataType', 'DataTypeClass', 'Field',
-]
+    'start_web_ui'
+]
+logging.getLogger("PipelineHandler").setLevel(logging.WARNING)

flowfile/__main__.py

@@ -1,24 +1,59 @@
-"""
-Main entry point for the FlowFile package.
-"""
-
+#  flowfile/__main__.py
 
 def main():
     """
     Display information about FlowFile when run directly as a module.
     """
     import flowfile
+    import argparse
 
-    print(f"FlowFile v{flowfile.__version__}")
-    print("A framework combining visual ETL with a Polars-like API")
-    print("\nUsage examples:")
-    print("  import flowfile as ff")
-    print("  df = ff.read_csv('data.csv')")
-    print("  result = df.filter(ff.col('value') > 10)")
-    print("  result.write_csv('output.csv')")
-    print("\nFor visual ETL:")
-    print("  ff.open_graph_in_editor(result.to_graph())")
+    parser = argparse.ArgumentParser(description="FlowFile: A visual ETL tool with a Polars-like API")
+    parser.add_argument("command", nargs="?", choices=["run"], help="Command to execute")
+    parser.add_argument("component", nargs="?", choices=["web", "core", "worker"],
+                        help="Component to run (web, core, or worker)")
+    parser.add_argument("--host", default="127.0.0.1", help="Host to bind the server to")
+    parser.add_argument("--port", type=int, default=63578, help="Port to bind the server to")
+    parser.add_argument("--no-browser", action="store_true", help="Don't open a browser window")
 
+    # Parse arguments
+    args = parser.parse_args()
 
-if __name__ == "__main__":
-    main()
+    if args.command == "run" and args.component:
+        if args.component == "ui":
+            try:
+                flowfile.start_web_ui(
+                    host=args.host,
+                    port=args.port,
+                    open_browser=not args.no_browser
+                )
+            except KeyboardInterrupt:
+                print("\nFlowFile service stopped.")
+        elif args.component == "core":
+            # Only for direct core service usage
+            from flowfile_core.main import run as run_core
+            run_core(host=args.host, port=args.port)
+        elif args.component == "worker":
+            # Only for direct worker service usage
+            from flowfile_worker.main import run as run_worker
+            run_worker(host=args.host, port=args.port)
+    else:
+        # Default action - show info
+        print(f"FlowFile v{flowfile.__version__}")
+        print("A framework combining visual ETL with a Polars-like API")
+        print("\nUsage:")
+        print("  # Start the FlowFile web UI with integrated services")
+        print("  flowfile run ui")
+        print("")
+        print("  # Advanced: Run individual components")
+        print("  flowfile run core  # Start only the core service")
+        print("  flowfile run worker  # Start only the worker service")
+        print("")
+        print("  # Options")
+        print("  flowfile run ui --host 0.0.0.0 --port 8080  # Custom host/port")
+        print("  flowfile run ui --no-browser  # Don't open browser")
+        print("")
+        print("  # Python API usage examples")
+        print("  import flowfile as ff")
+        print("  df = ff.read_csv('data.csv')")
+        print("  result = df.filter(ff.col('value') > 10)")
+        print("  ff.open_graph_in_editor(result)")

flowfile/api.py

@@ -0,0 +1,383 @@
+# flowfile/api.py
+import uuid
+import time
+import os
+import requests
+import subprocess
+import sys
+import atexit
+import logging
+import webbrowser
+import shutil
+from pathlib import Path
+from typing import Optional, Dict, Any, Union, Tuple, List
+from subprocess import Popen
+from flowfile_core.flowfile.FlowfileFlow import FlowGraph
+from tempfile import TemporaryDirectory
+
+# Configuration
+FLOWFILE_HOST: str = os.environ.get("FLOWFILE_HOST", "127.0.0.1")
+FLOWFILE_PORT: int = int(os.environ.get("FLOWFILE_PORT", 63578))
+FLOWFILE_BASE_URL: str = f"http://{FLOWFILE_HOST}:{FLOWFILE_PORT}"
+DEFAULT_MODULE_NAME: str = os.environ.get("FLOWFILE_MODULE_NAME", "flowfile")
+FORCE_POETRY: bool = os.environ.get("FORCE_POETRY", "").lower() in ("true", "1", "yes")
+POETRY_PATH: str = os.environ.get("POETRY_PATH", "poetry")
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+# Global variable to track the managed server process
+_server_process: Optional[Popen] = None
+
+
+def is_flowfile_running() -> bool:
+    """Check if the Flowfile core API endpoint is responsive."""
+    try:
+        response: requests.Response = requests.get(f"{FLOWFILE_BASE_URL}/docs", timeout=1)
+        return 200 <= response.status_code < 300
+    except (requests.ConnectionError, requests.Timeout):
+        return False
+    except Exception as e:
+        logger.error(f"Unexpected error checking Flowfile status: {e}")
+        return False
+
+
+def stop_flowfile_server_process() -> None:
+    """Stop the Flowfile server process if it was started by this module."""
+    global _server_process
+    if _server_process and _server_process.poll() is None:
+        logger.info(f"Stopping managed Flowfile server process (PID: {_server_process.pid})...")
+        _server_process.terminate()
+        try:
+            _server_process.wait(timeout=5)
+            logger.info("Server process terminated gracefully.")
+        except subprocess.TimeoutExpired:
+            logger.warning("Server process did not terminate gracefully, killing...")
+            _server_process.kill()
+            _server_process.wait()
+            logger.info("Server process killed.")
+        except Exception as e:
+            logger.error(f"Error during server process termination: {e}")
+        finally:
+            _server_process = None
+
+
+def is_poetry_environment() -> bool:
+    """
+    Detect if we're running in a Poetry environment by checking:
+    1. If pyproject.toml exists up the directory tree
+    2. If VIRTUAL_ENV points to a poetry environment
+    3. If POETRY_ACTIVE environment variable is set
+    """
+    # Check if explicitly set via env variable
+    if FORCE_POETRY:
+        return True
+
+    # Check if POETRY_ACTIVE is set
+    if os.environ.get("POETRY_ACTIVE", "").lower() in ("true", "1", "yes"):
+        return True
+
+    # Check if we're in a poetry virtual environment
+    venv_path = os.environ.get("VIRTUAL_ENV", "")
+    if venv_path and (
+            "poetry" in venv_path.lower() or
+            Path(venv_path).joinpath(".poetry-venv").exists()
+    ):
+        return True
+
+    # Look for pyproject.toml with poetry section
+    cwd = Path.cwd()
+    for parent in [cwd, *cwd.parents]:
+        pyproject = parent / "pyproject.toml"
+        if pyproject.exists():
+            with open(pyproject, "r") as f:
+                content = f.read()
+                if "[tool.poetry]" in content:
+                    return True
+
+    return False
+
+
+def is_command_available(command: str) -> bool:
+    """Check if a command is available in the PATH."""
+    return shutil.which(command) is not None
+
+
+def build_server_command(module_name: str) -> List[str]:
+    """
+    Build the appropriate command to start the server based on environment detection.
+    Tries Poetry first if in a Poetry environment, falls back to direct module execution.
+    """
+    command: List[str] = []
+
+    # Case 1: Check if we're in a Poetry environment
+    if is_poetry_environment():
+        logger.info("Poetry environment detected.")
+        if is_command_available(POETRY_PATH):
+            logger.info(f"Using Poetry to run {module_name}")
+            command = [
+                POETRY_PATH,
+                "run",
+                module_name,
+                "run",
+                "ui",
+                "--no-browser",
+                f"--port={FLOWFILE_PORT}",
+            ]
+            return command
+        else:
+            logger.warning(f"Poetry command not found at '{POETRY_PATH}'. Falling back to Python module.")
+
+    # Case 2: Try direct module execution
+    logger.info(f"Using Python module approach with {module_name}")
+    command = [
+        sys.executable,
+        "-m",
+        module_name,
+        "run",
+        "ui",
+        "--no-browser",
+        f"--port={FLOWFILE_PORT}",
+    ]
+
+    return command
+
+
+def start_flowfile_server_process(module_name: str = DEFAULT_MODULE_NAME) -> bool:
+    """
+    Start the Flowfile server as a background process if it's not already running.
+    Automatically detects and uses Poetry if in a Poetry environment.
+
+    Parameters:
+        module_name: The module name to run. Defaults to the value from environment
+                    variable or "flowfile".
+    """
+    global _server_process
+    if is_flowfile_running():
+        return True
+
+    if _server_process and _server_process.poll() is None:
+        logger.warning("Server process object exists but API not responding. Attempting to restart.")
+        stop_flowfile_server_process()
+
+    # Build command automatically based on environment detection
+    command = build_server_command(module_name)
+    logger.info(f"Starting server with command: {' '.join(command)}")
+
+    try:
+        _server_process = Popen(
+            command,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.PIPE,
+        )
+        logger.info(f"Started server process with PID: {_server_process.pid}")
+
+        atexit.register(stop_flowfile_server_process)
+
+        logger.info("Waiting for server to initialize...")
+
+        for i in range(10):
+            time.sleep(1)
+            if is_flowfile_running():
+                logger.info("Server started successfully.")
+                return True
+        else:
+            logger.error("Failed to start server: API did not become responsive.")
+            if _server_process and _server_process.stderr:
+                try:
+                    stderr_output: str = _server_process.stderr.read().decode(errors='ignore')
+                    logger.error(f"Server process stderr:\n{stderr_output[:1000]}...")
+                except Exception as read_err:
+                    logger.error(f"Could not read stderr from server process: {read_err}")
+                    stop_flowfile_server_process()
+                    return False
+            else:
+                stop_flowfile_server_process()
+                return False
+
+    except FileNotFoundError:
+        logger.error(f"Error: Could not execute command: '{' '.join(command)}'.")
+        logger.error(f"Ensure '{module_name}' is installed correctly.")
+        _server_process = None
+        return False
+    except Exception as e:
+        logger.error(f"An unexpected error occurred while starting the server process: {e}")
+        if _server_process and _server_process.stderr:
+            try:
+                stderr_output = _server_process.stderr.read().decode(errors='ignore')
+                logger.error(f"Server process stderr:\n{stderr_output[:1000]}...")
+            except Exception as read_err:
+                logger.error(f"Could not read stderr from server process: {read_err}")
+        stop_flowfile_server_process()
+        _server_process = None
+        return False
+
+
+def get_auth_token() -> Optional[str]:
+    """Get an authentication token from the Flowfile API."""
+    try:
+        response: requests.Response = requests.post(
+            f"{FLOWFILE_BASE_URL}/auth/token",
+            json={},
+            timeout=5
+        )
+        response.raise_for_status()
+        token_data: Dict[str, Any] = response.json()
+        access_token: Optional[str] = token_data.get("access_token")
+        if not access_token:
+            logger.error("Auth token endpoint succeeded but 'access_token' was missing in response.")
+            return None
+        return access_token
+
+    except requests.exceptions.RequestException as e:
+        logger.error(f"Failed to get auth token: {e}")
+        return None
+    except Exception as e:
+        logger.error(f"An unexpected error occurred getting auth token: {e}")
+        return None
+
+
+def import_flow_to_editor(flow_path: Path, auth_token: str) -> Optional[int]:
+    """Import the flow into the Flowfile editor using the API endpoint."""
+    if not flow_path.is_file():
+        logger.error(f"Flow file not found: {flow_path}")
+        return None
+    if not auth_token:
+        logger.error("Cannot import flow without auth token.")
+        return None
+
+    try:
+        headers: Dict[str, str] = {"Authorization": f"Bearer {auth_token}"}
+        params: Dict[str, str] = {"flow_path": str(flow_path)}
+
+        response: requests.Response = requests.get(
+            f"{FLOWFILE_BASE_URL}/import_flow/",
+            params=params,
+            headers=headers,
+            timeout=10
+        )
+        response.raise_for_status()
+
+        flow_id_data: Union[int, Dict[str, Any], Any] = response.json()
+        flow_id: Optional[int] = None
+
+        if isinstance(flow_id_data, int):
+            flow_id = flow_id_data
+        elif isinstance(flow_id_data, dict) and "flow_id" in flow_id_data:
+            flow_id = int(flow_id_data["flow_id"])
+        else:
+            logger.error(f"Unexpected response format from import_flow endpoint: {flow_id_data}")
+            return None
+
+        logger.info(f"Flow '{flow_path.name}' imported successfully with ID: {flow_id}")
+        return flow_id
+
+    except requests.exceptions.RequestException as e:
+        logger.error(f"Failed to import flow: {e}")
+        if e.response is not None:
+            logger.error(f"Server response: {e.response.status_code} - {e.response.text[:500]}")
+        return None
+    except Exception as e:
+        logger.error(f"An unexpected error occurred importing flow: {e}")
+        return None
+
+
+def _save_flow_to_location(
+        flow_graph: FlowGraph, storage_location: Optional[str]
+) -> Tuple[Optional[Path], Optional[TemporaryDirectory]]:
+    """Handles graph saving, path resolution, and temporary directory creation."""
+    temp_dir_obj: Optional[TemporaryDirectory] = None
+    flow_file_path: Path
+    try:
+        if storage_location:
+            flow_file_path = Path(storage_location).resolve()
+            flow_file_path.parent.mkdir(parents=True, exist_ok=True)
+        else:
+            temp_dir_obj = TemporaryDirectory(prefix="flowfile_graph_")
+            flow_file_path = Path(temp_dir_obj.name) / f"temp_flow_{uuid.uuid4().hex[:8]}.flowfile"
+
+        logger.info(f"Applying layout and saving flow to: {flow_file_path}")
+        flow_graph.apply_layout()
+        flow_graph.save_flow(str(flow_file_path))
+        logger.info("Flow saved successfully.")
+        return flow_file_path, temp_dir_obj
+    except Exception as e:
+        logger.error(f"Failed to save flow graph to location '{storage_location}': {e}", exc_info=True)
+        if temp_dir_obj:
+            try:
+                temp_dir_obj.cleanup()
+                logger.info(f"Cleaned up temp dir {temp_dir_obj.name} after save failure.")
+            except Exception as cleanup_err:
+                logger.error(f"Error during immediate cleanup of temp dir: {cleanup_err}")
+        return None, None
+
+
+def _open_flow_in_browser(flow_id: int) -> None:
+    """Opens the specified flow ID in a browser tab if in unified mode."""
+    if os.environ.get("FLOWFILE_MODE") == "electron":
+        flow_url = f"http://{FLOWFILE_HOST}:{FLOWFILE_PORT}/ui/flow/{flow_id}"
+        logger.info(f"Unified mode detected. Opening imported flow in browser: {flow_url}")
+        try:
+            time.sleep(0.5)
+            webbrowser.open_new_tab(flow_url)
+        except Exception as wb_err:
+            logger.warning(f"Could not automatically open browser tab: {wb_err}")
+    else:
+        logger.info("Not in unified mode ('electron'), browser will not be opened automatically.")
+
+
+def _cleanup_temporary_storage(temp_dir_obj: Optional[TemporaryDirectory]) -> None:
+    """Safely cleans up the temporary directory if one was created."""
+    if temp_dir_obj:
+        try:
+            temp_dir_obj.cleanup()
+            logger.info(f"Cleaned up temporary directory: {temp_dir_obj.name}")
+        except Exception as e:
+            logger.error(f"Error cleaning up temporary directory {temp_dir_obj.name}: {e}")
+
+
+def open_graph_in_editor(flow_graph: FlowGraph, storage_location: Optional[str] = None,
+                         module_name: str = DEFAULT_MODULE_NAME) -> bool:
+    """
+    Save the ETL graph, ensure the Flowfile server is running (starting it
+    if necessary), import the graph via API, and open it in a new browser
+    tab if running in unified mode.
+
+    Parameters:
+        flow_graph: The FlowGraph object to save and open.
+        storage_location: Optional path to save the .flowfile. If None,
+                          a temporary file is used.
+        module_name: The module name to run if server needs to be started.
+                    Use your Poetry package name if not using "flowfile".
+
+    Returns:
+        True if the graph was successfully imported, False otherwise.
+    """
+    temp_dir_obj: Optional[TemporaryDirectory] = None
+    try:
+        original_execution_settings = flow_graph.flow_settings.model_copy()
+        flow_graph.flow_settings.execution_location = "auto"
+        flow_graph.flow_settings.execution_mode = "Development"
+        flow_file_path, temp_dir_obj = _save_flow_to_location(flow_graph, storage_location)
+        if not flow_file_path:
+            return False
+        flow_graph.flow_settings = original_execution_settings
+        if not start_flowfile_server_process(module_name):
+            return False
+
+        auth_token = get_auth_token()
+        if not auth_token:
+            return False
+
+        flow_id = import_flow_to_editor(flow_file_path, auth_token)
+
+        if flow_id is not None:
+            _open_flow_in_browser(flow_id)
+            return True
+        else:
+            return False
+
+    except Exception as e:
+        logger.error(f"An unexpected error occurred in open_graph_in_editor: {e}", exc_info=True)
+        return False
+    finally:
+        _cleanup_temporary_storage(temp_dir_obj)

flowfile/readme.md

@@ -0,0 +1,130 @@
+# Flowfile Web UI Documentation
+
+## Overview
+
+Flowfile now supports a web-based user interface that can be launched directly from the pip-installed package. This enhancement allows users to quickly get started with the visual ETL tool without needing to install the desktop application, set up Docker, or manually configure the services.
+
+## Key Features
+
+- **Integrated Web UI**: Launch the Flowfile interface directly in your browser
+- **Unified Service**: Combined API that serves both the web UI and processes worker operations
+- **Easy Installation**: Simple pip installation and startup process
+- **Visual ETL**: Access to all the visual ETL capabilities through a web interface
+
+## Installation
+
+Install Flowfile from PyPI using pip:
+
+```bash
+pip install Flowfile
+```
+
+## Starting the Web UI
+
+You can start the Flowfile web UI using either the Python module or the command-line interface:
+
+### Using the Command-Line Interface
+
+```bash
+# Start the web UI with default settings
+flowfile run ui
+
+# Customize host and port
+flowfile run ui --host 0.0.0.0 --port 8080
+
+# Start without automatically opening a browser window
+flowfile run ui --no-browser
+```
+
+### Using Python
+
+```python
+import flowfile
+
+# Start the web UI with default settings
+flowfile.start_web_ui()
+
+# Customize host, port, and browser launch
+flowfile.start_web_ui(host="0.0.0.0", port=8080, open_browser=False)
+```
+
+## Architecture Overview
+
+The web UI functionality combines multiple components:
+
+1. **Core Service**: The main ETL engine (flowfile_core) that processes data transformations
+2. **Worker Service**: Handles computation and caching of data operations (flowfile_worker)
+3. **Web UI**: A Vue.js frontend that provides the visual interface
+
+When you start the web UI, all these services are launched together in a unified mode, making it simple to get started without configuration.
+
+## Using the Web UI with FlowFrame API
+
+You can create data pipelines programmatically with the FlowFrame API and then visualize them in the web UI:
+
+```python
+import flowfile as ff
+from flowfile import open_graph_in_editor
+
+# Create a data pipeline
+df = ff.from_dict({
+    "id": [1, 2, 3, 4, 5],
+    "category": ["A", "B", "A", "C", "B"],
+    "value": [100, 200, 150, 300, 250]
+})
+
+# Process the data
+result = df.filter(ff.col("value") > 150).with_columns([
+    (ff.col("value") * 2).alias("double_value")
+])
+
+# Open the graph in the web UI (starts the server if it's not running)
+open_graph_in_editor(result.flow_graph)
+```
+
+The `open_graph_in_editor` function automatically:
+1. Saves the flow graph to a temporary file
+2. Starts the Flowfile server if it's not already running
+3. Imports the flow into the editor
+4. Opens a browser tab with the imported flow
+
+## Advanced Server Configuration
+
+For advanced users who need to customize the server behavior:
+
+### Environment Variables
+
+- `FLOWFILE_HOST`: Host to bind the server to (default: "127.0.0.1")
+- `FLOWFILE_PORT`: Port to bind the server to (default: 63578)
+- `FLOWFILE_MODE`: Set to "electron" to enable browser auto-opening behavior
+- `WORKER_URL`: URL for the worker service
+- `SINGLE_FILE_MODE`: Set to "1" to run in unified mode with worker functionality
+- `FLOWFILE_MODULE_NAME`: Module name to run (default: "flowfile")
+
+### Running Individual Components
+
+For development or specialized deployments, you can run the components separately:
+
+```bash
+# Run only the core service
+flowfile run core --host 0.0.0.0 --port 8080
+
+# Run only the worker service
+flowfile run worker --host 0.0.0.0 --port 8081
+```
+
+## Troubleshooting
+
+- If the web UI doesn't open automatically, manually navigate to http://localhost:63578/ui
+- If you encounter connection issues, check if the port is already in use
+- Look for server logs in the terminal where you started the service for error messages
+- For issues with the API, navigate to http://localhost:63578/docs to verify the API is running
+
+## Next Steps
+
+Once you're familiar with the web UI, you might want to explore:
+
+1. The desktop application for a more native experience
+2. Docker deployment for production environments
+3. Advanced ETL operations using the FlowFrame API
+4. Custom node development for specialized transformations