Flowfile 0.3.9__py3-none-any.whl → 0.5.1__py3-none-any.whl

shared/storage_config.py

@@ -0,0 +1,258 @@
+# shared/storage_config.py - Updated for Option 3
+"""
+Centralized storage configuration for Flowfile.
+This module can be imported by both core and worker without creating dependencies.
+"""
+import os
+from pathlib import Path
+from typing import Optional, Literal
+
+DirectoryOptions = Literal["temp_directory", "logs_directory",
+                            "system_logs_directory", "database_directory",
+                            "cache_directory", "flows_directory", "user_defined_nodes_directory"]
+
+
+class FlowfileStorage:
+    """Centralized storage manager for Flowfile applications."""
+
+    def __init__(self):
+        self._base_dir: Optional[Path] = None
+        self._user_data_dir: Optional[Path] = None
+        self._ensure_directories()
+
+    @property
+    def base_directory(self) -> Path:
+        """Get the base Flowfile storage directory (for internal container communication)."""
+        if self._base_dir is None:
+            if os.environ.get("RUNNING_IN_DOCKER") == "true":
+                # In Docker, internal storage stays inside /app
+                base_path = os.environ.get("FLOWFILE_STORAGE_DIR", "/app/internal_storage")
+            else:
+                # Local development
+                base_path = os.environ.get("FLOWFILE_STORAGE_DIR")
+                if not base_path:
+                    home_dir = Path.home()
+                    base_path = home_dir / ".flowfile"
+
+            self._base_dir = Path(base_path)
+        return self._base_dir
+
+    @property
+    def user_data_directory(self) -> Path:
+        """Get the user data directory (completely separate from application code)."""
+        if self._user_data_dir is None:
+            if os.environ.get("RUNNING_IN_DOCKER") == "true":
+                # In Docker, user data is at /data/user (completely outside /app)
+                user_data_path = os.environ.get("FLOWFILE_USER_DATA_DIR", "/data/user")
+            else:
+                # Local development - use user's home directory
+                user_data_path = Path.home()
+
+            self._user_data_dir = Path(user_data_path)
+        return self._user_data_dir
+
+    @property
+    def cache_directory(self) -> Path:
+        """Cache directory for worker-core communication (internal)."""
+        return self.base_directory / "cache"
+
+    def get_flow_cache_directory(self, flow_id: int) -> Path:
+        """Get or create a cache directory for a specific flow (internal)."""
+        flow_cache_dir = self.cache_directory / str(flow_id)
+        flow_cache_dir.mkdir(parents=True, exist_ok=True)
+        return flow_cache_dir
+
+    @property
+    def system_logs_directory(self) -> Path:
+        """Directory for system logs (internal)."""
+        return self.base_directory / "system_logs"
+
+    @property
+    def flows_directory(self) -> Path:
+        """Directory for flow storage (user-accessible)."""
+        if os.environ.get("RUNNING_IN_DOCKER") == "true":
+            # In Docker, flows are in separate user data area
+            return self.user_data_directory / "flows"
+        else:
+            # Local development - flows in ~/.flowfile/flows
+            return self.base_directory / "flows"
+
+    @property
+    def uploads_directory(self) -> Path:
+        """Directory for user uploads (user-accessible)."""
+        if os.environ.get("RUNNING_IN_DOCKER") == "true":
+            # In Docker, uploads are in separate user data area
+            return self.user_data_directory / "uploads"
+        else:
+            # Local development - uploads in ~/.flowfile/uploads
+            return self.base_directory / "uploads"
+
+    @property
+    def user_defined_nodes_directory(self) -> Path:
+        """Directory for user-defined custom nodes (user-accessible)."""
+        if os.environ.get("RUNNING_IN_DOCKER") == "true":
+            return self.user_data_directory / "user_defined_nodes"
+        else:
+            return self.base_directory / "user_defined_nodes"
+
+    @property
+    def user_defined_nodes_icons(self) -> Path:
+        """Directory for user-defined custom node icon (user-accessible)."""
+        return self.user_defined_nodes_directory / "icons"
+
+    @property
+    def outputs_directory(self) -> Path:
+        """Directory for user outputs (user-accessible)."""
+        if os.environ.get("RUNNING_IN_DOCKER") == "true":
+            # In Docker, outputs are in separate user data area
+            return self.user_data_directory / "outputs"
+        else:
+            # Local development - outputs in ~/.flowfile/outputs
+            return self.base_directory / "outputs"
+
+    @property
+    def database_directory(self) -> Path:
+        """Directory for local database files (internal)."""
+        return self.base_directory / "database"
+
+    @property
+    def logs_directory(self) -> Path:
+        """Directory for application logs (internal)."""
+        return self.base_directory / "logs"
+
+    @property
+    def temp_directory(self) -> Path:
+        """Directory for temporary files (internal)."""
+        return self.base_directory / "temp"
+
+    @property
+    def temp_directory_for_flows(self) -> Path:
+        """Directory for temporary files specific to flows (internal)."""
+        return self.temp_directory / "flows"
+
+    def _ensure_directories(self) -> None:
+        """Create all necessary directories if they don't exist."""
+        # Internal directories (always created in base_directory)
+        internal_directories = [
+            self.cache_directory,
+            self.database_directory,
+            self.logs_directory,
+            self.temp_directory,
+            self.system_logs_directory,
+            self.temp_directory_for_flows,
+        ]
+
+        # User-accessible directories (location depends on environment)
+        user_directories = [
+            self.flows_directory,
+            self.uploads_directory,
+            self.outputs_directory,
+            self.user_defined_nodes_directory,
+            self.user_defined_nodes_icons,
+        ]
+
+        for directory in internal_directories + user_directories:
+            directory.mkdir(parents=True, exist_ok=True)
+
+    def get_cache_file_path(self, filename: str) -> Path:
+        """Get full path for a cache file (internal)."""
+        return self.cache_directory / filename
+
+    def get_flow_file_path(self, filename: str) -> Path:
+        """Get full path for a flow file (user-accessible)."""
+        return self.flows_directory / filename
+
+    def get_upload_file_path(self, filename: str) -> Path:
+        """Get full path for an uploaded file (user-accessible)."""
+        return self.uploads_directory / filename
+
+    def get_output_file_path(self, filename: str) -> Path:
+        """Get full path for an output file (user-accessible)."""
+        return self.outputs_directory / filename
+
+    def get_log_file_path(self, filename: str) -> Path:
+        """Get full path for an application log file (internal)."""
+        return self.logs_directory / filename
+
+    def get_system_log_file_path(self, filename: str) -> Path:
+        """Get full path for a system log file (internal)."""
+        return self.system_logs_directory / filename
+
+    def get_temp_file_path(self, filename: str) -> Path:
+        """Get full path for a temporary file (internal)."""
+        return self.temp_directory / filename
+
+    def cleanup_directory(self, directory_option: DirectoryOptions, storage_duration_hours: int = 24) -> None:
+        """Clean up any directory of the folder"""
+        import time
+        import shutil
+
+        if not hasattr(self, directory_option):
+            raise Exception(f"Directory does not exist in {self.base_directory}")
+
+        directory = getattr(self, directory_option)
+        if not isinstance(directory, Path):
+            raise Exception(f"Directory attribute {directory_option} is not a Path object")
+
+        if not directory.exists():
+            return
+
+        current_time = time.time()
+        cutoff_time = current_time - (storage_duration_hours * 60 * 60)
+
+        for item in directory.iterdir():
+            try:
+                if item.stat().st_mtime < cutoff_time:
+                    if item.is_file():
+                        item.unlink()
+                    elif item.is_dir():
+                        shutil.rmtree(item)
+            except (OSError, FileNotFoundError):
+                # Handle permission errors or files that disappeared
+                continue
+
+    def cleanup_directories(self) -> None:
+        """Clean up temporary files older than specified hours."""
+        self.cleanup_directory("temp_directory", storage_duration_hours=24)
+        self.cleanup_directory("cache_directory", storage_duration_hours=1)
+        self.cleanup_directory("logs_directory", storage_duration_hours=168)
+        self.cleanup_directory("system_logs_directory", storage_duration_hours=168)
+
+
+storage = FlowfileStorage()
+
+
+# Convenience functions for backward compatibility
+def get_cache_directory() -> str:
+    """Get cache directory path as string."""
+    return str(storage.cache_directory)
+
+
+def get_temp_directory() -> str:
+    """Get temp directory path as string."""
+    return str(storage.temp_directory)
+
+
+def get_flows_directory() -> str:
+    """Get flows directory path as string."""
+    return str(storage.flows_directory)
+
+
+def get_uploads_directory() -> str:
+    """Get uploads directory path as string."""
+    return str(storage.uploads_directory)
+
+
+def get_outputs_directory() -> str:
+    """Get outputs directory path as string."""
+    return str(storage.outputs_directory)
+
+
+def get_logs_directory() -> str:
+    """Get application logs directory path as string."""
+    return str(storage.logs_directory)
+
+
+def get_system_logs_directory() -> str:
+    """Get system logs directory path as string."""
+    return str(storage.system_logs_directory)

tools/migrate/README.md

@@ -0,0 +1,56 @@
+# Flowfile Migration Tool
+
+Migrates `.flowfile` (pickle format, used in v0.4.1 and earlier) to YAML (v0.5+).
+
+## Installation
+
+```bash
+pip install pyyaml  # Required for YAML output
+```
+
+## Usage
+
+```bash
+# Single file
+python -m tools.migrate path/to/flow.flowfile
+
+# Directory (recursive)
+python -m tools.migrate ./flows/
+
+# Specify output path
+python -m tools.migrate flow.flowfile -o /output/path/flow.yaml
+
+# Output as JSON
+python -m tools.migrate flow.flowfile --format json
+
+# Dry run
+python -m tools.migrate ./flows/ --dry-run
+
+# Verbose (show tracebacks)
+python -m tools.migrate flow.flowfile -v
+```
+
+## Output Structure
+
+```yaml
+_version: '2.0'
+_migrated_from: pickle
+flow_id: 1
+flow_name: my_analysis
+flow_settings:
+  name: my_analysis
+  description: null
+  execution_mode: Development
+nodes:
+  - id: 1
+    type: read
+    position: {x: 100, y: 200}
+    settings:
+      received_file:
+        path: data/input.csv
+        file_type: csv
+connections:
+  - [1, 2]
+node_starts:
+  - 1
+```

tools/migrate/__init__.py

@@ -0,0 +1,12 @@
+"""
+Flowfile Migration Tool
+
+Converts old pickle-based .flowfile format to new YAML format.
+
+Usage:
+    python -m tools.migrate <path>
+    python -m tools.migrate old_flow.flowfile
+    python -m tools.migrate ./flows/  # migrate entire directory
+"""
+
+__version__ = "1.0.0"

tools/migrate/__main__.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+"""
+Flowfile Migration Tool - CLI Entry Point
+
+Converts old pickle-based .flowfile format to new YAML format.
+
+Usage:
+    python -m tools.migrate <path> [options]
+
+Examples:
+    # Migrate a single file
+    python -m tools.migrate my_flow.flowfile
+
+    # Migrate to specific output
+    python -m tools.migrate my_flow.flowfile -o my_flow.yaml
+
+    # Migrate entire directory
+    python -m tools.migrate ./flows/
+
+    # Migrate to JSON instead of YAML
+    python -m tools.migrate my_flow.flowfile --format json
+
+    # Migrate directory to different output location
+    python -m tools.migrate ./old_flows/ -o ./new_flows/
+"""
+
+import argparse
+import sys
+from pathlib import Path
+
+from tools.migrate.migrate import migrate_flowfile, migrate_directory
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog='flowfile-migrate',
+        description='Migrate old .flowfile pickles to YAML format',
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  %(prog)s my_flow.flowfile              Migrate single file to YAML
+  %(prog)s ./flows/                      Migrate all files in directory
+  %(prog)s flow.flowfile -o flow.yaml    Specify output path
+  %(prog)s ./flows/ --format json        Output as JSON instead of YAML
+        """
+    )
+
+    parser.add_argument(
+        'path',
+        type=Path,
+        help='Path to .flowfile or directory containing .flowfile files'
+    )
+
+    parser.add_argument(
+        '-o', '--output',
+        type=Path,
+        default=None,
+        help='Output path (file or directory). Default: same location with new extension'
+    )
+
+    parser.add_argument(
+        '-f', '--format',
+        choices=['yaml', 'json'],
+        default='yaml',
+        help='Output format (default: yaml)'
+    )
+
+    parser.add_argument(
+        '-v', '--verbose',
+        action='store_true',
+        help='Verbose output'
+    )
+
+    parser.add_argument(
+        '--dry-run',
+        action='store_true',
+        help='Show what would be migrated without actually migrating'
+    )
+
+    args = parser.parse_args()
+
+    # Validate input path
+    if not args.path.exists():
+        print(f"Error: Path not found: {args.path}", file=sys.stderr)
+        sys.exit(1)
+
+    # Dry run mode
+    if args.dry_run:
+        if args.path.is_file():
+            print(f"Would migrate: {args.path}")
+            suffix = '.yaml' if args.format == 'yaml' else '.json'
+            output = args.output or args.path.with_suffix(suffix)
+            print(f"  → {output}")
+        else:
+            flowfiles = list(args.path.glob('**/*.flowfile'))
+            print(f"Would migrate {len(flowfiles)} file(s):")
+            for f in flowfiles:
+                print(f"  - {f}")
+        sys.exit(0)
+
+    # Check for yaml dependency
+    if args.format == 'yaml':
+        try:
+            import yaml
+        except ImportError:
+            print("Error: PyYAML is required for YAML output.", file=sys.stderr)
+            print("Install with: pip install pyyaml", file=sys.stderr)
+            sys.exit(1)
+
+    # Run migration
+    try:
+        if args.path.is_file():
+            migrate_flowfile(args.path, args.output, args.format)
+        elif args.path.is_dir():
+            migrate_directory(args.path, args.output, args.format)
+        else:
+            print(f"Error: {args.path} is neither a file nor a directory", file=sys.stderr)
+            sys.exit(1)
+
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        if args.verbose:
+            import traceback
+            traceback.print_exc()
+        sys.exit(1)
+
+    print("\nMigration complete!")
+
+
+if __name__ == '__main__':
+    main()