flowyml 1.1.0__py3-none-any.whl

flowyml/cli/init.py

@@ -0,0 +1,256 @@
+"""Project initialization CLI commands."""
+
+from pathlib import Path
+
+
+TEMPLATES = {
+    "basic": {
+        "description": "Basic flowyml project structure",
+        "files": {
+            "pipelines/__init__.py": "",
+            "pipelines/training_pipeline.py": """\"\"\"Example training pipeline.\"\"\"
+
+from flowyml import Pipeline, step, context
+
+# Define context
+ctx = context(
+    learning_rate=0.001,
+    batch_size=32,
+    epochs=10
+)
+
+
+@step(outputs=["data/processed"])
+def load_data():
+    \"\"\"Load and preprocess data.\"\"\"
+    # Your data loading logic here
+    print("Loading data...")
+    data = {"samples": 1000, "features": 10}
+    return data
+
+
+@step(inputs=["data/processed"], outputs=["model/trained"])
+def train_model(data, learning_rate, epochs):
+    \"\"\"Train the model.\"\"\"
+    print(f"Training model with lr={learning_rate}, epochs={epochs}")
+    # Your training logic here
+    model = {"accuracy": 0.95, "loss": 0.05}
+    return model
+
+
+@step(inputs=["model/trained"], outputs=["metrics/evaluation"])
+def evaluate_model(model):
+    \"\"\"Evaluate the trained model.\"\"\"
+    print("Evaluating model...")
+    metrics = {"test_accuracy": 0.93, "test_loss": 0.07}
+    return metrics
+
+
+# Create pipeline
+def create_pipeline():
+    pipeline = Pipeline("training_pipeline", context=ctx)
+    pipeline.add_step(load_data)
+    pipeline.add_step(train_model)
+    pipeline.add_step(evaluate_model)
+    return pipeline
+
+
+if __name__ == "__main__":
+    pipeline = create_pipeline()
+    result = pipeline.run()
+    print(f"\\nPipeline completed successfully!")
+    print(f"Results: {result.outputs}")
+""",
+            "README.md": """# {project_name}
+
+A flowyml ML pipeline project.
+
+## Getting Started
+
+1. Install dependencies:
+```bash
+pip install flowyml
+```
+
+2. Run the pipeline:
+```bash
+flowyml run training_pipeline
+# or
+python pipelines/training_pipeline.py
+```
+
+3. View results:
+```bash
+flowyml ui start
+```
+
+## Project Structure
+
+```
+{project_name}/
+├── pipelines/           # Pipeline definitions
+│   └── training_pipeline.py
+├── flowyml.yaml          # Project configuration
+└── README.md
+```
+
+## Learn More
+
+- [flowyml Documentation](https://docs.flowyml.ai)
+- [Examples](https://github.com/flowyml/flowyml/examples)
+""",
+            "flowyml.yaml": """# flowyml project configuration
+
+name: {project_name}
+version: "0.1.0"
+
+# Stack configuration
+stack:
+  default: local
+
+# Execution settings
+execution:
+  max_parallel_steps: 4
+  enable_caching: true
+
+# Experiment tracking
+tracking:
+  auto_log_params: true
+  auto_log_metrics: true
+  auto_log_artifacts: true
+""",
+            ".gitignore": """.flowyml/
+*.pyc
+__pycache__/
+.ipynb_checkpoints/
+*.egg-info/
+dist/
+build/
+.DS_Store
+""",
+        },
+    },
+    "pytorch": {
+        "description": "PyTorch ML project template",
+        "files": {
+            "pipelines/__init__.py": "",
+            "pipelines/training_pipeline.py": """\"\"\"PyTorch training pipeline.\"\"\"
+
+from flowyml import Pipeline, step, context
+import torch
+import torch.nn as nn
+
+
+ctx = context(
+    learning_rate=0.001,
+    batch_size=64,
+    epochs=10,
+    device="cuda" if torch.cuda.is_available() else "cpu"
+)
+
+
+class SimpleModel(nn.Module):
+    def __init__(self, input_dim, hidden_dim, output_dim):
+        super().__init__()
+        self.network = nn.Sequential(
+            nn.Linear(input_dim, hidden_dim),
+            nn.ReLU(),
+            nn.Linear(hidden_dim, output_dim)
+        )
+
+    def forward(self, x):
+        return self.network(x)
+
+
+@step(outputs=["data/train", "data/val"])
+def prepare_data():
+    \"\"\"Prepare training and validation datasets.\"\"\"
+    # Your data preparation logic here
+    print("Preparing PyTorch datasets...")
+    return {"train_size": 8000}, {"val_size": 2000}
+
+
+@step(inputs=["data/train", "data/val"], outputs=["model/trained"])
+def train_model(train_data, val_data, learning_rate, epochs, device):
+    \"\"\"Train PyTorch model.\"\"\"
+    print(f"Training on {device}...")
+
+    model = SimpleModel(input_dim=10, hidden_dim=64, output_dim=2)
+    model = model.to(device)
+
+    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
+
+    for epoch in range(epochs):
+        # Training loop here
+        print(f"Epoch {epoch+1}/{epochs}")
+
+    return model
+
+
+def create_pipeline():
+    pipeline = Pipeline("pytorch_training", context=ctx)
+    pipeline.add_step(prepare_data)
+    pipeline.add_step(train_model)
+    return pipeline
+
+
+if __name__ == "__main__":
+    pipeline = create_pipeline()
+    result = pipeline.run()
+    print("\\nTraining complete!")
+""",
+            "requirements.txt": """flowyml
+torch>=2.0.0
+numpy
+""",
+        },
+    },
+}
+
+
+def init_project(name: str, template: str, directory: Path) -> None:
+    """Initialize a new flowyml project.
+
+    Args:
+        name: Project name
+        template: Template to use (basic, pytorch, tensorflow, sklearn)
+        directory: Directory to create project in
+    """
+    if template not in TEMPLATES:
+        raise ValueError(f"Unknown template: {template}. Choose from: {', '.join(TEMPLATES.keys())}")
+
+    # Create project directory
+    directory.mkdir(parents=True, exist_ok=True)
+
+    # Get template files
+    template_data = TEMPLATES[template]
+    files = template_data["files"]
+
+    # Create files from template
+    for file_path, content in files.items():
+        full_path = directory / file_path
+
+        # Create parent directories
+        full_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Format content with project name
+        formatted_content = content.replace("{project_name}", name)
+
+        # Write file
+        with open(full_path, "w") as f:
+            f.write(formatted_content)
+
+    # Create .flowyml directory
+    (directory / ".flowyml").mkdir(exist_ok=True)
+    (directory / ".flowyml" / "artifacts").mkdir(exist_ok=True)
+    (directory / ".flowyml" / "cache").mkdir(exist_ok=True)
+    (directory / ".flowyml" / "runs").mkdir(exist_ok=True)
+
+
+def list_templates() -> dict:
+    """List available project templates.
+
+    Returns:
+        Dictionary of template names and descriptions
+    """
+    return {name: data["description"] for name, data in TEMPLATES.items()}

flowyml/cli/main.py

@@ -0,0 +1,327 @@
+"""Main CLI entry point for flowyml."""
+
+import click
+from pathlib import Path
+from flowyml.utils.config import get_config
+
+
+@click.group()
+@click.version_option(version="0.1.0", prog_name="flowyml")
+def cli() -> None:
+    """Flowyml - Next-Generation ML Pipeline Framework.
+
+    A developer-first ML pipeline orchestration framework that makes
+    ML pipelines feel effortless while providing production-grade capabilities.
+    """
+    pass
+
+
+@cli.command()
+@click.option("--name", prompt="Project name", help="Name of the project")
+@click.option(
+    "--template",
+    default="basic",
+    type=click.Choice(["basic", "pytorch", "tensorflow", "sklearn"]),
+    help="Project template",
+)
+@click.option("--dir", "directory", default=".", help="Directory to create project in")
+def init(name: str, template: str, directory: str) -> None:
+    """Initialize a new flowyml project."""
+    from flowyml.cli.init import init_project
+
+    project_dir = Path(directory) / name
+    click.echo(f"Initializing flowyml project '{name}' with template '{template}'...")
+
+    try:
+        init_project(name, template, project_dir)
+        click.echo(f"✓ Project '{name}' created successfully at {project_dir}")
+        click.echo("\nNext steps:")
+        click.echo(f"  cd {name}")
+        click.echo("  flowyml run training_pipeline")
+    except Exception as e:
+        click.echo(f"✗ Error creating project: {e}", err=True)
+        raise click.Abort()
+
+
+@cli.command()
+@click.argument("pipeline_name")
+@click.option("--stack", default="local", help="Stack to use for execution")
+@click.option("--context", "-c", multiple=True, help="Context parameters (key=value)")
+@click.option("--debug", is_flag=True, help="Enable debug mode")
+def run(pipeline_name: str, stack: str, context: tuple, debug: bool) -> None:
+    """Run a pipeline."""
+    from flowyml.cli.run import run_pipeline
+
+    # Parse context parameters
+    ctx_params = {}
+    for param in context:
+        key, value = param.split("=", 1)
+        ctx_params[key] = value
+
+    click.echo(f"Running pipeline '{pipeline_name}' on stack '{stack}'...")
+
+    try:
+        result = run_pipeline(pipeline_name, stack, ctx_params, debug)
+        click.echo("✓ Pipeline completed successfully")
+        click.echo(f"  Run ID: {result.get('run_id', 'N/A')}")
+        click.echo(f"  Duration: {result.get('duration', 'N/A')}")
+    except Exception as e:
+        click.echo(f"✗ Pipeline failed: {e}", err=True)
+        raise click.Abort()
+
+
+@cli.group()
+def ui() -> None:
+    """UI server commands."""
+    pass
+
+
+@ui.command()
+@click.option("--host", default="localhost", help="Host to bind to")
+@click.option("--port", default=8080, help="Port to bind to")
+@click.option("--dev", is_flag=True, help="Run in development mode")
+@click.option("--open-browser", "-o", is_flag=True, help="Open browser automatically")
+def start(host: str, port: int, dev: bool, open_browser: bool) -> None:
+    """Start the flowyml UI server."""
+    from flowyml.ui.utils import is_ui_running
+
+    # Check if already running
+    if is_ui_running(host, port):
+        click.echo(f"ℹ️  UI server is already running at http://{host}:{port}")
+        if open_browser:
+            import webbrowser
+
+            webbrowser.open(f"http://{host}:{port}")
+        return
+
+    click.echo(f"🚀 Starting flowyml UI on http://{host}:{port}...")
+    if dev:
+        click.echo("   Development mode: Auto-reload enabled")
+
+    try:
+        from flowyml.cli.ui import start_ui_server
+
+        # Open browser if requested
+        if open_browser:
+            import webbrowser
+            import threading
+
+            def open_browser_delayed() -> None:
+                import time
+
+                time.sleep(1.5)  # Wait for server to start
+                webbrowser.open(f"http://{host}:{port}")
+
+            threading.Thread(target=open_browser_delayed, daemon=True).start()
+
+        start_ui_server(host, port, dev)
+    except ImportError:
+        click.echo("✗ UI server not available. Install with: pip install flowyml[ui]", err=True)
+        raise click.Abort()
+    except Exception as e:
+        click.echo(f"✗ Error starting UI: {e}", err=True)
+        raise click.Abort()
+
+
+@ui.command()
+def stop() -> None:
+    """Stop the flowyml UI server."""
+    click.echo("Stopping flowyml UI server...")
+    click.echo("ℹ️  To stop the UI server:")
+    click.echo("   - If running in foreground: Press Ctrl+C")
+    click.echo("   - If running in background: pkill -f 'flowyml ui start'")
+
+
+@ui.command()
+@click.option("--host", default="localhost", help="Host to check")
+@click.option("--port", default=8080, help="Port to check")
+def status(host: str, port: int) -> None:
+    """Check if the UI server is running."""
+    from flowyml.ui.utils import is_ui_running, get_ui_url
+
+    if is_ui_running(host, port):
+        url = get_ui_url(host, port)
+        click.echo(f"✅ UI server is running at {url}")
+        click.echo("   Status: Healthy")
+        click.echo(f"   Health endpoint: {url}/api/health")
+    else:
+        click.echo(f"❌ UI server is not running on {host}:{port}")
+        click.echo(f"   Start with: flowyml ui start --host {host} --port {port}")
+
+
+@cli.group()
+def experiment() -> None:
+    """Experiment tracking commands."""
+    pass
+
+
+@experiment.command("list")
+@click.option("--limit", default=10, help="Number of experiments to show")
+@click.option("--pipeline", help="Filter by pipeline name")
+def list_experiments(limit: int, pipeline: str) -> None:
+    """List experiments."""
+    from flowyml.cli.experiment import list_experiments_cmd
+
+    try:
+        experiments = list_experiments_cmd(limit, pipeline)
+        click.echo(f"Found {len(experiments)} experiments:\n")
+
+        for exp in experiments:
+            click.echo(f"  {exp['name']}")
+            click.echo(f"    Runs: {exp.get('num_runs', 0)}")
+            click.echo(f"    Created: {exp.get('created_at', 'N/A')}")
+            click.echo()
+    except Exception as e:
+        click.echo(f"✗ Error listing experiments: {e}", err=True)
+
+
+@experiment.command()
+@click.argument("run_ids", nargs=-1, required=True)
+def compare(run_ids: tuple) -> None:
+    """Compare multiple experiment runs."""
+    from flowyml.cli.experiment import compare_runs
+
+    click.echo(f"Comparing {len(run_ids)} runs...")
+
+    try:
+        comparison = compare_runs(list(run_ids))
+        click.echo("\nComparison Results:")
+        click.echo(comparison)
+    except Exception as e:
+        click.echo(f"✗ Error comparing runs: {e}", err=True)
+
+
+@cli.group()
+def stack() -> None:
+    """Stack management commands."""
+    pass
+
+
+@stack.command("list")
+def list_stacks() -> None:
+    """List available stacks."""
+    click.echo("Available stacks:\n")
+    click.echo("  local (default) - Local execution")
+    click.echo("  aws             - AWS (SageMaker, S3, Step Functions)")
+    click.echo("  gcp             - Google Cloud (Vertex AI, GCS)")
+    click.echo("  azure           - Azure (ML, Blob Storage)")
+
+
+@stack.command()
+@click.argument("stack_name")
+def switch(stack_name: str) -> None:
+    """Switch active stack."""
+    config = get_config()
+    config.default_stack = stack_name
+    config.save()
+    click.echo(f"✓ Switched to stack '{stack_name}'")
+
+
+@cli.group()
+def cache() -> None:
+    """Cache management commands."""
+    pass
+
+
+@cache.command()
+def stats() -> None:
+    """Show cache statistics."""
+    from flowyml.core.cache import CacheStore
+
+    try:
+        cache = CacheStore()
+        stats = cache.get_stats()
+
+        click.echo("Cache Statistics:\n")
+        click.echo(f"  Hits: {stats['hits']}")
+        click.echo(f"  Misses: {stats['misses']}")
+        click.echo(f"  Hit Rate: {stats.get('hit_rate', 0):.1%}")
+        click.echo(f"  Total Entries: {stats.get('total_entries', 0)}")
+        click.echo(f"  Size: {stats.get('size_mb', 0):.2f} MB")
+    except Exception as e:
+        click.echo(f"✗ Error getting cache stats: {e}", err=True)
+
+
+@cache.command()
+@click.confirmation_option(prompt="Are you sure you want to clear the cache?")
+def clear() -> None:
+    """Clear all cache."""
+    from flowyml.core.cache import CacheStore
+
+    try:
+        cache = CacheStore()
+        cache.clear()
+        click.echo("✓ Cache cleared successfully")
+    except Exception as e:
+        click.echo(f"✗ Error clearing cache: {e}", err=True)
+
+
+@cli.group()
+def config() -> None:
+    """Configuration management commands."""
+    pass
+
+
+@config.command("show")
+def show_config() -> None:
+    """Show current configuration."""
+    cfg = get_config()
+
+    click.echo("flowyml Configuration:\n")
+    click.echo(f"  flowyml Home: {cfg.flowyml_home}")
+    click.echo(f"  Artifacts Dir: {cfg.artifacts_dir}")
+    click.echo(f"  Metadata DB: {cfg.metadata_db}")
+    click.echo(f"  Default Stack: {cfg.default_stack}")
+    click.echo(f"  Execution Mode: {cfg.execution_mode}")
+    if cfg.execution_mode == "remote":
+        click.echo(f"  Remote Server URL: {cfg.remote_server_url}")
+        click.echo(f"  Remote UI URL: {cfg.remote_ui_url}")
+    click.echo(f"  Enable Caching: {cfg.enable_caching}")
+    click.echo(f"  Log Level: {cfg.log_level}")
+    click.echo(f"  UI Port: {cfg.ui_port}")
+    click.echo(f"  Debug Mode: {cfg.debug_mode}")
+
+
+@config.command("set-mode")
+@click.argument("mode", type=click.Choice(["local", "remote"]))
+def set_mode(mode: str) -> None:
+    """Set execution mode (local or remote)."""
+    cfg = get_config()
+    cfg.execution_mode = mode
+    cfg.save()
+    click.echo(f"✓ Execution mode set to '{mode}'")
+
+
+@config.command("set-url")
+@click.option("--server", help="Remote server URL")
+@click.option("--ui", help="Remote UI URL")
+def set_url(server: str, ui: str) -> None:
+    """Set remote server and UI URLs."""
+    cfg = get_config()
+    if server:
+        cfg.remote_server_url = server
+        click.echo(f"✓ Remote server URL set to '{server}'")
+    if ui:
+        cfg.remote_ui_url = ui
+        click.echo(f"✓ Remote UI URL set to '{ui}'")
+    cfg.save()
+
+
+@cli.command()
+@click.argument("run_id")
+@click.option("--step", help="Filter by step name")
+@click.option("--tail", default=100, help="Number of lines to show")
+def logs(run_id: str, step: str, tail: int) -> None:
+    """View logs for a pipeline run."""
+    click.echo(f"Logs for run '{run_id}':")
+
+    if step:
+        click.echo(f"  Step: {step}")
+
+    click.echo("\nLog entries:")
+    click.echo(f"  (Showing last {tail} lines)")
+    click.echo("  [Log output would appear here]")
+
+
+if __name__ == "__main__":
+    cli()

flowyml/cli/run.py

@@ -0,0 +1,75 @@
+"""Pipeline execution CLI commands."""
+
+import importlib.util
+import sys
+from pathlib import Path
+from typing import Any
+
+
+def run_pipeline(pipeline_name: str, stack: str, context_params: dict[str, Any], debug: bool) -> dict[str, Any]:
+    """Run a pipeline by name.
+
+    Args:
+        pipeline_name: Name of the pipeline to run
+        stack: Stack to use for execution
+        context_params: Context parameters to override
+        debug: Enable debug mode
+
+    Returns:
+        Dictionary with run results
+    """
+    # Try to find pipeline file
+    pipeline_paths = [
+        Path(f"pipelines/{pipeline_name}.py"),
+        Path(f"{pipeline_name}.py"),
+        Path(f"pipelines/{pipeline_name}_pipeline.py"),
+    ]
+
+    pipeline_file = None
+    for path in pipeline_paths:
+        if path.exists():
+            pipeline_file = path
+            break
+
+    if pipeline_file is None:
+        raise FileNotFoundError(
+            f"Pipeline '{pipeline_name}' not found. Looked in:\n" + "\n".join(f"  - {p}" for p in pipeline_paths),
+        )
+
+    # Load pipeline module
+    spec = importlib.util.spec_from_file_location(pipeline_name, pipeline_file)
+    if spec is None or spec.loader is None:
+        raise ImportError(f"Could not load pipeline from {pipeline_file}")
+
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[pipeline_name] = module
+    spec.loader.exec_module(module)
+
+    # Get pipeline
+    if hasattr(module, "create_pipeline"):
+        pipeline = module.create_pipeline()
+    elif hasattr(module, "pipeline"):
+        pipeline = module.pipeline
+    else:
+        raise AttributeError(
+            "Pipeline module must define 'create_pipeline()' function or 'pipeline' variable",
+        )
+
+    # Override context parameters
+    if context_params:
+        for key, value in context_params.items():
+            setattr(pipeline.context, key, value)
+
+    # Set stack if not default
+    if stack != "local":
+        pipeline.set_stack(stack)
+
+    # Run pipeline
+    result = pipeline.run(debug=debug)
+
+    return {
+        "run_id": result.run_id,
+        "status": result.status,
+        "duration": result.duration,
+        "outputs": result.outputs,
+    }