morphml 1.0.0__py3-none-any.whl

morphml/benchmarks/suite.py

@@ -0,0 +1,318 @@
+"""Benchmark suite for running comprehensive optimizer comparisons."""
+
+import time
+from typing import Any, Dict, List, Optional
+
+from morphml.benchmarks.problems import BenchmarkProblem
+from morphml.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+
+class BenchmarkResult:
+    """Results from a single benchmark run."""
+
+    def __init__(
+        self,
+        problem_name: str,
+        optimizer_name: str,
+        best_fitness: float,
+        mean_fitness: float,
+        evaluations: int,
+        time_seconds: float,
+        convergence_generation: Optional[int] = None,
+        final_diversity: Optional[float] = None,
+    ):
+        """Initialize benchmark result."""
+        self.problem_name = problem_name
+        self.optimizer_name = optimizer_name
+        self.best_fitness = best_fitness
+        self.mean_fitness = mean_fitness
+        self.evaluations = evaluations
+        self.time_seconds = time_seconds
+        self.convergence_generation = convergence_generation
+        self.final_diversity = final_diversity
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "problem": self.problem_name,
+            "optimizer": self.optimizer_name,
+            "best_fitness": self.best_fitness,
+            "mean_fitness": self.mean_fitness,
+            "evaluations": self.evaluations,
+            "time_seconds": self.time_seconds,
+            "convergence_generation": self.convergence_generation,
+            "final_diversity": self.final_diversity,
+        }
+
+    def __repr__(self) -> str:
+        return (
+            f"BenchmarkResult({self.optimizer_name} on {self.problem_name}: "
+            f"best={self.best_fitness:.4f}, time={self.time_seconds:.2f}s)"
+        )
+
+
+class BenchmarkSuite:
+    """
+    Comprehensive benchmark suite for NAS optimizers.
+
+    Example:
+        >>> suite = BenchmarkSuite()
+        >>> suite.add_optimizer("GA", GeneticAlgorithm, {...})
+        >>> suite.add_optimizer("RS", RandomSearch, {...})
+        >>> results = suite.run(problems, num_runs=5)
+        >>> suite.print_summary(results)
+    """
+
+    def __init__(self):
+        """Initialize benchmark suite."""
+        self.optimizers: Dict[str, tuple] = {}
+        self.results: List[BenchmarkResult] = []
+
+    def add_optimizer(self, name: str, optimizer_class: type, config: Dict[str, Any]) -> None:
+        """
+        Add an optimizer to benchmark.
+
+        Args:
+            name: Optimizer name
+            optimizer_class: Optimizer class
+            config: Configuration dict
+        """
+        self.optimizers[name] = (optimizer_class, config)
+        logger.info(f"Added optimizer: {name}")
+
+    def run_single(
+        self,
+        problem: BenchmarkProblem,
+        optimizer_name: str,
+        optimizer_class: type,
+        config: Dict[str, Any],
+    ) -> BenchmarkResult:
+        """Run single benchmark."""
+        logger.info(f"Running {optimizer_name} on {problem.name}...")
+
+        start_time = time.time()
+
+        # Create optimizer
+        optimizer = optimizer_class(search_space=problem.search_space, **config)
+
+        # Run optimization
+        try:
+            if hasattr(optimizer, "optimize"):
+                best = optimizer.optimize(problem.evaluate)
+            else:
+                raise ValueError(f"Optimizer {optimizer_name} has no optimize method")
+
+            end_time = time.time()
+            elapsed = end_time - start_time
+
+            # Collect metrics
+            best_fitness = best.fitness if best else 0.0
+
+            # Get population stats if available
+            mean_fitness = 0.0
+            final_diversity = None
+            convergence_gen = None
+
+            if hasattr(optimizer, "population"):
+                pop = optimizer.population
+                if pop.size() > 0:
+                    stats = pop.get_statistics()
+                    mean_fitness = stats.get("mean_fitness", 0.0)
+                    final_diversity = pop.get_diversity()
+                    convergence_gen = pop.generation
+
+            # Count evaluations
+            evaluations = config.get("num_generations", 0) * config.get("population_size", 1)
+            if hasattr(optimizer, "num_samples"):
+                evaluations = config.get("num_samples", 0)
+
+            result = BenchmarkResult(
+                problem_name=problem.name,
+                optimizer_name=optimizer_name,
+                best_fitness=best_fitness,
+                mean_fitness=mean_fitness,
+                evaluations=evaluations,
+                time_seconds=elapsed,
+                convergence_generation=convergence_gen,
+                final_diversity=final_diversity,
+            )
+
+            logger.info(f"Completed: {result}")
+            return result
+
+        except Exception as e:
+            logger.error(f"Benchmark failed: {e}")
+            # Return failure result
+            return BenchmarkResult(
+                problem_name=problem.name,
+                optimizer_name=optimizer_name,
+                best_fitness=0.0,
+                mean_fitness=0.0,
+                evaluations=0,
+                time_seconds=time.time() - start_time,
+            )
+
+    def run(self, problems: List[BenchmarkProblem], num_runs: int = 5) -> List[BenchmarkResult]:
+        """
+        Run full benchmark suite.
+
+        Args:
+            problems: List of benchmark problems
+            num_runs: Number of runs per optimizer-problem pair
+
+        Returns:
+            List of benchmark results
+        """
+        all_results = []
+
+        total_runs = len(self.optimizers) * len(problems) * num_runs
+        current_run = 0
+
+        logger.info(f"Starting benchmark: {total_runs} total runs")
+
+        for problem in problems:
+            for opt_name, (opt_class, config) in self.optimizers.items():
+                for run in range(num_runs):
+                    current_run += 1
+                    logger.info(
+                        f"Run {current_run}/{total_runs}: "
+                        f"{opt_name} on {problem.name} (run {run + 1}/{num_runs})"
+                    )
+
+                    result = self.run_single(problem, opt_name, opt_class, config)
+                    all_results.append(result)
+
+        self.results = all_results
+        logger.info("Benchmark complete!")
+
+        return all_results
+
+    def get_summary(self, results: Optional[List[BenchmarkResult]] = None) -> Dict[str, Any]:
+        """
+        Get summary statistics.
+
+        Args:
+            results: Results to summarize (uses self.results if None)
+
+        Returns:
+            Summary dictionary
+        """
+        if results is None:
+            results = self.results
+
+        if not results:
+            return {}
+
+        summary = {}
+
+        # Group by optimizer and problem
+        for result in results:
+            key = (result.optimizer_name, result.problem_name)
+
+            if key not in summary:
+                summary[key] = {
+                    "optimizer": result.optimizer_name,
+                    "problem": result.problem_name,
+                    "best_fitnesses": [],
+                    "mean_fitnesses": [],
+                    "times": [],
+                    "evaluations": [],
+                }
+
+            summary[key]["best_fitnesses"].append(result.best_fitness)
+            summary[key]["mean_fitnesses"].append(result.mean_fitness)
+            summary[key]["times"].append(result.time_seconds)
+            summary[key]["evaluations"].append(result.evaluations)
+
+        # Compute statistics
+        import statistics
+
+        for key, data in summary.items():
+            data["mean_best_fitness"] = statistics.mean(data["best_fitnesses"])
+            data["std_best_fitness"] = (
+                statistics.stdev(data["best_fitnesses"]) if len(data["best_fitnesses"]) > 1 else 0.0
+            )
+            data["median_best_fitness"] = statistics.median(data["best_fitnesses"])
+            data["mean_time"] = statistics.mean(data["times"])
+            data["mean_evaluations"] = statistics.mean(data["evaluations"])
+
+        return summary
+
+    def print_summary(self, results: Optional[List[BenchmarkResult]] = None) -> None:
+        """Print summary of results."""
+        summary = self.get_summary(results)
+
+        if not summary:
+            print("No results to summarize")
+            return
+
+        print("\n" + "=" * 80)
+        print("BENCHMARK SUMMARY")
+        print("=" * 80)
+
+        # Group by problem
+        problems = {data["problem"] for data in summary.values()}
+
+        for problem in sorted(problems):
+            print(f"\n{problem}")
+            print("-" * 80)
+            print(f"{'Optimizer':<20} {'Best (Mean±Std)':<25} {'Median':<10} {'Time (s)':<12}")
+            print("-" * 80)
+
+            problem_results = [
+                (data["optimizer"], data)
+                for key, data in summary.items()
+                if data["problem"] == problem
+            ]
+
+            # Sort by mean best fitness (descending)
+            problem_results.sort(key=lambda x: x[1]["mean_best_fitness"], reverse=True)
+
+            for opt_name, data in problem_results:
+                mean_best = data["mean_best_fitness"]
+                std_best = data["std_best_fitness"]
+                median_best = data["median_best_fitness"]
+                mean_time = data["mean_time"]
+
+                print(
+                    f"{opt_name:<20} "
+                    f"{mean_best:.4f}±{std_best:.4f}          "
+                    f"{median_best:.4f}     "
+                    f"{mean_time:>10.2f}"
+                )
+
+        print("\n" + "=" * 80)
+
+    def get_winner(
+        self, problem_name: str, results: Optional[List[BenchmarkResult]] = None
+    ) -> Optional[str]:
+        """Get best optimizer for a problem."""
+        summary = self.get_summary(results)
+
+        problem_results = [
+            (data["optimizer"], data["mean_best_fitness"])
+            for key, data in summary.items()
+            if data["problem"] == problem_name
+        ]
+
+        if not problem_results:
+            return None
+
+        problem_results.sort(key=lambda x: x[1], reverse=True)
+        return problem_results[0][0]
+
+    def export_results(self, filename: str) -> None:
+        """Export results to JSON."""
+        import json
+
+        data = {
+            "results": [r.to_dict() for r in self.results],
+            "summary": {str(k): v for k, v in self.get_summary().items()},
+        }
+
+        with open(filename, "w") as f:
+            json.dump(data, f, indent=2)
+
+        logger.info(f"Results exported to {filename}")

morphml/cli/__init__.py

@@ -0,0 +1,5 @@
+"""Command-line interface for MorphML."""
+
+from morphml.cli.main import cli
+
+__all__ = ["cli"]

morphml/cli/commands/experiment.py

@@ -0,0 +1,329 @@
+"""Enhanced experiment management CLI commands.
+
+Provides interactive commands for managing NAS experiments.
+
+Example:
+    morphml experiment create --name my-experiment
+    morphml experiment list
+    morphml experiment show exp_123
+"""
+
+from typing import Optional
+
+import click
+from rich.console import Console
+from rich.progress import Progress
+from rich.prompt import Confirm, Prompt
+from rich.table import Table
+
+from morphml.logging_config import get_logger
+
+logger = get_logger(__name__)
+console = Console()
+
+
+@click.group()
+def experiment():
+    """Manage NAS experiments."""
+    pass
+
+
+@experiment.command()
+@click.option("--name", "-n", help="Experiment name")
+@click.option("--optimizer", "-o", default="genetic", help="Optimizer type")
+@click.option("--budget", "-b", default=500, type=int, help="Evaluation budget")
+@click.option("--interactive", "-i", is_flag=True, help="Interactive mode")
+def create(name: Optional[str], optimizer: str, budget: int, interactive: bool):
+    """
+    Create a new experiment.
+
+    Example:
+        morphml experiment create --name cifar10-search
+        morphml experiment create -i  # Interactive mode
+    """
+    console.print("[bold cyan]Create New Experiment[/bold cyan]\n")
+
+    # Interactive mode
+    if interactive or not name:
+        name = Prompt.ask("Experiment name", default="my-experiment")
+        optimizer = Prompt.ask(
+            "Optimizer",
+            choices=["genetic", "random", "hillclimbing", "bayesian"],
+            default="genetic",
+        )
+        budget = int(Prompt.ask("Evaluation budget", default="500"))
+
+    # Define search space interactively
+    console.print("\n[bold]Define Search Space[/bold]")
+
+    layers = []
+
+    # Input layer
+    console.print("Input layer:")
+    input_shape = Prompt.ask("  Input shape (e.g., 3,32,32)", default="3,32,32")
+    shape = tuple(map(int, input_shape.split(",")))
+    layers.append({"type": "input", "shape": shape})
+
+    # Add layers
+    while True:
+        add_layer = Confirm.ask("Add a layer?", default=True)
+        if not add_layer:
+            break
+
+        layer_type = Prompt.ask(
+            "  Layer type",
+            choices=["conv2d", "dense", "maxpool", "flatten", "relu", "dropout"],
+            default="conv2d",
+        )
+
+        layer_config = {"type": layer_type}
+
+        if layer_type == "conv2d":
+            filters = int(Prompt.ask("    Filters", default="64"))
+            kernel = int(Prompt.ask("    Kernel size", default="3"))
+            layer_config.update({"filters": filters, "kernel_size": kernel})
+
+        elif layer_type == "dense":
+            units = int(Prompt.ask("    Units", default="128"))
+            layer_config.update({"units": units})
+
+        elif layer_type == "maxpool":
+            pool_size = int(Prompt.ask("    Pool size", default="2"))
+            layer_config.update({"pool_size": pool_size})
+
+        elif layer_type == "dropout":
+            rate = float(Prompt.ask("    Dropout rate", default="0.5"))
+            layer_config.update({"rate": rate})
+
+        layers.append(layer_config)
+
+    # Create experiment config
+
+    # Display summary
+    console.print("\n[bold green]Experiment Configuration:[/bold green]")
+    console.print(f"  Name: {name}")
+    console.print(f"  Optimizer: {optimizer}")
+    console.print(f"  Budget: {budget}")
+    console.print(f"  Layers: {len(layers)}")
+
+    # Confirm
+    if Confirm.ask("\nCreate experiment?", default=True):
+        # TODO: Actually create experiment via API or directly
+        console.print(f"\n[green]✓ Created experiment: {name}[/green]")
+        console.print(f"  Run with: morphml experiment start {name}")
+    else:
+        console.print("[yellow]Cancelled[/yellow]")
+
+
+@experiment.command()
+@click.option("--status", "-s", help="Filter by status")
+@click.option("--limit", "-l", default=20, help="Maximum results")
+def list(status: Optional[str], limit: int):
+    """
+    List all experiments.
+
+    Example:
+        morphml experiment list
+        morphml experiment list --status running
+    """
+    console.print("[bold cyan]Experiments[/bold cyan]\n")
+
+    # TODO: Fetch from API
+    # For now, show example data
+    experiments = [
+        {
+            "id": "exp_abc123",
+            "name": "CIFAR-10 Search",
+            "status": "running",
+            "best_accuracy": 0.9234,
+            "generation": 25,
+            "total_generations": 50,
+        },
+        {
+            "id": "exp_def456",
+            "name": "ImageNet Transfer",
+            "status": "completed",
+            "best_accuracy": 0.8567,
+            "generation": 100,
+            "total_generations": 100,
+        },
+        {
+            "id": "exp_ghi789",
+            "name": "Quick Test",
+            "status": "failed",
+            "best_accuracy": None,
+            "generation": 5,
+            "total_generations": 20,
+        },
+    ]
+
+    # Filter by status
+    if status:
+        experiments = [e for e in experiments if e["status"] == status]
+
+    # Limit results
+    experiments = experiments[:limit]
+
+    # Create table
+    table = Table(title=f"Experiments ({len(experiments)} total)")
+    table.add_column("ID", style="cyan")
+    table.add_column("Name", style="white")
+    table.add_column("Status", style="yellow")
+    table.add_column("Progress", style="blue")
+    table.add_column("Best Accuracy", style="green")
+
+    for exp in experiments:
+        # Status color
+        status_color = {
+            "running": "yellow",
+            "completed": "green",
+            "failed": "red",
+            "pending": "blue",
+        }.get(exp["status"], "white")
+
+        # Progress
+        progress = f"{exp['generation']}/{exp['total_generations']}"
+
+        # Accuracy
+        accuracy = f"{exp['best_accuracy']:.4f}" if exp["best_accuracy"] else "-"
+
+        table.add_row(
+            exp["id"],
+            exp["name"],
+            f"[{status_color}]{exp['status']}[/{status_color}]",
+            progress,
+            accuracy,
+        )
+
+    console.print(table)
+
+    console.print(f"\n[dim]Showing {len(experiments)} of {len(experiments)} experiments[/dim]")
+
+
+@experiment.command()
+@click.argument("experiment_id")
+def show(experiment_id: str):
+    """
+    Show detailed experiment information.
+
+    Example:
+        morphml experiment show exp_abc123
+    """
+    console.print(f"[bold cyan]Experiment: {experiment_id}[/bold cyan]\n")
+
+    # TODO: Fetch from API
+    # Example data
+    exp = {
+        "id": experiment_id,
+        "name": "CIFAR-10 Search",
+        "status": "running",
+        "optimizer": "genetic",
+        "created_at": "2024-11-11T05:00:00Z",
+        "started_at": "2024-11-11T05:01:00Z",
+        "best_accuracy": 0.9234,
+        "generation": 25,
+        "total_generations": 50,
+        "population_size": 20,
+        "best_architecture": {"parameters": 1234567, "depth": 8, "nodes": 12},
+    }
+
+    # Display info
+    console.print(f"[bold]Name:[/bold] {exp['name']}")
+    console.print(f"[bold]Status:[/bold] {exp['status']}")
+    console.print(f"[bold]Optimizer:[/bold] {exp['optimizer']}")
+    console.print(f"[bold]Created:[/bold] {exp['created_at']}")
+    console.print(
+        f"[bold]Progress:[/bold] {exp['generation']}/{exp['total_generations']} generations"
+    )
+
+    console.print("\n[bold green]Best Architecture:[/bold green]")
+    console.print(f"  Accuracy: {exp['best_accuracy']:.4f}")
+    console.print(f"  Parameters: {exp['best_architecture']['parameters']:,}")
+    console.print(f"  Depth: {exp['best_architecture']['depth']}")
+    console.print(f"  Nodes: {exp['best_architecture']['nodes']}")
+
+
+@experiment.command()
+@click.argument("experiment_id")
+def start(experiment_id: str):
+    """
+    Start an experiment.
+
+    Example:
+        morphml experiment start exp_abc123
+    """
+    console.print(f"[bold cyan]Starting experiment: {experiment_id}[/bold cyan]\n")
+
+    with Progress() as progress:
+        task = progress.add_task("[cyan]Initializing...", total=100)
+
+        # Simulate startup
+        import time
+
+        for _i in range(100):
+            time.sleep(0.01)
+            progress.update(task, advance=1)
+
+    console.print("[green]✓ Experiment started[/green]")
+    console.print(f"  Monitor with: morphml experiment show {experiment_id}")
+
+
+@experiment.command()
+@click.argument("experiment_id")
+def stop(experiment_id: str):
+    """
+    Stop a running experiment.
+
+    Example:
+        morphml experiment stop exp_abc123
+    """
+    if Confirm.ask(f"Stop experiment {experiment_id}?"):
+        console.print(f"[yellow]Stopping experiment: {experiment_id}[/yellow]")
+        # TODO: Call API to stop
+        console.print("[green]✓ Experiment stopped[/green]")
+    else:
+        console.print("[dim]Cancelled[/dim]")
+
+
+@experiment.command()
+@click.argument("experiment_id")
+@click.option("--force", "-f", is_flag=True, help="Force delete without confirmation")
+def delete(experiment_id: str, force: bool):
+    """
+    Delete an experiment.
+
+    Example:
+        morphml experiment delete exp_abc123
+        morphml experiment delete exp_abc123 --force
+    """
+    if force or Confirm.ask(f"[red]Delete experiment {experiment_id}?[/red]"):
+        console.print(f"[red]Deleting experiment: {experiment_id}[/red]")
+        # TODO: Call API to delete
+        console.print("[green]✓ Experiment deleted[/green]")
+    else:
+        console.print("[dim]Cancelled[/dim]")
+
+
+@experiment.command()
+@click.argument("experiment_id")
+@click.option("--format", "-f", type=click.Choice(["pytorch", "keras", "both"]), default="pytorch")
+@click.option("--output", "-o", help="Output file path")
+def export(experiment_id: str, format: str, output: Optional[str]):
+    """
+    Export best architecture from experiment.
+
+    Example:
+        morphml experiment export exp_abc123 --format pytorch
+        morphml experiment export exp_abc123 --format keras -o model.py
+    """
+    console.print(f"[bold cyan]Exporting architecture from: {experiment_id}[/bold cyan]\n")
+
+    # TODO: Fetch best architecture and export
+
+    if not output:
+        output = f"{experiment_id}_model.py"
+
+    console.print(f"[bold]Format:[/bold] {format}")
+    console.print(f"[bold]Output:[/bold] {output}")
+
+    console.print(f"\n[green]✓ Exported to {output}[/green]")