morphml 1.0.0__py3-none-any.whl

morphml/benchmarks/metrics.py

@@ -0,0 +1,199 @@
+"""Performance metrics for benchmarking.
+
+Author: Eshan Roy <eshanized@proton.me>
+Organization: TONMOY INFRASTRUCTURE & VISION
+"""
+
+from typing import Dict, List, Optional
+
+import numpy as np
+
+from morphml.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+
+def sample_efficiency(history: List[Dict], target_fitness: float) -> int:
+    """
+    Calculate number of evaluations to reach target fitness.
+
+    Args:
+        history: Optimization history
+        target_fitness: Target fitness threshold
+
+    Returns:
+        Number of evaluations to reach target, or -1 if not reached
+    """
+    for i, entry in enumerate(history):
+        if entry.get("best_fitness", 0) >= target_fitness:
+            return i + 1
+    return -1
+
+
+def convergence_rate(history: List[Dict]) -> float:
+    """
+    Calculate convergence rate (improvement per evaluation).
+
+    Args:
+        history: Optimization history
+
+    Returns:
+        Average improvement per evaluation
+    """
+    if len(history) < 2:
+        return 0.0
+
+    fitnesses = [entry.get("best_fitness", 0) for entry in history]
+    improvements = np.diff(fitnesses)
+
+    return np.mean(improvements[improvements > 0]) if len(improvements) > 0 else 0.0
+
+
+def final_best_fitness(history: List[Dict]) -> float:
+    """
+    Get final best fitness achieved.
+
+    Args:
+        history: Optimization history
+
+    Returns:
+        Best fitness value
+    """
+    if not history:
+        return 0.0
+
+    return max(entry.get("best_fitness", 0) for entry in history)
+
+
+def time_efficiency(history: List[Dict], target_fitness: float) -> float:
+    """
+    Calculate time to reach target fitness.
+
+    Args:
+        history: Optimization history
+        target_fitness: Target fitness threshold
+
+    Returns:
+        Time in seconds, or -1 if not reached
+    """
+    for entry in history:
+        if entry.get("best_fitness", 0) >= target_fitness:
+            return entry.get("time_elapsed", -1)
+    return -1.0
+
+
+def stability_score(histories: List[List[Dict]]) -> float:
+    """
+    Calculate stability across multiple runs.
+
+    Args:
+        histories: List of optimization histories from multiple runs
+
+    Returns:
+        Stability score (1 - coefficient of variation of final fitness)
+    """
+    if not histories:
+        return 0.0
+
+    final_fitnesses = [final_best_fitness(h) for h in histories]
+
+    mean = np.mean(final_fitnesses)
+    std = np.std(final_fitnesses)
+
+    if mean == 0:
+        return 0.0
+
+    cv = std / mean  # Coefficient of variation
+    stability = max(0.0, 1.0 - cv)
+
+    return stability
+
+
+def area_under_curve(history: List[Dict]) -> float:
+    """
+    Calculate area under convergence curve (AUC).
+
+    Higher AUC indicates faster convergence.
+
+    Args:
+        history: Optimization history
+
+    Returns:
+        AUC value
+    """
+    if not history:
+        return 0.0
+
+    fitnesses = [entry.get("best_fitness", 0) for entry in history]
+
+    # Trapezoidal integration
+    auc = np.trapz(fitnesses)
+
+    return auc
+
+
+def success_rate(histories: List[List[Dict]], target_fitness: float) -> float:
+    """
+    Calculate success rate (fraction of runs reaching target).
+
+    Args:
+        histories: List of optimization histories
+        target_fitness: Target fitness threshold
+
+    Returns:
+        Success rate [0, 1]
+    """
+    if not histories:
+        return 0.0
+
+    successes = sum(1 for h in histories if final_best_fitness(h) >= target_fitness)
+
+    return successes / len(histories)
+
+
+def compute_all_metrics(
+    history: List[Dict], target_fitness: Optional[float] = None
+) -> Dict[str, float]:
+    """
+    Compute all available metrics for an optimization run.
+
+    Args:
+        history: Optimization history
+        target_fitness: Optional target fitness for efficiency metrics
+
+    Returns:
+        Dictionary of metric names and values
+    """
+    metrics = {
+        "final_best_fitness": final_best_fitness(history),
+        "convergence_rate": convergence_rate(history),
+        "auc": area_under_curve(history),
+        "num_evaluations": len(history),
+    }
+
+    if target_fitness is not None:
+        metrics["sample_efficiency"] = sample_efficiency(history, target_fitness)
+        metrics["time_efficiency"] = time_efficiency(history, target_fitness)
+
+    return metrics
+
+
+def compare_optimizers(
+    results: Dict[str, List[Dict]], target_fitness: Optional[float] = None
+) -> Dict[str, Dict[str, float]]:
+    """
+    Compare multiple optimizers across metrics.
+
+    Args:
+        results: Dict mapping optimizer names to their histories
+        target_fitness: Optional target for efficiency metrics
+
+    Returns:
+        Dict of optimizer names to their metric dictionaries
+    """
+    comparison = {}
+
+    for optimizer_name, history in results.items():
+        comparison[optimizer_name] = compute_all_metrics(history, target_fitness)
+
+    return comparison

morphml/benchmarks/openml_suite.py

@@ -0,0 +1,201 @@
+"""OpenML integration for benchmarking.
+
+Author: Eshan Roy <eshanized@proton.me>
+Organization: TONMOY INFRASTRUCTURE & VISION
+"""
+
+from typing import Dict, List, Tuple
+
+import numpy as np
+
+from morphml.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+# Check if OpenML is available
+try:
+    import openml
+
+    OPENML_AVAILABLE = True
+except ImportError:
+    OPENML_AVAILABLE = False
+    logger.warning("OpenML not available. Install with: pip install openml")
+
+
+class OpenMLSuite:
+    """
+    OpenML benchmark suite for NAS evaluation.
+
+    Provides access to curated machine learning datasets from OpenML.
+
+    Example:
+        >>> suite = OpenMLSuite()
+        >>> task = suite.get_task(3)  # CIFAR-10
+        >>> X_train, y_train, X_test, y_test = suite.load_task_data(task)
+    """
+
+    # Curated task IDs for benchmarking
+    BENCHMARK_TASKS = {
+        "mnist": 3573,
+        "fashion_mnist": 146825,
+        "cifar10": 167124,
+        "svhn": 168757,
+    }
+
+    def __init__(self):
+        """Initialize OpenML suite."""
+        if not OPENML_AVAILABLE:
+            raise ImportError(
+                "OpenML required for benchmark suite. " "Install with: pip install openml"
+            )
+
+        self.tasks = {}
+        logger.info("Initialized OpenML benchmark suite")
+
+    def get_task(self, task_id: int):
+        """
+        Get an OpenML task.
+
+        Args:
+            task_id: OpenML task ID
+
+        Returns:
+            OpenML task object
+        """
+        if task_id in self.tasks:
+            return self.tasks[task_id]
+
+        logger.info(f"Fetching OpenML task {task_id}...")
+        task = openml.tasks.get_task(task_id)
+        self.tasks[task_id] = task
+
+        return task
+
+    def get_task_by_name(self, name: str):
+        """
+        Get a benchmark task by name.
+
+        Args:
+            name: Task name (e.g., 'mnist', 'cifar10')
+
+        Returns:
+            OpenML task object
+        """
+        task_id = self.BENCHMARK_TASKS.get(name.lower())
+        if task_id is None:
+            raise ValueError(f"Unknown benchmark task: {name}")
+
+        return self.get_task(task_id)
+
+    def load_task_data(
+        self, task, normalize: bool = True
+    ) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
+        """
+        Load data for a task.
+
+        Args:
+            task: OpenML task object
+            normalize: Whether to normalize features
+
+        Returns:
+            Tuple of (X_train, y_train, X_test, y_test)
+        """
+        logger.info(f"Loading data for task {task.task_id}...")
+
+        dataset = task.get_dataset()
+        X, y, _, _ = dataset.get_data(target=task.target_name)
+
+        X = np.array(X)
+        y = np.array(y)
+
+        if normalize and X.dtype in [np.float32, np.float64]:
+            X = (X - X.mean(axis=0)) / (X.std(axis=0) + 1e-8)
+
+        # Split train/test (80/20)
+        split_idx = int(0.8 * len(X))
+        X_train, X_test = X[:split_idx], X[split_idx:]
+        y_train, y_test = y[:split_idx], y[split_idx:]
+
+        logger.info(f"Loaded task data: train={X_train.shape}, test={X_test.shape}")
+
+        return X_train, y_train, X_test, y_test
+
+    def list_benchmark_tasks(self) -> List[str]:
+        """List available benchmark tasks."""
+        return list(self.BENCHMARK_TASKS.keys())
+
+    def get_task_info(self, task_name: str) -> Dict:
+        """
+        Get information about a benchmark task.
+
+        Args:
+            task_name: Task name
+
+        Returns:
+            Dictionary with task metadata
+        """
+        task_id = self.BENCHMARK_TASKS.get(task_name)
+        if task_id is None:
+            return {}
+
+        try:
+            task = self.get_task(task_id)
+            dataset = task.get_dataset()
+
+            return {
+                "name": task_name,
+                "task_id": task_id,
+                "dataset_name": dataset.name,
+                "num_instances": dataset.qualities.get("NumberOfInstances"),
+                "num_features": dataset.qualities.get("NumberOfFeatures"),
+                "num_classes": dataset.qualities.get("NumberOfClasses"),
+            }
+        except Exception as e:
+            logger.error(f"Error fetching task info: {e}")
+            return {"name": task_name, "task_id": task_id, "error": str(e)}
+
+
+def run_openml_benchmark(optimizer, task_name: str, evaluator, num_runs: int = 5) -> List[Dict]:
+    """
+    Run optimizer on an OpenML benchmark task.
+
+    Args:
+        optimizer: Optimizer instance
+        task_name: OpenML task name
+        evaluator: Evaluator function
+        num_runs: Number of independent runs
+
+    Returns:
+        List of result dictionaries
+    """
+    if not OPENML_AVAILABLE:
+        raise ImportError("OpenML required")
+
+    OpenMLSuite()
+
+    logger.info(f"Running benchmark: {task_name} with {num_runs} runs")
+
+    results = []
+
+    for run in range(num_runs):
+        logger.info(f"Run {run + 1}/{num_runs}")
+
+        # Reset optimizer for each run
+        if hasattr(optimizer, "reset"):
+            optimizer.reset()
+
+        # Run optimization
+        best = optimizer.optimize(evaluator)
+
+        result = {
+            "run": run + 1,
+            "task": task_name,
+            "best_fitness": best.fitness if hasattr(best, "fitness") else 0.0,
+            "history": optimizer.get_history() if hasattr(optimizer, "get_history") else [],
+        }
+
+        results.append(result)
+
+    logger.info(f"Benchmark complete: {task_name}")
+
+    return results

morphml/benchmarks/problems.py

@@ -0,0 +1,289 @@
+"""Benchmark problems for optimizer evaluation."""
+
+import math
+
+from morphml.core.dsl import Layer, SearchSpace
+from morphml.core.graph import ModelGraph
+
+
+class BenchmarkProblem:
+    """Base class for benchmark problems."""
+
+    def __init__(self, name: str, search_space: SearchSpace):
+        """Initialize benchmark problem."""
+        self.name = name
+        self.search_space = search_space
+
+    def evaluate(self, graph: ModelGraph) -> float:
+        """Evaluate architecture on this problem."""
+        raise NotImplementedError
+
+    def get_optimal_fitness(self) -> float:
+        """Get known optimal fitness if available."""
+        return 1.0
+
+
+class SimpleProblem(BenchmarkProblem):
+    """Simple benchmark problem."""
+
+    def __init__(self):
+        """Initialize simple problem."""
+        space = SearchSpace("simple_problem")
+        space.add_layers(
+            Layer.input(shape=(3, 32, 32)),
+            Layer.conv2d(filters=[32, 64], kernel_size=3),
+            Layer.relu(),
+            Layer.output(units=10),
+        )
+        super().__init__("SimpleProblem", space)
+
+    def evaluate(self, graph: ModelGraph) -> float:
+        """Simple evaluation based on node count."""
+        num_nodes = len(graph.nodes)
+        depth = graph.get_depth()
+
+        # Prefer moderate complexity
+        node_score = 1.0 / (1.0 + abs(num_nodes - 5))
+        depth_score = 1.0 / (1.0 + abs(depth - 4))
+
+        return 0.5 * node_score + 0.5 * depth_score
+
+
+class ComplexProblem(BenchmarkProblem):
+    """Complex benchmark problem with multiple criteria."""
+
+    def __init__(self):
+        """Initialize complex problem."""
+        space = SearchSpace("complex_problem")
+        space.add_layers(
+            Layer.input(shape=(3, 224, 224)),
+            Layer.conv2d(filters=[32, 64, 128, 256], kernel_size=[3, 5, 7]),
+            Layer.relu(),
+            Layer.batchnorm(),
+            Layer.maxpool(pool_size=[2, 3]),
+            Layer.conv2d(filters=[64, 128, 256, 512], kernel_size=[3, 5]),
+            Layer.relu(),
+            Layer.batchnorm(),
+            Layer.maxpool(pool_size=2),
+            Layer.dense(units=[256, 512, 1024, 2048]),
+            Layer.dropout(rate=[0.3, 0.5, 0.7]),
+            Layer.dense(units=[128, 256, 512]),
+            Layer.output(units=1000),
+        )
+        super().__init__("ComplexProblem", space)
+
+    def evaluate(self, graph: ModelGraph) -> float:
+        """Complex evaluation considering multiple factors."""
+        num_nodes = len(graph.nodes)
+        depth = graph.get_depth()
+        width = graph.get_max_width()
+        params = graph.estimate_parameters()
+
+        # Multiple objectives
+        node_score = math.exp(-abs(num_nodes - 12) / 5.0)
+        depth_score = math.exp(-abs(depth - 8) / 3.0)
+        width_score = math.exp(-abs(width - 4) / 2.0)
+        param_score = 1.0 / (1.0 + params / 5000000.0)
+
+        # Weighted combination
+        score = 0.25 * node_score + 0.25 * depth_score + 0.25 * width_score + 0.25 * param_score
+
+        return min(1.0, max(0.0, score))
+
+
+class MultiModalProblem(BenchmarkProblem):
+    """Multi-modal problem with multiple local optima."""
+
+    def __init__(self):
+        """Initialize multi-modal problem."""
+        space = SearchSpace("multimodal_problem")
+        space.add_layers(
+            Layer.input(shape=(3, 32, 32)),
+            Layer.conv2d(filters=[16, 32, 64, 128, 256], kernel_size=[3, 5, 7]),
+            Layer.relu(),
+            Layer.maxpool(pool_size=[2, 3]),
+            Layer.conv2d(filters=[32, 64, 128, 256], kernel_size=[3, 5]),
+            Layer.relu(),
+            Layer.dense(units=[64, 128, 256, 512, 1024]),
+            Layer.dropout(rate=[0.2, 0.3, 0.4, 0.5, 0.6]),
+            Layer.output(units=10),
+        )
+        super().__init__("MultiModalProblem", space)
+
+    def evaluate(self, graph: ModelGraph) -> float:
+        """Evaluation with multiple peaks."""
+        num_nodes = len(graph.nodes)
+        depth = graph.get_depth()
+
+        # Create multiple peaks
+        peak1 = math.exp(-((num_nodes - 6) ** 2 + (depth - 3) ** 2) / 4.0)
+        peak2 = 0.8 * math.exp(-((num_nodes - 9) ** 2 + (depth - 5) ** 2) / 3.0)
+        peak3 = 0.6 * math.exp(-((num_nodes - 12) ** 2 + (depth - 7) ** 2) / 5.0)
+
+        return max(peak1, peak2, peak3)
+
+
+class ConstrainedProblem(BenchmarkProblem):
+    """Problem with hard constraints."""
+
+    def __init__(self):
+        """Initialize constrained problem."""
+        space = SearchSpace("constrained_problem")
+        space.add_layers(
+            Layer.input(shape=(3, 32, 32)),
+            Layer.conv2d(filters=[32, 64, 128], kernel_size=3),
+            Layer.relu(),
+            Layer.maxpool(pool_size=2),
+            Layer.conv2d(filters=[64, 128, 256], kernel_size=3),
+            Layer.relu(),
+            Layer.dense(units=[128, 256, 512]),
+            Layer.output(units=10),
+        )
+
+        # Add constraints
+        def max_depth_constraint(g):
+            return g.get_depth() <= 8
+
+        def max_params_constraint(g):
+            return g.estimate_parameters() <= 1000000
+
+        space.add_constraint(max_depth_constraint)
+        space.add_constraint(max_params_constraint)
+
+        super().__init__("ConstrainedProblem", space)
+
+    def evaluate(self, graph: ModelGraph) -> float:
+        """Evaluation with penalty for constraint violation."""
+        # Check constraints
+        depth = graph.get_depth()
+        params = graph.estimate_parameters()
+
+        penalty = 0.0
+
+        if depth > 8:
+            penalty += 0.5 * (depth - 8) / 8.0
+
+        if params > 1000000:
+            penalty += 0.5 * (params - 1000000) / 1000000.0
+
+        # Base fitness
+        base_fitness = 0.5 + 0.05 * len(graph.nodes)
+
+        # Apply penalty
+        return max(0.0, base_fitness - penalty)
+
+
+class NoisyProblem(BenchmarkProblem):
+    """Problem with noisy evaluations."""
+
+    def __init__(self, noise_level: float = 0.1):
+        """Initialize noisy problem."""
+        space = SearchSpace("noisy_problem")
+        space.add_layers(
+            Layer.input(shape=(3, 32, 32)),
+            Layer.conv2d(filters=[32, 64], kernel_size=3),
+            Layer.relu(),
+            Layer.maxpool(pool_size=2),
+            Layer.dense(units=[128, 256]),
+            Layer.output(units=10),
+        )
+        super().__init__("NoisyProblem", space)
+        self.noise_level = noise_level
+
+    def evaluate(self, graph: ModelGraph) -> float:
+        """Evaluation with added noise."""
+        import random
+
+        # Base evaluation
+        num_nodes = len(graph.nodes)
+        base_fitness = 0.6 + 0.04 * num_nodes
+
+        # Add noise
+        noise = random.gauss(0, self.noise_level)
+        noisy_fitness = base_fitness + noise
+
+        return max(0.0, min(1.0, noisy_fitness))
+
+
+class RastriginProblem(BenchmarkProblem):
+    """Rastrigin-like function adapted for graphs."""
+
+    def __init__(self):
+        """Initialize Rastrigin problem."""
+        space = SearchSpace("rastrigin_problem")
+        space.add_layers(
+            Layer.input(shape=(3, 32, 32)),
+            Layer.conv2d(filters=[16, 32, 64, 128], kernel_size=[3, 5, 7]),
+            Layer.relu(),
+            Layer.maxpool(pool_size=[2, 3]),
+            Layer.dense(units=[64, 128, 256, 512]),
+            Layer.output(units=10),
+        )
+        super().__init__("RastriginProblem", space)
+
+    def evaluate(self, graph: ModelGraph) -> float:
+        """Rastrigin-like evaluation."""
+        num_nodes = len(graph.nodes)
+        depth = graph.get_depth()
+
+        # Rastrigin-like function
+        A = 10
+        x = (num_nodes - 8) / 4.0
+        y = (depth - 5) / 3.0
+
+        value = 2 * A
+        value += x**2 - A * math.cos(2 * math.pi * x)
+        value += y**2 - A * math.cos(2 * math.pi * y)
+
+        # Convert to fitness (0-1 range, higher is better)
+        fitness = 1.0 / (1.0 + value / 20.0)
+
+        return fitness
+
+
+class RosenbrockProblem(BenchmarkProblem):
+    """Rosenbrock-like function adapted for graphs."""
+
+    def __init__(self):
+        """Initialize Rosenbrock problem."""
+        space = SearchSpace("rosenbrock_problem")
+        space.add_layers(
+            Layer.input(shape=(3, 32, 32)),
+            Layer.conv2d(filters=[32, 64, 128], kernel_size=[3, 5]),
+            Layer.relu(),
+            Layer.dense(units=[128, 256, 512]),
+            Layer.output(units=10),
+        )
+        super().__init__("RosenbrockProblem", space)
+
+    def evaluate(self, graph: ModelGraph) -> float:
+        """Rosenbrock-like evaluation."""
+        num_nodes = len(graph.nodes)
+        depth = graph.get_depth()
+
+        # Rosenbrock-like function
+        a = 1
+        b = 100
+
+        x = (num_nodes - 7) / 3.0
+        y = (depth - 4) / 2.0
+
+        value = (a - x) ** 2 + b * (y - x**2) ** 2
+
+        # Convert to fitness
+        fitness = 1.0 / (1.0 + value / 50.0)
+
+        return fitness
+
+
+def get_all_problems() -> list:
+    """Get all benchmark problems."""
+    return [
+        SimpleProblem(),
+        ComplexProblem(),
+        MultiModalProblem(),
+        ConstrainedProblem(),
+        NoisyProblem(noise_level=0.1),
+        RastriginProblem(),
+        RosenbrockProblem(),
+    ]