morphml 1.0.0__py3-none-any.whl

morphml/tracking/logger.py

@@ -0,0 +1,301 @@
+"""Metric logging utilities."""
+
+import json
+from collections import defaultdict
+from typing import Any, Dict, List, Optional
+
+from morphml.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+
+class MetricLogger:
+    """
+    Log metrics during optimization.
+
+    Example:
+        >>> logger = MetricLogger()
+        >>> logger.log("fitness", 0.85, step=10)
+        >>> logger.log("diversity", 0.45, step=10)
+        >>> logger.save("metrics.json")
+    """
+
+    def __init__(self):
+        """Initialize metric logger."""
+        self.metrics: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
+        self.step_counter = 0
+
+    def log(self, metric_name: str, value: Any, step: Optional[int] = None) -> None:
+        """
+        Log a metric value.
+
+        Args:
+            metric_name: Name of metric
+            value: Metric value
+            step: Optional step/generation number
+        """
+        if step is None:
+            step = self.step_counter
+            self.step_counter += 1
+
+        self.metrics[metric_name].append({"step": step, "value": value})
+
+    def log_dict(self, metrics: Dict[str, Any], step: Optional[int] = None) -> None:
+        """
+        Log multiple metrics at once.
+
+        Args:
+            metrics: Dictionary of metric name to value
+            step: Optional step number
+        """
+        for name, value in metrics.items():
+            self.log(name, value, step)
+
+    def get_metric(self, metric_name: str) -> List[Dict[str, Any]]:
+        """Get all values for a metric."""
+        return self.metrics.get(metric_name, [])
+
+    def get_last_value(self, metric_name: str) -> Optional[Any]:
+        """Get last logged value for a metric."""
+        values = self.metrics.get(metric_name, [])
+        return values[-1]["value"] if values else None
+
+    def get_values(self, metric_name: str) -> List[Any]:
+        """Get all values for a metric (without step info)."""
+        return [entry["value"] for entry in self.metrics.get(metric_name, [])]
+
+    def get_steps(self, metric_name: str) -> List[int]:
+        """Get all steps for a metric."""
+        return [entry["step"] for entry in self.metrics.get(metric_name, [])]
+
+    def clear(self) -> None:
+        """Clear all metrics."""
+        self.metrics.clear()
+        self.step_counter = 0
+
+    def save(self, filepath: str) -> None:
+        """Save metrics to JSON file."""
+        with open(filepath, "w") as f:
+            json.dump(dict(self.metrics), f, indent=2)
+
+        logger.info(f"Metrics saved to {filepath}")
+
+    def load(self, filepath: str) -> None:
+        """Load metrics from JSON file."""
+        with open(filepath, "r") as f:
+            data = json.load(f)
+
+        self.metrics = defaultdict(list, data)
+
+        # Update step counter
+        max_step = 0
+        for entries in self.metrics.values():
+            for entry in entries:
+                max_step = max(max_step, entry.get("step", 0))
+
+        self.step_counter = max_step + 1
+
+        logger.info(f"Metrics loaded from {filepath}")
+
+    def get_summary(self) -> Dict[str, Dict[str, Any]]:
+        """Get summary statistics for all metrics."""
+        import statistics
+
+        summary = {}
+
+        for metric_name, entries in self.metrics.items():
+            values = [e["value"] for e in entries if isinstance(e["value"], (int, float))]
+
+            if values:
+                summary[metric_name] = {
+                    "count": len(values),
+                    "mean": statistics.mean(values),
+                    "median": statistics.median(values),
+                    "std": statistics.stdev(values) if len(values) > 1 else 0.0,
+                    "min": min(values),
+                    "max": max(values),
+                    "last": values[-1],
+                }
+
+        return summary
+
+    def print_summary(self) -> None:
+        """Print summary of all metrics."""
+        summary = self.get_summary()
+
+        print("\n" + "=" * 60)
+        print("METRIC SUMMARY")
+        print("=" * 60)
+
+        for metric_name, stats in sorted(summary.items()):
+            print(f"\n{metric_name}:")
+            print(f"  Count:  {stats['count']}")
+            print(f"  Mean:   {stats['mean']:.4f}")
+            print(f"  Median: {stats['median']:.4f}")
+            print(f"  Std:    {stats['std']:.4f}")
+            print(f"  Range:  [{stats['min']:.4f}, {stats['max']:.4f}]")
+            print(f"  Last:   {stats['last']:.4f}")
+
+        print("=" * 60)
+
+
+class CSVLogger(MetricLogger):
+    """Log metrics to CSV file."""
+
+    def __init__(self, filepath: str):
+        """
+        Initialize CSV logger.
+
+        Args:
+            filepath: Path to CSV file
+        """
+        super().__init__()
+        self.filepath = filepath
+        self.file = None
+        self.writer = None
+        self.headers_written = False
+
+    def __enter__(self):
+        """Enter context."""
+        self.file = open(self.filepath, "w", newline="")
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Exit context."""
+        if self.file:
+            self.file.close()
+        return False
+
+    def log_dict(self, metrics: Dict[str, Any], step: Optional[int] = None) -> None:
+        """Log metrics to CSV."""
+        super().log_dict(metrics, step)
+
+        if not self.file:
+            return
+
+        # Write headers if needed
+        if not self.headers_written:
+            import csv
+
+            self.writer = csv.DictWriter(self.file, fieldnames=["step"] + list(metrics.keys()))
+            self.writer.writeheader()
+            self.headers_written = True
+
+        # Write row
+        if self.writer:
+            row = {"step": step if step is not None else self.step_counter - 1}
+            row.update(metrics)
+            self.writer.writerow(row)
+            self.file.flush()
+
+
+class TensorBoardLogger(MetricLogger):
+    """Log metrics to TensorBoard."""
+
+    def __init__(self, log_dir: str):
+        """
+        Initialize TensorBoard logger.
+
+        Args:
+            log_dir: TensorBoard log directory
+        """
+        super().__init__()
+        self.log_dir = log_dir
+        self.writer = None
+
+        try:
+            from torch.utils.tensorboard import SummaryWriter
+
+            self.writer = SummaryWriter(log_dir)
+            logger.info(f"TensorBoard logging to {log_dir}")
+        except ImportError:
+            logger.warning("tensorboard not available")
+
+    def log(self, metric_name: str, value: Any, step: Optional[int] = None) -> None:
+        """Log metric to TensorBoard."""
+        super().log(metric_name, value, step)
+
+        if self.writer and isinstance(value, (int, float)):
+            step = step if step is not None else self.step_counter - 1
+            self.writer.add_scalar(metric_name, value, step)
+
+    def log_histogram(self, name: str, values: List[float], step: int) -> None:
+        """Log histogram to TensorBoard."""
+        if self.writer:
+            import numpy as np
+
+            self.writer.add_histogram(name, np.array(values), step)
+
+    def close(self) -> None:
+        """Close TensorBoard writer."""
+        if self.writer:
+            self.writer.close()
+
+
+class WandbLogger(MetricLogger):
+    """Log metrics to Weights & Biases."""
+
+    def __init__(self, project: str, name: Optional[str] = None, config: Optional[Dict] = None):
+        """
+        Initialize W&B logger.
+
+        Args:
+            project: W&B project name
+            name: Run name
+            config: Configuration dict
+        """
+        super().__init__()
+        self.project = project
+        self.run = None
+
+        try:
+            import wandb
+
+            self.run = wandb.init(project=project, name=name, config=config)
+            logger.info(f"W&B logging to project {project}")
+        except ImportError:
+            logger.warning("wandb not available")
+
+    def log_dict(self, metrics: Dict[str, Any], step: Optional[int] = None) -> None:
+        """Log metrics to W&B."""
+        super().log_dict(metrics, step)
+
+        if self.run:
+            import wandb
+
+            wandb.log(metrics, step=step)
+
+    def finish(self) -> None:
+        """Finish W&B run."""
+        if self.run:
+            import wandb
+
+            wandb.finish()
+
+
+class MultiLogger(MetricLogger):
+    """Log to multiple backends simultaneously."""
+
+    def __init__(self, loggers: List[MetricLogger]):
+        """
+        Initialize multi-logger.
+
+        Args:
+            loggers: List of logger instances
+        """
+        super().__init__()
+        self.loggers = loggers
+
+    def log(self, metric_name: str, value: Any, step: Optional[int] = None) -> None:
+        """Log to all loggers."""
+        super().log(metric_name, value, step)
+
+        for logger_instance in self.loggers:
+            logger_instance.log(metric_name, value, step)
+
+    def log_dict(self, metrics: Dict[str, Any], step: Optional[int] = None) -> None:
+        """Log dict to all loggers."""
+        super().log_dict(metrics, step)
+
+        for logger_instance in self.loggers:
+            logger_instance.log_dict(metrics, step)

morphml/tracking/reporter.py

@@ -0,0 +1,357 @@
+"""Report generation for experiments."""
+
+import json
+from datetime import datetime
+from typing import Any, Dict, List
+
+from morphml.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+
+class Reporter:
+    """
+    Generate reports for experiments.
+
+    Example:
+        >>> reporter = Reporter()
+        >>> report = reporter.generate_markdown_report(experiment)
+        >>> reporter.save_report(report, "report.md")
+    """
+
+    def __init__(self):
+        """Initialize reporter."""
+        pass
+
+    def generate_markdown_report(self, experiment: Any) -> str:
+        """
+        Generate Markdown report.
+
+        Args:
+            experiment: Experiment instance
+
+        Returns:
+            Markdown string
+        """
+        lines = []
+
+        # Header
+        lines.append(f"# Experiment Report: {experiment.name}")
+        lines.append(f"\n**ID:** `{experiment.id}`\n")
+        lines.append(f"**Status:** {experiment.status}")
+        lines.append(f"**Duration:** {experiment.get_duration():.2f} seconds\n")
+
+        # Configuration
+        lines.append("## Configuration\n")
+        lines.append("```json")
+        lines.append(json.dumps(experiment.config, indent=2))
+        lines.append("```\n")
+
+        # Best Result
+        if experiment.best_result:
+            lines.append("## Best Result\n")
+            for key, value in experiment.best_result.items():
+                lines.append(f"- **{key}:** {value}")
+            lines.append("")
+
+        # Metrics Summary
+        if experiment.metrics:
+            lines.append("## Metrics Summary\n")
+            lines.append("| Metric | Count | Final Value |")
+            lines.append("|--------|-------|-------------|")
+
+            for metric_name, entries in experiment.metrics.items():
+                count = len(entries)
+                final_value = entries[-1]["value"] if entries else "N/A"
+                lines.append(f"| {metric_name} | {count} | {final_value} |")
+
+            lines.append("")
+
+        # Artifacts
+        if experiment.artifacts:
+            lines.append("## Artifacts\n")
+            for name, path in experiment.artifacts.items():
+                lines.append(f"- **{name}:** `{path}`")
+            lines.append("")
+
+        # Footer
+        lines.append("---")
+        lines.append(f"\n*Generated on {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}*")
+
+        return "\n".join(lines)
+
+    def generate_html_report(self, experiment: Any) -> str:
+        """
+        Generate HTML report.
+
+        Args:
+            experiment: Experiment instance
+
+        Returns:
+            HTML string
+        """
+        html = [
+            "<!DOCTYPE html>",
+            "<html>",
+            "<head>",
+            f"<title>Experiment Report: {experiment.name}</title>",
+            "<style>",
+            "body { font-family: Arial, sans-serif; margin: 40px; }",
+            "h1 { color: #333; }",
+            "table { border-collapse: collapse; width: 100%; margin: 20px 0; }",
+            "th, td { border: 1px solid #ddd; padding: 12px; text-align: left; }",
+            "th { background-color: #4CAF50; color: white; }",
+            "pre { background: #f4f4f4; padding: 15px; border-radius: 5px; }",
+            (
+                ".metric-card { background: #fff; border: 1px solid #ddd; "
+                "padding: 15px; margin: 10px 0; border-radius: 5px; }"
+            ),
+            "</style>",
+            "</head>",
+            "<body>",
+            f"<h1>Experiment Report: {experiment.name}</h1>",
+            f"<p><strong>ID:</strong> {experiment.id}</p>",
+            f"<p><strong>Status:</strong> {experiment.status}</p>",
+            f"<p><strong>Duration:</strong> {experiment.get_duration():.2f} seconds</p>",
+        ]
+
+        # Configuration
+        html.append("<h2>Configuration</h2>")
+        html.append("<pre>")
+        html.append(json.dumps(experiment.config, indent=2))
+        html.append("</pre>")
+
+        # Best Result
+        if experiment.best_result:
+            html.append("<h2>Best Result</h2>")
+            html.append("<table>")
+            html.append("<tr><th>Metric</th><th>Value</th></tr>")
+            for key, value in experiment.best_result.items():
+                html.append(f"<tr><td>{key}</td><td>{value}</td></tr>")
+            html.append("</table>")
+
+        # Metrics
+        if experiment.metrics:
+            html.append("<h2>Metrics</h2>")
+            for metric_name, entries in experiment.metrics.items():
+                html.append("<div class='metric-card'>")
+                html.append(f"<h3>{metric_name}</h3>")
+                html.append(f"<p>Total entries: {len(entries)}</p>")
+                if entries:
+                    html.append(f"<p>Final value: {entries[-1]['value']}</p>")
+                html.append("</div>")
+
+        # Footer
+        html.append("<hr>")
+        html.append(f"<p><em>Generated on {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}</em></p>")
+        html.append("</body>")
+        html.append("</html>")
+
+        return "\n".join(html)
+
+    def generate_comparison_report(
+        self, experiments: List[Any], metric: str = "best_fitness"
+    ) -> str:
+        """
+        Generate comparison report for multiple experiments.
+
+        Args:
+            experiments: List of experiments
+            metric: Metric to compare
+
+        Returns:
+            Markdown string
+        """
+        lines = []
+
+        lines.append("# Experiment Comparison\n")
+        lines.append(f"**Metric:** {metric}\n")
+        lines.append(f"**Experiments:** {len(experiments)}\n")
+
+        # Table
+        lines.append("| Experiment | Status | Duration (s) | Best Value |")
+        lines.append("|------------|--------|--------------|------------|")
+
+        for exp in experiments:
+            best_value = "N/A"
+            if exp.best_result and metric in exp.best_result:
+                best_value = f"{exp.best_result[metric]:.4f}"
+
+            lines.append(
+                f"| {exp.name} | {exp.status} | " f"{exp.get_duration():.2f} | {best_value} |"
+            )
+
+        lines.append("")
+
+        # Rankings
+        ranked = sorted(
+            [e for e in experiments if e.best_result and metric in e.best_result],
+            key=lambda x: x.best_result[metric],
+            reverse=True,
+        )
+
+        if ranked:
+            lines.append("## Rankings\n")
+            for i, exp in enumerate(ranked, 1):
+                value = exp.best_result[metric]
+                lines.append(f"{i}. **{exp.name}** - {value:.4f}")
+            lines.append("")
+
+        return "\n".join(lines)
+
+    def save_report(self, content: str, filepath: str) -> None:
+        """Save report to file."""
+        with open(filepath, "w") as f:
+            f.write(content)
+
+        logger.info(f"Report saved to {filepath}")
+
+    def generate_latex_report(self, experiment: Any) -> str:
+        """
+        Generate LaTeX report.
+
+        Args:
+            experiment: Experiment instance
+
+        Returns:
+            LaTeX string
+        """
+        lines = [
+            "\\documentclass{article}",
+            "\\usepackage{booktabs}",
+            "\\usepackage{hyperref}",
+            "\\begin{document}",
+            "",
+            f"\\title{{Experiment Report: {experiment.name}}}",
+            "\\maketitle",
+            "",
+            "\\section{Overview}",
+            "",
+            f"\\textbf{{Experiment ID:}} \\texttt{{{experiment.id}}}",
+            "",
+            f"\\textbf{{Status:}} {experiment.status}",
+            "",
+            f"\\textbf{{Duration:}} {experiment.get_duration():.2f} seconds",
+            "",
+        ]
+
+        # Best Result
+        if experiment.best_result:
+            lines.append("\\section{Best Result}")
+            lines.append("\\begin{table}[h]")
+            lines.append("\\centering")
+            lines.append("\\begin{tabular}{ll}")
+            lines.append("\\toprule")
+            lines.append("Metric & Value \\\\")
+            lines.append("\\midrule")
+
+            for key, value in experiment.best_result.items():
+                lines.append(f"{key} & {value} \\\\")
+
+            lines.append("\\bottomrule")
+            lines.append("\\end{tabular}")
+            lines.append("\\end{table}")
+            lines.append("")
+
+        lines.append("\\end{document}")
+
+        return "\n".join(lines)
+
+
+class ProgressReporter:
+    """Report progress during optimization."""
+
+    def __init__(self, report_every: int = 10):
+        """
+        Initialize progress reporter.
+
+        Args:
+            report_every: Report every N steps
+        """
+        self.report_every = report_every
+        self.step = 0
+
+    def report(self, metrics: Dict[str, Any]) -> None:
+        """Report progress."""
+        self.step += 1
+
+        if self.step % self.report_every == 0:
+            self._print_progress(metrics)
+
+    def _print_progress(self, metrics: Dict[str, Any]) -> None:
+        """Print progress to console."""
+        print(f"\n[Step {self.step}]")
+        for key, value in metrics.items():
+            if isinstance(value, float):
+                print(f"  {key}: {value:.4f}")
+            else:
+                print(f"  {key}: {value}")
+
+
+class EmailReporter:
+    """Send email reports (requires SMTP configuration)."""
+
+    def __init__(self, smtp_server: str, smtp_port: int, sender: str, password: str):
+        """Initialize email reporter."""
+        self.smtp_server = smtp_server
+        self.smtp_port = smtp_port
+        self.sender = sender
+        self.password = password
+
+    def send_report(self, recipient: str, subject: str, body: str) -> None:
+        """Send email report."""
+        try:
+            import smtplib
+            from email.mime.text import MIMEText
+
+            msg = MIMEText(body)
+            msg["Subject"] = subject
+            msg["From"] = self.sender
+            msg["To"] = recipient
+
+            with smtplib.SMTP(self.smtp_server, self.smtp_port) as server:
+                server.starttls()
+                server.login(self.sender, self.password)
+                server.send_message(msg)
+
+            logger.info(f"Email report sent to {recipient}")
+
+        except Exception as e:
+            logger.error(f"Failed to send email: {e}")
+
+
+class SlackReporter:
+    """Send reports to Slack."""
+
+    def __init__(self, webhook_url: str):
+        """Initialize Slack reporter."""
+        self.webhook_url = webhook_url
+
+    def send_message(self, text: str) -> None:
+        """Send message to Slack."""
+        try:
+            import requests
+
+            payload = {"text": text}
+            response = requests.post(self.webhook_url, json=payload)
+
+            if response.status_code == 200:
+                logger.info("Slack message sent")
+            else:
+                logger.error(f"Slack message failed: {response.status_code}")
+
+        except Exception as e:
+            logger.error(f"Failed to send Slack message: {e}")
+
+    def send_experiment_report(self, experiment: Any) -> None:
+        """Send experiment report to Slack."""
+        text = f"*Experiment Complete:* {experiment.name}\n"
+        text += f"Status: {experiment.status}\n"
+        text += f"Duration: {experiment.get_duration():.2f}s\n"
+
+        if experiment.best_result:
+            text += "\n*Best Result:*\n"
+            for key, value in list(experiment.best_result.items())[:5]:
+                text += f"• {key}: {value}\n"
+
+        self.send_message(text)

morphml/utils/__init__.py

@@ -0,0 +1,6 @@
+"""Utility functions and helpers."""
+
+from morphml.utils.checkpoint import Checkpoint
+from morphml.utils.export import ArchitectureExporter
+
+__all__ = ["Checkpoint", "ArchitectureExporter"]