mentis 0.1.0__py3-none-any.whl

mentis/__init__.py

@@ -0,0 +1,4 @@
+from mentis.config import MentisConfig
+from mentis.guardian import Guardian
+
+__all__ = ["Guardian", "MentisConfig"]

mentis/cli/__init__.py

@@ -0,0 +1,8 @@
+"""
+CLI subpackage: Typer-based command-line interface for Mentis.
+"""
+
+from mentis.cli.main import app
+
+__all__ = ["app"]
+

mentis/cli/main.py

@@ -0,0 +1,177 @@
+"""
+Command-line interface for Mentis, built with Typer.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pandas as pd
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from mentis import Guardian, MentisConfig
+from mentis.exceptions import MentisError
+
+app = typer.Typer(name="mentis", help="Mentis: an AI engineer's toolkit.")
+console = Console()
+
+
+def _load_guardian(config: str | None) -> Guardian:
+    return Guardian.from_yaml(config) if config else Guardian()
+
+
+def _load_dataframe(data_path: str) -> pd.DataFrame:
+    path = Path(data_path)
+    if not path.exists():
+        console.print(f"[red]File not found: {data_path}[/red]")
+        raise typer.Exit(code=1)
+    if path.suffix == ".parquet":
+        return pd.read_parquet(path)
+    return pd.read_csv(path)
+
+
+@app.command()
+def scan(
+    data: str = typer.Argument(..., help="Path to CSV/Parquet dataset."),
+    target: str | None = typer.Option(None, help="Target column name."),
+    config: str | None = typer.Option(None, help="Path to a Mentis YAML config."),
+) -> None:
+    """Run the Dataset Scanner on a CSV/Parquet file."""
+    guardian = _load_guardian(config)
+    df = _load_dataframe(data)
+
+    with console.status("Scanning dataset..."):
+        try:
+            result = guardian.scan(df, target=target)
+        except MentisError as exc:
+            console.print(f"[red]Scan failed: {exc}[/red]")
+            raise typer.Exit(code=1)
+
+    console.print(result)
+    for finding in result.warnings():
+        console.print(f"[yellow]⚠ {finding.message}[/yellow] -> {finding.suggestion}")
+
+
+@app.command()
+def compare(
+    config: str = typer.Argument(..., help="Path to a Mentis YAML config."),
+    data: str = typer.Option(..., help="Path to CSV/Parquet dataset."),
+) -> None:
+    """Train and compare models using settings from a YAML config."""
+    guardian = Guardian.from_yaml(config)
+    df = _load_dataframe(data)
+
+    target = guardian.config.project.target
+    if not target:
+        console.print("[red]Config must specify project.target for comparison.[/red]")
+        raise typer.Exit(code=1)
+
+    from sklearn.model_selection import train_test_split
+
+    X = df.drop(columns=[target])
+    y = df[target]
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
+
+    with console.status("Training and comparing models..."):
+        try:
+            leaderboard = guardian.compare_models(X_train, X_test, y_train, y_test)
+        except MentisError as exc:
+            console.print(f"[red]Comparison failed: {exc}[/red]")
+            raise typer.Exit(code=1)
+
+    table = Table(title="Model Leaderboard")
+    table.add_column("Model")
+    table.add_column(leaderboard.primary_metric)
+    for r in leaderboard.results:
+        score = r.metrics.get(leaderboard.primary_metric, "N/A")
+        table.add_row(r.model_name, f"{score:.4f}" if isinstance(score, float) else str(score))
+    console.print(table)
+
+
+@app.command()
+def audit(
+    project_path: str = typer.Argument(".", help="Path to the project to audit.")
+) -> None:
+    """Audit an ML project's structure and production readiness."""
+    guardian = Guardian()
+    with console.status("Auditing project..."):
+        result = guardian.audit_pipeline(project_path)
+
+    console.print(f"[bold]Production Readiness Score: {result.score}/100[/bold]")
+    for finding in result.failed():
+        console.print(f"[yellow]✗ {finding.name}[/yellow] ({finding.severity}) -> {finding.suggestion}")
+
+
+@app.command(name="deploy-check")
+def deploy_check(
+    project_path: str = typer.Argument(".", help="Path to the project to check.")
+) -> None:
+    """Check a project's deployment readiness."""
+    guardian = Guardian()
+    with console.status("Checking deployment readiness..."):
+        result = guardian.deploy_check(project_path)
+
+    console.print(f"[bold]Deployment Score: {result.score}/100[/bold]")
+    if result.detected_framework:
+        console.print(f"Framework detected: {result.detected_framework}")
+    for finding in result.failed():
+        console.print(f"[yellow]✗ {finding.name}[/yellow] ({finding.severity}) -> {finding.suggestion}")
+
+
+@app.command()
+def report(
+    data: str | None = typer.Option(None, help="Path to CSV/Parquet dataset (optional, enables scan)."),
+    target: str | None = typer.Option(None, help="Target column name."),
+    project_path: str = typer.Option(".", help="Project path for audit + deploy checks."),
+    output_dir: str = typer.Option("mentis_reports", help="Output directory for the report."),
+    fmt: str = typer.Option("html", help="Report format: html, markdown, or pdf."),
+    config: str | None = typer.Option(None, help="Path to a Mentis YAML config."),
+) -> None:
+    """
+    Generate a Mentis report: optionally scans a dataset, then always
+    runs pipeline audit and deployment checks, and writes the report.
+    """
+    guardian = _load_guardian(config)
+
+    if data:
+        df = _load_dataframe(data)
+        with console.status("Scanning dataset..."):
+            try:
+                guardian.scan(df, target=target)
+            except MentisError as exc:
+                console.print(f"[yellow]⚠ Scan skipped: {exc}[/yellow]")
+
+    with console.status("Auditing project structure..."):
+        try:
+            guardian.audit_pipeline(project_path)
+        except Exception as exc:
+            console.print(f"[yellow]⚠ Audit skipped: {exc}[/yellow]")
+
+    with console.status("Checking deployment readiness..."):
+        try:
+            guardian.deploy_check(project_path)
+        except Exception as exc:
+            console.print(f"[yellow]⚠ Deploy check skipped: {exc}[/yellow]")
+
+    with console.status(f"Generating {fmt.upper()} report..."):
+        try:
+            path = guardian.generate_report(output_path=output_dir, fmt=fmt)
+        except MentisError as exc:
+            console.print(f"[red]Report generation failed: {exc}[/red]")
+            raise typer.Exit(code=1)
+
+    console.print(f"[green]✓ Report written to:[/green] {path}")
+
+
+def main() -> None:
+    app()
+
+
+if __name__ == "__main__":
+    main()
+
+
+
+    

mentis/comparison/__init__.py

@@ -0,0 +1,21 @@
+"""
+Comparison subpackage: automated multi-model training, evaluation,
+and leaderboard ranking for Mentis.
+"""
+
+from mentis.comparison.leaderboard import Leaderboard, ModelResult, build_leaderboard
+from mentis.comparison.metrics import compute_metrics, primary_metric_for_task
+from mentis.comparison.model_zoo import get_model_zoo
+from mentis.comparison.trainer import ModelTrainer
+
+__all__ = [
+    "ModelTrainer",
+    "Leaderboard",
+    "ModelResult",
+    "build_leaderboard",
+    "get_model_zoo",
+    "compute_metrics",
+    "primary_metric_for_task",
+]
+
+

mentis/comparison/leaderboard.py

@@ -0,0 +1,155 @@
+"""
+Leaderboard construction for Mentis model comparison results.
+
+Separated from the trainer so ranking/formatting logic can evolve
+(e.g. weighted scoring, custom tie-breakers) without touching the
+training loop itself.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field, asdict
+from typing import Any
+
+from mentis.comparison.metrics import is_higher_better
+
+
+@dataclass
+class ModelResult:
+    """
+    Result of training and evaluating a single model.
+
+    Attributes:
+        model_name: Name of the model (e.g. "Random Forest").
+        metrics: Dict of metric name -> value on the test set.
+        cv_scores: Cross-validation scores (primary metric) per fold.
+        cv_mean: Mean cross-validation score.
+        cv_std: Standard deviation of cross-validation scores.
+        training_time_seconds: Wall-clock time spent fitting the model.
+        inference_time_seconds: Wall-clock time spent predicting on the
+            test set.
+        memory_usage_mb: Approximate memory footprint of the fitted
+            model.
+        feature_importances: Optional dict of feature name -> importance
+            score, if the model exposes one.
+        error: Populated if training/evaluation failed for this model,
+            instead of raising and aborting the whole comparison.
+    """
+
+    model_name: str
+    metrics: dict[str, float] = field(default_factory=dict)
+    cv_scores: list[float] = field(default_factory=list)
+    cv_mean: float = 0.0
+    cv_std: float = 0.0
+    training_time_seconds: float = 0.0
+    inference_time_seconds: float = 0.0
+    memory_usage_mb: float = 0.0
+    feature_importances: dict[str, float] | None = None
+    error: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+@dataclass
+class Leaderboard:
+    """
+    Ranked collection of `ModelResult`s produced by a model comparison
+    run.
+
+    Attributes:
+        task: "classification" or "regression".
+        primary_metric: The metric used to rank models.
+        results: All `ModelResult`s, sorted best-to-worst by
+            `primary_metric`.
+    """
+
+    task: str
+    primary_metric: str
+    results: list[ModelResult] = field(default_factory=list)
+
+    def best_model(self) -> ModelResult | None:
+        """Return the top-ranked `ModelResult`, or None if empty."""
+        successful = [r for r in self.results if r.error is None]
+        return successful[0] if successful else None
+
+    def to_dataframe(self):
+        """
+        Return the leaderboard as a pandas DataFrame, sorted by rank.
+
+        Returns:
+            A `pandas.DataFrame` with one row per model.
+
+        Examples:
+            >>> lb = Leaderboard(task="classification", primary_metric="f1")
+            >>> lb.to_dataframe().empty
+            True
+        """
+        import pandas as pd
+
+        rows = []
+        for r in self.results:
+            row = {"model": r.model_name, "error": r.error}
+            row.update(r.metrics)
+            row["cv_mean"] = r.cv_mean
+            row["cv_std"] = r.cv_std
+            row["training_time_s"] = r.training_time_seconds
+            row["inference_time_s"] = r.inference_time_seconds
+            row["memory_mb"] = r.memory_usage_mb
+            rows.append(row)
+
+        return pd.DataFrame(rows)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "task": self.task,
+            "primary_metric": self.primary_metric,
+            "results": [r.to_dict() for r in self.results],
+        }
+
+    def __repr__(self) -> str:
+        best = self.best_model()
+        best_name = best.model_name if best else "N/A"
+        return f"<Leaderboard task={self.task!r} best_model={best_name!r} n_models={len(self.results)}>"
+
+
+def build_leaderboard(
+    task: str,
+    model_results: list[ModelResult],
+    primary_metric: str,
+) -> Leaderboard:
+    """
+    Sort model results into a ranked `Leaderboard`.
+
+    Args:
+        task: "classification" or "regression".
+        model_results: Unsorted list of `ModelResult`s.
+        primary_metric: Metric name used for ranking.
+
+    Returns:
+        A `Leaderboard` with `results` sorted best-to-worst. Models
+        that errored are placed last, in original order.
+
+    Examples:
+        >>> results = [
+        ...     ModelResult(model_name="A", metrics={"f1": 0.8}),
+        ...     ModelResult(model_name="B", metrics={"f1": 0.9}),
+        ... ]
+        >>> lb = build_leaderboard("classification", results, "f1")
+        >>> lb.results[0].model_name
+        'B'
+    """
+    higher_better = is_higher_better(primary_metric)
+
+    successful = [r for r in model_results if r.error is None]
+    failed = [r for r in model_results if r.error is not None]
+
+    successful.sort(
+        key=lambda r: r.metrics.get(primary_metric, float("-inf") if higher_better else float("inf")),
+        reverse=higher_better,
+    )
+
+    return Leaderboard(task=task, primary_metric=primary_metric, results=successful + failed)
+
+
+

mentis/comparison/metrics.py

@@ -0,0 +1,183 @@
+"""
+Metric computation for classification and regression model comparison.
+
+Centralizing metric logic here means the trainer and leaderboard never
+duplicate scikit-learn calls, and adding a new metric only requires a
+change in one place.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import numpy as np
+from sklearn.metrics import (
+    accuracy_score,
+    f1_score,
+    mean_absolute_error,
+    mean_absolute_percentage_error,
+    mean_squared_error,
+    precision_score,
+    r2_score,
+    recall_score,
+    roc_auc_score,
+)
+
+from mentis.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def compute_classification_metrics(
+    y_true: np.ndarray,
+    y_pred: np.ndarray,
+    y_proba: np.ndarray | None = None,
+) -> dict[str, float]:
+    """
+    Compute standard classification metrics.
+
+    Args:
+        y_true: Ground-truth labels.
+        y_pred: Predicted labels.
+        y_proba: Predicted probabilities for the positive class
+            (binary) or full probability matrix (multiclass), used for
+            ROC AUC. If None, ROC AUC is omitted.
+
+    Returns:
+        Dict mapping metric name -> value. Keys: "accuracy",
+        "precision", "recall", "f1", and "roc_auc" if computable.
+
+    Examples:
+        >>> import numpy as np
+        >>> y_true = np.array([0, 1, 1, 0])
+        >>> y_pred = np.array([0, 1, 0, 0])
+        >>> metrics = compute_classification_metrics(y_true, y_pred)
+        >>> round(metrics["accuracy"], 2)
+        0.75
+    """
+    average = "binary" if len(np.unique(y_true)) == 2 else "macro"
+
+    metrics: dict[str, float] = {
+        "accuracy": float(accuracy_score(y_true, y_pred)),
+        "precision": float(precision_score(y_true, y_pred, average=average, zero_division=0)),
+        "recall": float(recall_score(y_true, y_pred, average=average, zero_division=0)),
+        "f1": float(f1_score(y_true, y_pred, average=average, zero_division=0)),
+    }
+
+    if y_proba is not None:
+        try:
+            if y_proba.ndim == 2 and y_proba.shape[1] == 2:
+                auc = roc_auc_score(y_true, y_proba[:, 1])
+            elif y_proba.ndim == 1:
+                auc = roc_auc_score(y_true, y_proba)
+            else:
+                auc = roc_auc_score(y_true, y_proba, multi_class="ovr", average="macro")
+            metrics["roc_auc"] = float(auc)
+        except ValueError as exc:
+            logger.warning(f"Could not compute ROC AUC: {exc}")
+
+    return metrics
+
+
+def compute_regression_metrics(y_true: np.ndarray, y_pred: np.ndarray) -> dict[str, float]:
+    """
+    Compute standard regression metrics.
+
+    Args:
+        y_true: Ground-truth target values.
+        y_pred: Predicted target values.
+
+    Returns:
+        Dict with keys "rmse", "mae", "mape", "r2".
+
+    Examples:
+        >>> import numpy as np
+        >>> y_true = np.array([3.0, 5.0, 2.5])
+        >>> y_pred = np.array([2.8, 5.1, 2.4])
+        >>> metrics = compute_regression_metrics(y_true, y_pred)
+        >>> metrics["r2"] > 0.9
+        True
+    """
+    rmse = float(np.sqrt(mean_squared_error(y_true, y_pred)))
+    mae = float(mean_absolute_error(y_true, y_pred))
+    r2 = float(r2_score(y_true, y_pred))
+
+    try:
+        mape = float(mean_absolute_percentage_error(y_true, y_pred))
+    except ValueError:
+        mape = float("nan")
+
+    return {"rmse": rmse, "mae": mae, "mape": mape, "r2": r2}
+
+
+def compute_metrics(
+    task: str,
+    y_true: np.ndarray,
+    y_pred: np.ndarray,
+    y_proba: np.ndarray | None = None,
+) -> dict[str, float]:
+    """
+    Dispatch to the correct metric set based on task type.
+
+    Args:
+        task: "classification" or "regression".
+        y_true: Ground-truth values.
+        y_pred: Predicted values.
+        y_proba: Predicted probabilities (classification only).
+
+    Returns:
+        Dict of computed metrics for the given task.
+
+    Raises:
+        ValueError: If `task` is not "classification" or "regression".
+
+    Examples:
+        >>> import numpy as np
+        >>> compute_metrics("regression", np.array([1.0, 2.0]), np.array([1.1, 1.9]))["mae"] < 0.2
+        True
+    """
+    if task == "classification":
+        return compute_classification_metrics(y_true, y_pred, y_proba)
+    if task == "regression":
+        return compute_regression_metrics(y_true, y_pred)
+    raise ValueError(f"Unsupported task type: {task!r}")
+
+
+def primary_metric_for_task(task: str) -> str:
+    """
+    Return the default metric used to rank models on the leaderboard.
+
+    Args:
+        task: "classification" or "regression".
+
+    Returns:
+        "f1" for classification, "r2" for regression.
+
+    Examples:
+        >>> primary_metric_for_task("classification")
+        'f1'
+    """
+    return "f1" if task == "classification" else "r2"
+
+
+def is_higher_better(metric_name: str) -> bool:
+    """
+    Whether a higher value of the given metric indicates better
+    performance (used to sort the leaderboard correctly).
+
+    Args:
+        metric_name: Name of the metric, e.g. "rmse", "f1", "r2".
+
+    Returns:
+        False for error-style metrics (rmse, mae, mape); True otherwise.
+
+    Examples:
+        >>> is_higher_better("rmse")
+        False
+        >>> is_higher_better("f1")
+        True
+    """
+    lower_is_better = {"rmse", "mae", "mape"}
+    return metric_name not in lower_is_better
+
+