autoforge-engine 0.1.0__py3-none-any.whl

modelforge/ranking.py

@@ -0,0 +1,351 @@
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+
+class RankingEngine:
+    """
+    Rank ModelForge models using multiple objectives.
+
+    The engine does not train models.
+    It only consumes model evaluation results.
+    """
+
+    DEFAULT_WEIGHTS = {
+        "balanced": {
+            "primary": 0.45,
+            "error": 0.25,
+            "stability": 0.20,
+            "speed": 0.10,
+        },
+        "performance": {
+            "primary": 0.70,
+            "error": 0.15,
+            "stability": 0.10,
+            "speed": 0.05,
+        },
+        "error": {
+            "primary": 0.20,
+            "error": 0.60,
+            "stability": 0.15,
+            "speed": 0.05,
+        },
+        "speed": {
+            "primary": 0.20,
+            "error": 0.15,
+            "stability": 0.15,
+            "speed": 0.50,
+        },
+    }
+
+    def rank(
+        self,
+        results: pd.DataFrame,
+        task_type: str,
+        objective: str = "balanced",
+    ) -> pd.DataFrame:
+        """
+        Rank models using multiple objectives.
+        """
+
+        self._validate_inputs(
+            results=results,
+            task_type=task_type,
+            objective=objective,
+        )
+
+        ranked = results.copy()
+
+        weights = self.DEFAULT_WEIGHTS[
+            objective
+        ]
+
+        primary_metric = (
+            self._primary_metric(
+                ranked,
+                task_type,
+            )
+        )
+
+        error_metric = (
+            self._error_metric(
+                ranked,
+                task_type,
+            )
+        )
+
+        stability_metric = (
+            self._stability_metric(
+                ranked,
+                task_type,
+            )
+        )
+
+        speed_metric = self._speed_metric(ranked)
+
+        ranked["primary_score"] = (
+            self._normalize_metric(
+                ranked[primary_metric],
+                maximize=True,
+            )
+        )
+
+        ranked["error_score"] = (
+            self._normalize_metric(
+                ranked[error_metric],
+                maximize=False,
+            )
+        )
+
+        ranked["stability_score"] = (
+            self._normalize_metric(
+                ranked[stability_metric],
+                maximize=False,
+            )
+        )
+
+        if speed_metric is None:
+            ranked["speed_score"] = 1.0
+        else:
+            ranked["speed_score"] = self._normalize_metric(
+                ranked[speed_metric],
+                maximize=False,
+            )
+
+        ranked["overall_score"] = (
+            ranked["primary_score"]
+            * weights["primary"]
+            + ranked["error_score"]
+            * weights["error"]
+            + ranked["stability_score"]
+            * weights["stability"]
+            + ranked["speed_score"]
+            * weights["speed"]
+        )
+
+        ranked["rank"] = (
+            ranked["overall_score"]
+            .rank(
+                ascending=False,
+                method="min",
+            )
+            .astype(int)
+        )
+
+        return (
+            ranked.sort_values(
+                by=[
+                    "rank",
+                    "overall_score",
+                ],
+                ascending=[
+                    True,
+                    False,
+                ],
+            )
+            .reset_index(drop=True)
+        )
+
+    @staticmethod
+    def _primary_metric(
+        results: pd.DataFrame,
+        task_type: str,
+    ) -> str:
+        """Return the primary predictive metric."""
+
+        if task_type == "regression":
+            if "cv_mean_r2" in results.columns:
+                return "cv_mean_r2"
+
+            return "r2"
+
+        if "cv_mean_f1" in results.columns:
+            return "cv_mean_f1"
+
+        return "f1"
+
+    @staticmethod
+    def _error_metric(
+        results: pd.DataFrame,
+        task_type: str,
+    ) -> str:
+        """Return the primary error metric."""
+
+        if task_type == "regression":
+            if "cv_mean_rmse" in results.columns:
+                return "cv_mean_rmse"
+
+            if "mae" in results.columns:
+                return "mae"
+
+            return "rmse"
+
+        if "cv_mean_log_loss" in results.columns:
+            return "cv_mean_log_loss"
+
+        if "log_loss" in results.columns:
+            return "log_loss"
+
+        return "f1"
+
+    @staticmethod
+    def _stability_metric(
+        results: pd.DataFrame,
+        task_type: str,
+    ) -> str:
+        """Return the CV stability metric."""
+
+        if task_type == "regression":
+            if "cv_std_r2" in results.columns:
+                return "cv_std_r2"
+
+            return "r2"
+
+        if "cv_std_f1" in results.columns:
+            return "cv_std_f1"
+
+        return "f1"
+
+    @staticmethod
+    def _speed_metric(
+        results: pd.DataFrame,
+    ) -> str | None:
+        """Return the available training-time metric."""
+
+        if (
+            "total_time_seconds"
+            in results.columns
+        ):
+            return "total_time_seconds"
+
+        if (
+            "training_time_seconds"
+            in results.columns
+        ):
+            return "training_time_seconds"
+
+        return None
+
+    @staticmethod
+    def _normalize_metric(
+        values: pd.Series,
+        maximize: bool,
+    ) -> pd.Series:
+        """
+        Normalize values into [0, 1].
+
+        Higher values are better when maximize=True.
+        Lower values are better when maximize=False.
+        """
+
+        numeric = pd.to_numeric(
+            values,
+            errors="coerce",
+        )
+
+        if numeric.isna().all():
+            return pd.Series(
+                0.0,
+                index=values.index,
+            )
+
+        minimum = numeric.min()
+        maximum = numeric.max()
+
+        if np.isclose(
+            minimum,
+            maximum,
+        ):
+            return pd.Series(
+                1.0,
+                index=values.index,
+            )
+
+        if maximize:
+            normalized = (
+                numeric - minimum
+            ) / (
+                maximum - minimum
+            )
+        else:
+            normalized = (
+                maximum - numeric
+            ) / (
+                maximum - minimum
+            )
+
+        return normalized.fillna(0.0)
+
+    @classmethod
+    def available_objectives(
+        cls,
+    ) -> list[str]:
+        """Return supported ranking objectives."""
+
+        return list(
+            cls.DEFAULT_WEIGHTS.keys()
+        )
+
+    @classmethod
+    def objective_weights(
+        cls,
+        objective: str,
+    ) -> dict[str, float]:
+        """Return weights for an objective."""
+
+        if objective not in cls.DEFAULT_WEIGHTS:
+            raise KeyError(
+                f"Unknown objective: {objective}"
+            )
+
+        return cls.DEFAULT_WEIGHTS[
+            objective
+        ].copy()
+
+    @staticmethod
+    def _validate_inputs(
+        results: pd.DataFrame,
+        task_type: str,
+        objective: str,
+    ) -> None:
+        """Validate ranking inputs."""
+
+        if not isinstance(
+            results,
+            pd.DataFrame,
+        ):
+            raise TypeError(
+                "results must be a pandas DataFrame."
+            )
+
+        if results.empty:
+            raise ValueError(
+                "Cannot rank an empty results dataset."
+            )
+
+        if task_type not in {
+            "regression",
+            "classification",
+        }:
+            raise ValueError(
+                "task_type must be 'regression' "
+                "or 'classification'."
+            )
+
+        if objective not in {
+            "balanced",
+            "performance",
+            "error",
+            "speed",
+        }:
+            raise ValueError(
+                "objective must be one of: "
+                "balanced, performance, "
+                "error, speed."
+            )
+
+        if "model" not in results.columns:
+            raise ValueError(
+                "Results must contain a 'model' column."
+            )

modelforge/reproducibility.py

@@ -0,0 +1,295 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import platform
+import sys
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+import pandas as pd
+import sklearn
+
+
+class ReproducibilityManager:
+    """
+    Creates deterministic fingerprints and reproducibility metadata
+    for ModelForge experiments.
+    """
+
+    def __init__(self, random_state: int = 42) -> None:
+        if not isinstance(random_state, int):
+            raise TypeError("random_state must be an integer.")
+
+        self.random_state = random_state
+
+    def dataset_fingerprint(
+        self,
+        data: pd.DataFrame,
+    ) -> str:
+        """
+        Generate a deterministic SHA-256 fingerprint for a DataFrame.
+        """
+
+        if not isinstance(data, pd.DataFrame):
+            raise TypeError("data must be a pandas DataFrame.")
+
+        if data.empty:
+            raise ValueError("data must not be empty.")
+
+        normalized = data.copy()
+
+        # Normalize column ordering only through explicit representation.
+        # Original column order is preserved because it can affect a pipeline.
+        normalized.columns = [str(column) for column in normalized.columns]
+
+        payload = pd.util.hash_pandas_object(
+            normalized,
+            index=True,
+        ).values.tobytes()
+
+        metadata = {
+            "columns": normalized.columns.tolist(),
+            "dtypes": {
+                column: str(dtype)
+                for column, dtype in normalized.dtypes.items()
+            },
+            "shape": list(normalized.shape),
+        }
+
+        metadata_bytes = json.dumps(
+            metadata,
+            sort_keys=True,
+            default=str,
+        ).encode("utf-8")
+
+        digest = hashlib.sha256()
+        digest.update(payload)
+        digest.update(metadata_bytes)
+
+        return digest.hexdigest()
+
+    def configuration_fingerprint(
+        self,
+        configuration: dict[str, Any],
+    ) -> str:
+        """
+        Generate a deterministic SHA-256 fingerprint for configuration.
+        """
+
+        if not isinstance(configuration, dict):
+            raise TypeError("configuration must be a dictionary.")
+
+        serialized = json.dumps(
+            configuration,
+            sort_keys=True,
+            default=str,
+            separators=(",", ":"),
+        )
+
+        return hashlib.sha256(
+            serialized.encode("utf-8")
+        ).hexdigest()
+
+    def artifact_fingerprint(
+        self,
+        path: str | Path,
+    ) -> str:
+        """
+        Generate a SHA-256 fingerprint for a file artifact.
+        """
+
+        file_path = Path(path)
+
+        if not file_path.exists():
+            raise FileNotFoundError(
+                f"Artifact does not exist: {file_path}"
+            )
+
+        if not file_path.is_file():
+            raise ValueError(
+                f"Artifact path is not a file: {file_path}"
+            )
+
+        digest = hashlib.sha256()
+
+        with file_path.open("rb") as file:
+            for chunk in iter(lambda: file.read(1024 * 1024), b""):
+                digest.update(chunk)
+
+        return digest.hexdigest()
+
+    def environment_metadata(self) -> dict[str, Any]:
+        """
+        Capture environment information required to reproduce an experiment.
+        """
+
+        return {
+            "python_version": platform.python_version(),
+            "python_implementation": platform.python_implementation(),
+            "platform": platform.platform(),
+            "machine": platform.machine(),
+            "numpy_version": np.__version__,
+            "pandas_version": pd.__version__,
+            "scikit_learn_version": sklearn.__version__,
+            "random_state": self.random_state,
+        }
+
+    def create_snapshot(
+        self,
+        data: pd.DataFrame,
+        configuration: dict[str, Any] | None = None,
+        target: str | None = None,
+        task_type: str | None = None,
+        extra_metadata: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        """
+        Create a complete reproducibility snapshot for an experiment.
+        """
+
+        if not isinstance(data, pd.DataFrame):
+            raise TypeError("data must be a pandas DataFrame.")
+
+        if data.empty:
+            raise ValueError("data must not be empty.")
+
+        configuration = configuration or {}
+        dataset_fingerprint = self.dataset_fingerprint(data)
+        configuration_fingerprint = (
+            self.configuration_fingerprint(configuration)
+        )
+
+        snapshot: dict[str, Any] = {
+            "dataset_fingerprint": dataset_fingerprint,
+            "configuration_fingerprint": configuration_fingerprint,
+            "dataset": {
+                "fingerprint": dataset_fingerprint,
+                "rows": int(data.shape[0]),
+                "columns": int(data.shape[1]),
+                "column_names": [
+                    str(column)
+                    for column in data.columns
+                ],
+            },
+            "configuration": {
+                "fingerprint": configuration_fingerprint,
+                "values": configuration,
+            },
+            "target": target,
+            "task_type": task_type,
+            "environment": self.environment_metadata(),
+        }
+
+        if extra_metadata:
+            snapshot["extra_metadata"] = extra_metadata
+
+        return snapshot
+
+    def save_snapshot(
+        self,
+        snapshot: dict[str, Any],
+        path: str | Path,
+        overwrite: bool = False,
+    ) -> str:
+        """
+        Save a reproducibility snapshot as JSON.
+        """
+
+        if not isinstance(snapshot, dict):
+            raise TypeError("snapshot must be a dictionary.")
+
+        output_path = Path(path)
+
+        if output_path.exists() and not overwrite:
+            raise FileExistsError(
+                f"Snapshot already exists: {output_path}"
+            )
+
+        output_path.parent.mkdir(
+            parents=True,
+            exist_ok=True,
+        )
+
+        with output_path.open(
+            "w",
+            encoding="utf-8",
+        ) as file:
+            json.dump(
+                snapshot,
+                file,
+                indent=2,
+                sort_keys=True,
+                default=str,
+            )
+
+        return str(output_path)
+
+    def load_snapshot(
+        self,
+        path: str | Path,
+    ) -> dict[str, Any]:
+        """
+        Load a reproducibility snapshot from JSON.
+        """
+
+        input_path = Path(path)
+
+        if not input_path.exists():
+            raise FileNotFoundError(
+                f"Snapshot does not exist: {input_path}"
+            )
+
+        if not input_path.is_file():
+            raise ValueError(
+                f"Snapshot path is not a file: {input_path}"
+            )
+
+        with input_path.open(
+            "r",
+            encoding="utf-8",
+        ) as file:
+            snapshot = json.load(file)
+
+        if not isinstance(snapshot, dict):
+            raise ValueError(
+                "Invalid reproducibility snapshot."
+            )
+
+        return snapshot
+
+    def verify_dataset(
+        self,
+        data: pd.DataFrame,
+        expected_fingerprint: str,
+    ) -> bool:
+        """
+        Verify that a DataFrame matches an expected fingerprint.
+        """
+
+        if not isinstance(expected_fingerprint, str):
+            raise TypeError(
+                "expected_fingerprint must be a string."
+            )
+
+        actual_fingerprint = self.dataset_fingerprint(data)
+
+        return actual_fingerprint == expected_fingerprint
+
+    def verify_artifact(
+        self,
+        path: str | Path,
+        expected_fingerprint: str,
+    ) -> bool:
+        """
+        Verify that a file matches an expected SHA-256 fingerprint.
+        """
+
+        if not isinstance(expected_fingerprint, str):
+            raise TypeError(
+                "expected_fingerprint must be a string."
+            )
+
+        actual_fingerprint = self.artifact_fingerprint(path)
+
+        return actual_fingerprint == expected_fingerprint