autoforge-engine 0.1.0__py3-none-any.whl

modelforge/reproducibility_integration.py

@@ -0,0 +1,192 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+from modelforge.reproducibility import ReproducibilityManager
+
+
+class ReproducibilityIntegration:
+    """
+    Integrates reproducibility metadata with ModelForge runs,
+    experiments, and saved model artifacts.
+    """
+
+    def __init__(self, random_state: int = 42) -> None:
+        self.manager = ReproducibilityManager(
+            random_state=random_state
+        )
+
+    def create_run_snapshot(
+        self,
+        data: pd.DataFrame,
+        configuration: dict[str, Any],
+        target: str | None = None,
+        task_type: str | None = None,
+        run_id: str | None = None,
+        extra_metadata: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        """
+        Create reproducibility information for a ModelForge run.
+        """
+
+        metadata = dict(extra_metadata or {})
+
+        if run_id is not None:
+            metadata["run_id"] = run_id
+
+        return self.manager.create_snapshot(
+            data=data,
+            configuration=configuration,
+            target=target,
+            task_type=task_type,
+            extra_metadata=metadata,
+        )
+
+    def attach_to_result(
+        self,
+        result: dict[str, Any],
+        snapshot: dict[str, Any],
+    ) -> dict[str, Any]:
+        """
+        Attach reproducibility information to a ModelForge result.
+        """
+
+        if not isinstance(result, dict):
+            raise TypeError("result must be a dictionary.")
+
+        if not isinstance(snapshot, dict):
+            raise TypeError("snapshot must be a dictionary.")
+
+        result["reproducibility"] = snapshot
+
+        return result
+
+    def save_snapshot(
+        self,
+        snapshot: dict[str, Any],
+        directory: str | Path,
+        run_id: str,
+        overwrite: bool = False,
+    ) -> str:
+        """
+        Save a run reproducibility snapshot inside an experiment directory.
+        """
+
+        if not isinstance(run_id, str) or not run_id.strip():
+            raise ValueError(
+                "run_id must be a non-empty string."
+            )
+
+        directory = Path(directory)
+
+        path = (
+            directory
+            / run_id
+            / "reproducibility.json"
+        )
+
+        return self.manager.save_snapshot(
+            snapshot=snapshot,
+            path=path,
+            overwrite=overwrite,
+        )
+
+    def load_snapshot(
+        self,
+        directory: str | Path,
+        run_id: str,
+    ) -> dict[str, Any]:
+        """
+        Load a saved run reproducibility snapshot.
+        """
+
+        if not isinstance(run_id, str) or not run_id.strip():
+            raise ValueError(
+                "run_id must be a non-empty string."
+            )
+
+        path = (
+            Path(directory)
+            / run_id
+            / "reproducibility.json"
+        )
+
+        return self.manager.load_snapshot(path)
+
+    def attach_artifact_integrity(
+        self,
+        metadata: dict[str, Any],
+        artifact_path: str | Path,
+    ) -> dict[str, Any]:
+        """
+        Attach a SHA-256 fingerprint for a saved model artifact.
+        """
+
+        if not isinstance(metadata, dict):
+            raise TypeError(
+                "metadata must be a dictionary."
+            )
+
+        fingerprint = self.manager.artifact_fingerprint(
+            artifact_path
+        )
+
+        updated = dict(metadata)
+
+        updated["artifact"] = {
+            "path": str(artifact_path),
+            "sha256": fingerprint,
+        }
+
+        return updated
+
+    def verify_artifact(
+        self,
+        artifact_path: str | Path,
+        expected_fingerprint: str,
+    ) -> bool:
+        """
+        Verify the integrity of a saved artifact.
+        """
+
+        return self.manager.verify_artifact(
+            artifact_path,
+            expected_fingerprint,
+        )
+
+    def verify_dataset(
+        self,
+        data: pd.DataFrame,
+        snapshot: dict[str, Any],
+    ) -> bool:
+        """
+        Verify that a dataset matches the snapshot fingerprint.
+        """
+
+        if not isinstance(snapshot, dict):
+            raise TypeError(
+                "snapshot must be a dictionary."
+            )
+
+        dataset = snapshot.get("dataset")
+
+        if not isinstance(dataset, dict):
+            raise ValueError(
+                "Snapshot does not contain dataset metadata."
+            )
+
+        expected = dataset.get("fingerprint")
+
+        if not isinstance(expected, str):
+            raise ValueError(
+                "Snapshot does not contain a valid "
+                "dataset fingerprint."
+            )
+
+        return self.manager.verify_dataset(
+            data,
+            expected,
+        )

modelforge/run_manager.py

@@ -0,0 +1,200 @@
+from __future__ import annotations
+
+import time
+import uuid
+from datetime import datetime, timezone
+from typing import Any
+
+
+class RunManager:
+    """
+    Manage the lifecycle and metadata of a ModelForge run.
+
+    A run represents one execution of an AutoML workflow.
+    """
+
+    def __init__(self) -> None:
+        self.run_id: str | None = None
+        self.status: str = "not_started"
+        self.started_at: str | None = None
+        self.finished_at: str | None = None
+        self.duration_seconds: float | None = None
+        self.metadata: dict[str, Any] = {}
+        self._start_time: float | None = None
+
+    def start(
+        self,
+        metadata: dict[str, Any] | None = None,
+    ) -> str:
+        """
+        Start a new run.
+
+        Returns the generated run ID.
+        """
+
+        if self.status == "running":
+            raise RuntimeError(
+                "A run is already in progress."
+            )
+
+        self.run_id = self._create_run_id()
+        self.status = "running"
+        self.started_at = (
+            datetime.now(timezone.utc).isoformat()
+        )
+        self.finished_at = None
+        self.duration_seconds = None
+        self._start_time = time.perf_counter()
+
+        self.metadata = dict(
+            metadata or {}
+        )
+
+        return self.run_id
+
+    def complete(
+        self,
+        metadata: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        """
+        Mark the current run as completed.
+        """
+
+        self._require_running()
+
+        self._finish(
+            status="completed",
+            metadata=metadata,
+        )
+
+        return self.summary()
+
+    def fail(
+        self,
+        error: Exception | str,
+        metadata: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        """
+        Mark the current run as failed.
+        """
+
+        self._require_running()
+
+        error_message = str(error)
+
+        failure_metadata = dict(
+            metadata or {}
+        )
+
+        failure_metadata["error"] = (
+            error_message
+        )
+
+        self._finish(
+            status="failed",
+            metadata=failure_metadata,
+        )
+
+        return self.summary()
+
+    def update(
+        self,
+        metadata: dict[str, Any],
+    ) -> None:
+        """
+        Add or update run metadata.
+        """
+
+        if not isinstance(
+            metadata,
+            dict,
+        ):
+            raise TypeError(
+                "metadata must be a dictionary."
+            )
+
+        if self.status != "running":
+            raise RuntimeError(
+                "Run metadata can only be "
+                "updated while a run is running."
+            )
+
+        self.metadata.update(
+            metadata
+        )
+
+    def summary(self) -> dict[str, Any]:
+        """
+        Return the current run summary.
+        """
+
+        return {
+            "run_id": self.run_id,
+            "status": self.status,
+            "started_at": self.started_at,
+            "finished_at": self.finished_at,
+            "duration_seconds": (
+                self.duration_seconds
+            ),
+            "metadata": dict(
+                self.metadata
+            ),
+        }
+
+    def is_running(self) -> bool:
+        """
+        Return whether a run is currently active.
+        """
+
+        return self.status == "running"
+
+    def reset(self) -> None:
+        """
+        Reset the manager to its initial state.
+        """
+
+        self.run_id = None
+        self.status = "not_started"
+        self.started_at = None
+        self.finished_at = None
+        self.duration_seconds = None
+        self.metadata = {}
+        self._start_time = None
+
+    def _finish(
+        self,
+        status: str,
+        metadata: dict[str, Any] | None,
+    ) -> None:
+        self.status = status
+
+        self.finished_at = (
+            datetime.now(timezone.utc).isoformat()
+        )
+
+        if self._start_time is not None:
+            self.duration_seconds = (
+                time.perf_counter()
+                - self._start_time
+            )
+
+        if metadata:
+            self.metadata.update(
+                metadata
+            )
+
+    def _require_running(self) -> None:
+        if self.status != "running":
+            raise RuntimeError(
+                "No active run is currently running."
+            )
+
+    @staticmethod
+    def _create_run_id() -> str:
+        timestamp = datetime.now(
+            timezone.utc
+        ).strftime("%Y%m%d%H%M%S")
+
+        short_uuid = uuid.uuid4().hex[:8]
+
+        return f"run_{timestamp}_{short_uuid}"

modelforge/target_selector.py

@@ -0,0 +1,108 @@
+import pandas as pd
+
+
+class TargetSelector:
+    """Validate and analyze the target column selected by the user."""
+
+    def select(
+        self,
+        data: pd.DataFrame,
+        target: str,
+        task_type: str | None = None,
+    ) -> dict:
+        """
+        Validate the selected target and determine the ML task type.
+
+        Parameters
+        ----------
+        data : pd.DataFrame
+            Input dataset.
+
+        target : str
+            Name of the target column selected by the user.
+
+        task_type : str | None
+            Optional explicit task type:
+            'regression' or 'classification'.
+
+        Returns
+        -------
+        dict
+            Target information and task type.
+        """
+
+        if not isinstance(data, pd.DataFrame):
+            raise TypeError("data must be a pandas DataFrame.")
+
+        if data.empty:
+            raise ValueError("Dataset is empty.")
+
+        if not target:
+            raise ValueError("Target column must be provided.")
+
+        if target not in data.columns:
+            raise ValueError(
+                f"Target column '{target}' does not exist in the dataset."
+            )
+
+        target_series = data[target]
+
+        if target_series.isna().all():
+            raise ValueError(
+                f"Target column '{target}' contains only missing values."
+            )
+
+        if task_type is not None:
+            task_type = task_type.lower()
+
+            if task_type not in {"regression", "classification"}:
+                raise ValueError(
+                    "task_type must be 'regression' or 'classification'."
+                )
+        else:
+            task_type = self._detect_task_type(target_series)
+
+        return {
+            "target": target,
+            "task_type": task_type,
+            "dtype": str(target_series.dtype),
+            "unique_values": int(target_series.nunique(dropna=True)),
+            "missing_values": int(target_series.isna().sum()),
+            "rows": int(len(target_series)),
+        }
+
+    @staticmethod
+    def _detect_task_type(target: pd.Series) -> str:
+        """Detect the likely ML task from the target column."""
+
+        if pd.api.types.is_bool_dtype(target):
+            return "classification"
+
+        if pd.api.types.is_object_dtype(target):
+            return "classification"
+
+        if isinstance(target.dtype, pd.CategoricalDtype):
+            return "classification"
+
+        if pd.api.types.is_numeric_dtype(target):
+            unique_values = target.nunique(dropna=True)
+            total_values = target.notna().sum()
+
+            if total_values == 0:
+                raise ValueError("Target contains no valid values.")
+
+            unique_ratio = unique_values / total_values
+
+            # A small number of repeated integer values is likely
+            # classification. High-cardinality numeric targets are
+            # treated as regression.
+            if (
+                pd.api.types.is_integer_dtype(target)
+                and unique_values <= 10
+                and unique_ratio <= 0.20
+            ):
+                return "classification"
+
+            return "regression"
+
+        return "classification"