D4CMPP2 0.4.0__py3-none-any.whl

D4CMPP2/src/utils/leaderboard.py

@@ -0,0 +1,212 @@
+"""Read-only experiment comparison for current and legacy model directories."""
+
+import json
+import re
+from pathlib import Path
+
+import pandas as pd
+import yaml
+
+
+DEFAULT_METRIC = "val_rmse"
+_SUMMARY_PATTERNS = {
+    "parameter_count": re.compile(r"#params:\s*(\d+)"),
+    "runtime_seconds": re.compile(r"learning_time:\s*([+-]?(?:\d+(?:\.\d*)?|\.\d+))"),
+}
+
+
+def _model_directories(paths):
+    if isinstance(paths, (str, Path)):
+        paths = [paths]
+    if not paths:
+        raise ValueError("At least one model directory or search root is required.")
+
+    discovered = set()
+    for value in paths:
+        root = Path(value).expanduser()
+        if not root.exists():
+            raise FileNotFoundError(
+                f"Experiment path {str(root)!r} does not exist. "
+                "Provide a model directory or a directory containing model folders."
+            )
+        if root.is_file():
+            raise ValueError(
+                f"Experiment path {str(root)!r} is a file. Provide its model directory instead."
+            )
+        if (root / "config.yaml").is_file():
+            discovered.add(root.resolve())
+            continue
+        for config_path in root.rglob("config.yaml"):
+            discovered.add(config_path.parent.resolve())
+    if not discovered:
+        shown = ", ".join(repr(str(Path(value))) for value in paths)
+        raise ValueError(
+            f"No model directories containing config.yaml were found below {shown}. "
+            "Check the search roots or pass model directories directly."
+        )
+    return sorted(discovered, key=lambda path: str(path).casefold())
+
+
+def _read_yaml(path):
+    try:
+        with path.open("r", encoding="utf-8") as stream:
+            value = yaml.safe_load(stream)
+    except (OSError, yaml.YAMLError) as exc:
+        raise ValueError(
+            f"Could not read model config {str(path)!r}: {exc}. "
+            "Check that config.yaml is valid UTF-8 YAML."
+        ) from exc
+    if not isinstance(value, dict):
+        raise ValueError(f"Model config {str(path)!r} must contain a YAML mapping.")
+    return value
+
+
+def _read_manifests(model_path):
+    manifests = []
+    for path in sorted((model_path / "runs").glob("*/run_manifest.json")):
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError) as exc:
+            raise ValueError(
+                f"Could not read run manifest {str(path)!r}: {exc}. "
+                "Repair or remove the malformed manifest before comparison."
+            ) from exc
+        if not isinstance(data, dict):
+            raise ValueError(f"Run manifest {str(path)!r} must contain a JSON object.")
+        data["_manifest_path"] = str(path)
+        manifests.append(data)
+    return manifests
+
+
+def _read_metrics(model_path):
+    candidates = [model_path / "result" / "metrics.csv", model_path / "metrics.csv"]
+    path = next((candidate for candidate in candidates if candidate.is_file()), None)
+    if path is None:
+        return None, None
+    try:
+        frame = pd.read_csv(path, index_col=0)
+    except (OSError, pd.errors.ParserError, UnicodeError) as exc:
+        raise ValueError(
+            f"Could not read metrics file {str(path)!r}: {exc}. "
+            "Check that metrics.csv is a valid CSV file."
+        ) from exc
+    frame.index = frame.index.map(str)
+    return frame, str(path)
+
+
+def _summary(model_path):
+    path = model_path / "model_summary.txt"
+    if not path.is_file():
+        return {}
+    try:
+        text = path.read_text(encoding="utf-8")
+    except (OSError, UnicodeError):
+        return {}
+    values = {}
+    for key, pattern in _SUMMARY_PATTERNS.items():
+        match = pattern.search(text)
+        if match:
+            values[key] = float(match.group(1)) if key == "runtime_seconds" else int(match.group(1))
+    return values
+
+
+def _targets(config, metrics):
+    configured = config.get("target")
+    if isinstance(configured, str):
+        configured = [configured]
+    targets = [str(value) for value in configured or []]
+    if metrics is not None:
+        targets.extend(str(value) for value in metrics.index)
+    return list(dict.fromkeys(targets)) or [None]
+
+
+def _base_row(model_path, config, manifest, summary):
+    network = manifest.get("network", {}) if manifest else {}
+    manifest_config = manifest.get("config", {}) if manifest else {}
+    effective = dict(config)
+    effective.update(manifest_config if isinstance(manifest_config, dict) else {})
+    return {
+        "model_path": str(model_path),
+        "run_id": manifest.get("run_id") if manifest else "legacy",
+        "status": manifest.get("status") if manifest else "legacy",
+        "mode": manifest.get("mode") if manifest else "legacy",
+        "started_at": manifest.get("started_at") if manifest else None,
+        "ended_at": manifest.get("ended_at") if manifest else None,
+        "duration_seconds": manifest.get("duration_seconds", summary.get("runtime_seconds")) if manifest else summary.get("runtime_seconds"),
+        "network": network.get("id") or effective.get("network_id") or effective.get("network"),
+        "data": effective.get("data") or Path(str(effective.get("DATA_PATH", ""))).stem or None,
+        "best_epoch": manifest.get("best_epoch") if manifest else None,
+        "parameter_count": summary.get("parameter_count"),
+        "batch_size": effective.get("batch_size"),
+        "learning_rate": effective.get("learning_rate", effective.get("lr")),
+        "optimizer": effective.get("optimizer"),
+        "max_epoch": effective.get("max_epoch"),
+        "manifest_path": manifest.get("_manifest_path") if manifest else None,
+    }
+
+
+def collect_experiments(paths):
+    """Return one row per run and target, including legacy model folders."""
+    rows = []
+    for model_path in _model_directories(paths):
+        config = _read_yaml(model_path / "config.yaml")
+        manifests = _read_manifests(model_path)
+        metrics, metrics_path = _read_metrics(model_path)
+        summary = _summary(model_path)
+        latest_completed = None
+        completed = [item for item in manifests if item.get("status") == "completed"]
+        if completed:
+            latest_completed = max(
+                completed,
+                key=lambda item: (item.get("ended_at") or "", item.get("run_id") or ""),
+            )
+        entries = manifests or [None]
+        for manifest in entries:
+            attach_metrics = manifest is None or manifest is latest_completed
+            for target in _targets(config, metrics if attach_metrics else None):
+                row = _base_row(model_path, config, manifest, summary)
+                row["target"] = target
+                row["metric_source"] = metrics_path if attach_metrics else None
+                if attach_metrics and metrics is not None and target in metrics.index:
+                    for name, value in metrics.loc[target].items():
+                        row[str(name)] = value
+                rows.append(row)
+    return pd.DataFrame(rows)
+
+
+def compare_experiments(paths, output_path="leaderboard.csv", metric=DEFAULT_METRIC, target=None, ascending=None):
+    """Collect, rank within each target, save CSV, and return a DataFrame."""
+    frame = collect_experiments(paths)
+    if target is not None:
+        frame = frame[frame["target"] == str(target)].copy()
+        if frame.empty:
+            available = sorted(value for value in collect_experiments(paths)["target"].dropna().unique())
+            raise ValueError(
+                f"Target {target!r} was not found. Available targets: {available}."
+            )
+    if metric not in frame.columns:
+        available = sorted(
+            column for column in frame.columns
+            if column.startswith(("train_", "val_", "test_"))
+        )
+        raise ValueError(
+            f"Metric {metric!r} was not found. Available metric columns: {available}."
+        )
+    if ascending is None:
+        ascending = not metric.endswith(("_r2", "_accuracy"))
+    frame["selected_metric"] = metric
+    frame["selected_metric_value"] = pd.to_numeric(frame[metric], errors="coerce")
+    frame["rank"] = frame.groupby("target", dropna=False)["selected_metric_value"].rank(
+        method="min",
+        ascending=ascending,
+        na_option="bottom",
+    )
+    frame = frame.sort_values(
+        ["target", "rank", "model_path", "run_id"],
+        na_position="last",
+        kind="stable",
+    ).reset_index(drop=True)
+    destination = Path(output_path)
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    frame.to_csv(destination, index=False)
+    return frame

D4CMPP2/src/utils/metrics.py

@@ -0,0 +1,31 @@
+import numpy as np
+import pandas as pd
+
+def get_metrics(targets, prediction_df, ):
+    metrics = {}
+    for target in targets:
+        metrics[target] = {}
+        for set in ['train','val','test']:
+            true = prediction_df[prediction_df['set']==set][f"{target}_true"]
+            pred = prediction_df[prediction_df['set']==set][f"{target}_pred"]
+            metrics[target][set+'_r2'] = r2_score(true,pred)
+            metrics[target][set+'_mae'] = mean_absolute_error(true,pred)
+            metrics[target][set+'_rmse'] = np.sqrt(mean_squared_error(true,pred))
+            metrics[target][set+'_max_error'] = max_error(true,pred)
+    metrics = pd.DataFrame(metrics).T
+    metrics = metrics[[f"{set}_{metric}" for set in ['train','val','test'] for metric in ['r2','mae','rmse','max_error']]]
+    return metrics  
+
+def r2_score(true,pred):
+    ss_res = np.sum((true - pred) ** 2)
+    ss_tot = np.sum((true - np.mean(true)) ** 2)
+    return 1 - (ss_res / ss_tot)
+
+def mean_absolute_error(true,pred):
+    return np.mean(np.abs(true-pred))
+
+def mean_squared_error(true,pred):
+    return np.mean((true-pred)**2)
+
+def max_error(true,pred):
+    return np.max(np.abs(true-pred))

D4CMPP2/src/utils/module_loader.py

@@ -0,0 +1,147 @@
+import hashlib
+import importlib
+import importlib.util
+import os
+from pathlib import Path
+import sys
+from typing import Any, Callable, Mapping, Union
+
+from D4CMPP2.exceptions import ManagerNotFoundError, ModuleLoadError
+
+Config = Mapping[str, Any]
+ManagerFactory = Callable[[str, str], Any]
+
+
+def load_train_manager(config: Config) -> Any:
+    """Construct the configured built-in or external training manager."""
+
+    return _load_manager(
+        "TrainManager_PATH",
+        "train_manager_module",
+        "train_manager_class",
+        load_default_train_manager,
+    )(config)
+
+
+def load_data_manager(config: Config) -> Any:
+    """Construct the configured built-in or external data manager."""
+
+    return _load_manager(
+        "DataManager_PATH",
+        "data_manager_module",
+        "data_manager_class",
+        load_default_data_manager,
+    )(config)
+
+
+def load_network_manager(config: Config) -> Any:
+    """Construct the configured built-in or external network manager."""
+
+    return _load_manager(
+        "NetworkManager_PATH",
+        "network_manager_module",
+        "network_manager_class",
+        load_default_network_manager,
+    )(config)
+
+
+def _load_manager(
+    path_key: str,
+    module_key: str,
+    class_key: str,
+    default_manager: ManagerFactory,
+) -> Callable[[Config], Any]:
+    def load_manager(config: Config) -> Any:
+        module_name = config.get(module_key)
+        class_name = config.get(class_key)
+        if not isinstance(module_name, str) or not isinstance(class_name, str):
+            raise ManagerNotFoundError(
+                f"Manager config requires string keys {module_key!r} and "
+                f"{class_key!r}; got module={module_name!r}, class={class_name!r}."
+            )
+        if path_key in config:
+            path = config[path_key]
+            module_path = os.path.join(path, module_name + ".py")
+            if not os.path.isfile(module_path):
+                raise ManagerNotFoundError(
+                    f"Custom manager module {module_path!r} was not found. "
+                    f"Check {path_key}, {module_key}, and {class_key}."
+                )
+            return load_module(path, module_name, class_name)
+        try:
+            return default_manager(module_name, class_name)
+        except (AttributeError, KeyError) as exc:
+            raise ManagerNotFoundError(
+                f"Built-in manager {module_name}.{class_name} was not found. "
+                f"Check {module_key} and {class_key} in the selected registry."
+            ) from exc
+
+    return load_manager
+
+
+def load_default_train_manager(module_name: str, class_name: str) -> Any:
+    """Resolve a training-manager class from the installed package."""
+
+    return _load_default_manager("TrainManager", module_name, class_name)
+
+
+def load_default_data_manager(module_name: str, class_name: str) -> Any:
+    """Resolve a data-manager class from the installed package."""
+
+    return _load_default_manager("DataManager", module_name, class_name)
+
+
+def load_default_network_manager(module_name: str, class_name: str) -> Any:
+    """Resolve a network-manager class from the installed package."""
+
+    return _load_default_manager("NetworkManager", module_name, class_name)
+
+
+def _load_default_manager(family: str, module_name: str, class_name: str) -> Any:
+    module = importlib.import_module(f"D4CMPP2.src.{family}.{module_name}")
+    return getattr(module, class_name)
+
+
+def load_module(
+    path: Union[str, os.PathLike[str]],
+    module: str,
+    class_name: str,
+) -> Any:
+    """Load one custom manager class without permanently changing ``sys.path``."""
+
+    directory = str(Path(path).resolve())
+    module_path = os.path.join(directory, module + ".py")
+    identity = hashlib.sha256(
+        os.path.normcase(module_path).encode("utf-8")
+    ).hexdigest()[:16]
+    internal_name = f"_d4cmpp2_custom_{module}_{identity}"
+    inserted_path = directory not in sys.path
+    try:
+        if inserted_path:
+            sys.path.insert(0, directory)
+        spec2 = importlib.util.spec_from_file_location(internal_name, module_path)
+        if spec2 is None or spec2.loader is None:
+            raise ImportError("Python could not create a module specification.")
+        module2 = importlib.util.module_from_spec(spec2)
+        sys.modules[internal_name] = module2
+        spec2.loader.exec_module(module2)
+    except (ImportError, OSError, SyntaxError) as exc:
+        sys.modules.pop(internal_name, None)
+        raise ModuleLoadError(
+            f"Custom module {module_path!r} could not be loaded: {exc}"
+        ) from exc
+    except BaseException:
+        sys.modules.pop(internal_name, None)
+        raise
+    finally:
+        if inserted_path:
+            try:
+                sys.path.remove(directory)
+            except ValueError:
+                pass
+    try:
+        return getattr(module2, class_name)
+    except AttributeError as exc:
+        raise ManagerNotFoundError(
+            f"Custom module {module_path!r} does not define class {class_name!r}."
+        ) from exc

D4CMPP2/src/utils/output.py

@@ -0,0 +1,80 @@
+"""Optional output/logging adapter that preserves current defaults."""
+
+import logging
+import sys
+import warnings
+from typing import Any, Mapping, Optional
+
+LOGGER_NAME = "D4CMPP2"
+logger = logging.getLogger(LOGGER_NAME)
+if not any(isinstance(handler, logging.NullHandler) for handler in logger.handlers):
+    logger.addHandler(logging.NullHandler())
+
+
+class OutputAdapter:
+    """Route opt-in diagnostics without changing default stdout/warnings."""
+
+    def __init__(
+        self,
+        *,
+        verbose: bool = True,
+        stream=None,
+        error_stream=None,
+        diagnostic_logger: Optional[logging.Logger] = None,
+    ) -> None:
+        if not isinstance(verbose, bool):
+            raise TypeError(
+                f"verbose must be a bool, got {type(verbose).__name__}: {verbose!r}."
+            )
+        self.verbose = verbose
+        # Keep None so redirect_stdout() applied after construction still works.
+        self.stream = stream
+        self.error_stream = error_stream if error_stream is not None else stream
+        self.logger = diagnostic_logger if diagnostic_logger is not None else logger
+
+    def info(self, *values: Any, **kwargs: Any) -> None:
+        """Match the existing print-based user output."""
+
+        if self.verbose:
+            print(*values, file=self.stream, **kwargs)
+
+    def error(self, *values: Any, **kwargs: Any) -> None:
+        """Write essential failure information regardless of verbosity."""
+
+        print(*values, file=self.error_stream, **kwargs)
+
+    def always(self, *values: Any, **kwargs: Any) -> None:
+        """Write an essential completion summary regardless of verbosity."""
+
+        print(*values, file=self.stream, **kwargs)
+
+    def progress(self, iterable, **kwargs):
+        """Wrap an iterable in tqdm while respecting the verbosity policy."""
+
+        from tqdm import tqdm
+
+        kwargs.setdefault("disable", not self.verbose)
+        return tqdm(iterable, **kwargs)
+
+    def warning(
+        self,
+        message: str,
+        category=UserWarning,
+        *,
+        stacklevel: int = 2,
+    ) -> None:
+        """Keep actionable fallback/deprecation messages visible."""
+
+        warnings.warn(message, category, stacklevel=stacklevel)
+
+    def debug(self, message: str, *args: Any, **kwargs: Any) -> None:
+        """Emit opt-in developer diagnostics through the package logger."""
+
+        self.logger.debug(message, *args, **kwargs)
+
+
+def get_output(config: Optional[Mapping[str, Any]] = None) -> OutputAdapter:
+    """Build the standard output adapter for a runtime configuration."""
+
+    verbose = True if config is None else config.get("verbose", True)
+    return OutputAdapter(verbose=verbose)

D4CMPP2/src/utils/reproducibility.py

@@ -0,0 +1,70 @@
+"""Opt-in training seed and deterministic-backend configuration."""
+
+import os
+import random
+
+import numpy as np
+import torch
+
+
+def capture_backend_determinism():
+    cudnn = getattr(torch.backends, "cudnn", None)
+    return {
+        "deterministic_algorithms": torch.are_deterministic_algorithms_enabled(),
+        "cudnn_deterministic": getattr(cudnn, "deterministic", None),
+        "cudnn_benchmark": getattr(cudnn, "benchmark", None),
+    }
+
+
+def restore_backend_determinism(state):
+    torch.use_deterministic_algorithms(state["deterministic_algorithms"])
+    cudnn = getattr(torch.backends, "cudnn", None)
+    if cudnn is not None:
+        if state["cudnn_deterministic"] is not None:
+            cudnn.deterministic = state["cudnn_deterministic"]
+        if state["cudnn_benchmark"] is not None:
+            cudnn.benchmark = state["cudnn_benchmark"]
+
+
+def configure_reproducibility(config, resume=False):
+    """Apply requested policy and return effective state for reporting."""
+    seed = config.get("random_seed")
+    deterministic = config.get("deterministic_algorithms", False)
+
+    if deterministic:
+        os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8")
+        torch.use_deterministic_algorithms(True)
+        if hasattr(torch.backends, "cudnn"):
+            torch.backends.cudnn.deterministic = True
+            torch.backends.cudnn.benchmark = False
+
+    if seed is not None and not resume:
+        random.seed(seed)
+        np.random.seed(seed)
+        torch.manual_seed(seed)
+        if torch.cuda.is_available():
+            torch.cuda.manual_seed_all(seed)
+
+    return effective_reproducibility_state(config, resume=resume)
+
+
+def effective_reproducibility_state(config, resume=False):
+    cudnn = getattr(torch.backends, "cudnn", None)
+    return {
+        "random_seed": config.get("random_seed"),
+        "split_random_seed": config.get("split_random_seed", 42),
+        "deterministic_algorithms_requested": config.get("deterministic_algorithms", False),
+        "deterministic_algorithms_enabled": torch.are_deterministic_algorithms_enabled(),
+        "cudnn_deterministic": getattr(cudnn, "deterministic", None),
+        "cudnn_benchmark": getattr(cudnn, "benchmark", None),
+        "python_hash_seed": os.environ.get("PYTHONHASHSEED"),
+        "rng_source": "checkpoint" if resume else "configured_seed" if config.get("random_seed") is not None else "ambient",
+    }
+
+
+def seed_data_loader_worker(worker_id):
+    """Seed Python and NumPy from the worker seed assigned by PyTorch."""
+    del worker_id
+    worker_seed = torch.initial_seed() % (2 ** 32)
+    random.seed(worker_seed)
+    np.random.seed(worker_seed)