ins-pricing 0.1.6__py3-none-any.whl

ins_pricing v2/governance/release.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass, field
+from datetime import datetime
+from pathlib import Path
+from typing import List, Optional
+
+from .audit import AuditLogger
+from .registry import ModelRegistry
+
+
+@dataclass
+class ModelRef:
+    name: str
+    version: str
+    activated_at: str
+    actor: Optional[str] = None
+    note: Optional[str] = None
+
+
+@dataclass
+class DeploymentState:
+    env: str
+    active: Optional[ModelRef] = None
+    history: List[ModelRef] = field(default_factory=list)
+    updated_at: Optional[str] = None
+
+
+class ReleaseManager:
+    """Environment release manager with rollback support."""
+
+    def __init__(
+        self,
+        state_dir: str | Path,
+        *,
+        registry: Optional[ModelRegistry] = None,
+        audit_logger: Optional[AuditLogger] = None,
+    ):
+        self.state_dir = Path(state_dir)
+        self.state_dir.mkdir(parents=True, exist_ok=True)
+        self.registry = registry
+        self.audit_logger = audit_logger
+
+    def _state_path(self, env: str) -> Path:
+        return self.state_dir / f"{env}.json"
+
+    def _load(self, env: str) -> DeploymentState:
+        path = self._state_path(env)
+        if not path.exists():
+            return DeploymentState(env=env)
+        with path.open("r", encoding="utf-8") as fh:
+            payload = json.load(fh)
+        active = payload.get("active")
+        history = payload.get("history", [])
+        return DeploymentState(
+            env=payload.get("env", env),
+            active=ModelRef(**active) if active else None,
+            history=[ModelRef(**item) for item in history],
+            updated_at=payload.get("updated_at"),
+        )
+
+    def _save(self, state: DeploymentState) -> None:
+        payload = {
+            "env": state.env,
+            "active": asdict(state.active) if state.active else None,
+            "history": [asdict(item) for item in state.history],
+            "updated_at": state.updated_at,
+        }
+        path = self._state_path(state.env)
+        with path.open("w", encoding="utf-8") as fh:
+            json.dump(payload, fh, indent=2, ensure_ascii=True)
+
+    def get_active(self, env: str) -> Optional[ModelRef]:
+        state = self._load(env)
+        return state.active
+
+    def list_history(self, env: str) -> List[ModelRef]:
+        return self._load(env).history
+
+    def deploy(
+        self,
+        env: str,
+        name: str,
+        version: str,
+        *,
+        actor: Optional[str] = None,
+        note: Optional[str] = None,
+        update_registry_status: bool = True,
+        registry_status: str = "production",
+    ) -> DeploymentState:
+        state = self._load(env)
+        if state.active and state.active.name == name and state.active.version == version:
+            return state
+
+        if state.active is not None:
+            state.history.append(state.active)
+
+        now = datetime.utcnow().isoformat()
+        state.active = ModelRef(
+            name=name,
+            version=version,
+            activated_at=now,
+            actor=actor,
+            note=note,
+        )
+        state.updated_at = now
+        self._save(state)
+
+        if self.registry and update_registry_status:
+            self.registry.promote(name, version, new_status=registry_status)
+
+        if self.audit_logger:
+            self.audit_logger.log(
+                "deploy",
+                actor or "unknown",
+                metadata={"env": env, "name": name, "version": version},
+                note=note,
+            )
+
+        return state
+
+    def rollback(
+        self,
+        env: str,
+        *,
+        actor: Optional[str] = None,
+        note: Optional[str] = None,
+        update_registry_status: bool = False,
+        registry_status: str = "production",
+    ) -> DeploymentState:
+        state = self._load(env)
+        if not state.history:
+            raise ValueError("No history available to rollback.")
+
+        previous = state.history.pop()
+        now = datetime.utcnow().isoformat()
+        state.active = ModelRef(
+            name=previous.name,
+            version=previous.version,
+            activated_at=now,
+            actor=actor or previous.actor,
+            note=note or previous.note,
+        )
+        state.updated_at = now
+        self._save(state)
+
+        if self.registry and update_registry_status:
+            self.registry.promote(previous.name, previous.version, new_status=registry_status)
+
+        if self.audit_logger:
+            self.audit_logger.log(
+                "rollback",
+                actor or "unknown",
+                metadata={"env": env, "name": previous.name, "version": previous.version},
+                note=note,
+            )
+
+        return state

ins_pricing v2/modelling/Explain_Run.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+from typing import Optional
+
+try:
+    from .notebook_utils import run_from_config  # type: ignore
+except Exception:  # pragma: no cover
+    from notebook_utils import run_from_config  # type: ignore
+
+
+def run(config_json: str | Path) -> None:
+    """Run explain by config.json (runner.mode=explain)."""
+    run_from_config(config_json)
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="Explain_Run: run explain by config.json (runner.mode=explain)."
+    )
+    parser.add_argument(
+        "--config-json",
+        required=True,
+        help="Path to config.json (relative paths are resolved from ins_pricing/modelling/ when possible).",
+    )
+    return parser
+
+
+def main(argv: Optional[list[str]] = None) -> None:
+    args = _build_parser().parse_args(argv)
+    run(args.config_json)
+
+
+if __name__ == "__main__":
+    main()

ins_pricing v2/modelling/Pricing_Run.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+from typing import Optional
+
+try:
+    from .notebook_utils import run_from_config  # type: ignore
+except Exception:  # pragma: no cover
+    from notebook_utils import run_from_config  # type: ignore
+
+
+def run(config_json: str | Path) -> None:
+    """Unified entry point: run entry/incremental/watchdog/DDP based on config.json runner."""
+    run_from_config(config_json)
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="Pricing_Run: run BayesOpt by config.json (entry/incremental/watchdog/DDP)."
+    )
+    parser.add_argument(
+        "--config-json",
+        required=True,
+        help="Path to config.json (relative paths are resolved from ins_pricing/modelling/ when possible).",
+    )
+    return parser
+
+
+def main(argv: Optional[list[str]] = None) -> None:
+    args = _build_parser().parse_args(argv)
+    run(args.config_json)
+
+
+if __name__ == "__main__":
+    main()

ins_pricing v2/modelling/__init__.py

@@ -0,0 +1,151 @@
+"""Modelling subpackage for ins_pricing."""
+
+from __future__ import annotations
+import types
+import sys
+from importlib import import_module
+from pathlib import Path
+
+# Exports
+from .config import BayesOptConfig
+from .config_preprocess import (
+    DatasetPreprocessor,
+    OutputManager,
+    VersionManager,
+)
+from .core import BayesOptModel
+from .models import (
+    FeatureTokenizer,
+    FTTransformerCore,
+    FTTransformerSklearn,
+    GraphNeuralNetSklearn,
+    MaskedTabularDataset,
+    ResBlock,
+    ResNetSequential,
+    ResNetSklearn,
+    ScaledTransformerEncoderLayer,
+    SimpleGraphLayer,
+    SimpleGNN,
+    TabularDataset,
+)
+from .trainers import (
+    FTTrainer,
+    GLMTrainer,
+    GNNTrainer,
+    ResNetTrainer,
+    TrainerBase,
+    XGBTrainer,
+    _xgb_cuda_available,
+)
+from .utils import (
+    EPS,
+    DistributedUtils,
+    IOUtils,
+    PlotUtils,
+    TorchTrainerMixin,
+    TrainingUtils,
+    compute_batch_size,
+    csv_to_dict,
+    ensure_parent_dir,
+    free_cuda,
+    infer_factor_and_cate_list,
+    plot_dlift_list,
+    plot_lift_list,
+    set_global_seed,
+    split_data,
+    tweedie_loss,
+)
+try:
+    import torch
+except ImportError:
+    torch = None
+
+# Lazy submodules
+_LAZY_SUBMODULES = {
+    "plotting": "ins_pricing.modelling.plotting",
+    "explain": "ins_pricing.modelling.explain",
+    "cli_common": "ins_pricing.modelling.cli_common",
+    "notebook_utils": "ins_pricing.modelling.notebook_utils",
+}
+
+_PACKAGE_PATHS = {
+    "plotting": Path(__file__).resolve().parent / "plotting",
+    "explain": Path(__file__).resolve().parent / "explain",
+}
+
+__all__ = [
+    "BayesOptConfig",
+    "DatasetPreprocessor",
+    "OutputManager",
+    "VersionManager",
+    "BayesOptModel",
+    "FeatureTokenizer",
+    "FTTransformerCore",
+    "FTTransformerSklearn",
+    "GraphNeuralNetSklearn",
+    "MaskedTabularDataset",
+    "ResBlock",
+    "ResNetSequential",
+    "ResNetSklearn",
+    "ScaledTransformerEncoderLayer",
+    "SimpleGraphLayer",
+    "SimpleGNN",
+    "TabularDataset",
+    "FTTrainer",
+    "GLMTrainer",
+    "GNNTrainer",
+    "ResNetTrainer",
+    "TrainerBase",
+    "XGBTrainer",
+    "_xgb_cuda_available",
+    "EPS",
+    "DistributedUtils",
+    "IOUtils",
+    "PlotUtils",
+    "TorchTrainerMixin",
+    "TrainingUtils",
+    "compute_batch_size",
+    "csv_to_dict",
+    "ensure_parent_dir",
+    "free_cuda",
+    "infer_factor_and_cate_list",
+    "plot_dlift_list",
+    "plot_lift_list",
+    "set_global_seed",
+    "split_data",
+    "tweedie_loss",
+    "torch",
+] + sorted(list(_LAZY_SUBMODULES.keys()))
+
+def _lazy_module(name: str, target: str, package_path: Path | None = None) -> types.ModuleType:
+    proxy = types.ModuleType(name)
+    if package_path is not None:
+        proxy.__path__ = [str(package_path)]
+
+    def _load():
+        module = import_module(target)
+        sys.modules[name] = module
+        return module
+
+    def __getattr__(attr: str):
+        module = _load()
+        return getattr(module, attr)
+
+    def __dir__() -> list[str]:
+        module = _load()
+        return sorted(set(dir(module)))
+
+    proxy.__getattr__ = __getattr__  # type: ignore[attr-defined]
+    proxy.__dir__ = __dir__  # type: ignore[attr-defined]
+    return proxy
+
+def _install_proxy(alias: str, target: str) -> None:
+    module_name = f"{__name__}.{alias}"
+    if module_name in sys.modules:
+        return
+    proxy = _lazy_module(module_name, target, _PACKAGE_PATHS.get(alias))
+    sys.modules[module_name] = proxy
+    globals()[alias] = proxy
+
+for _alias, _target in _LAZY_SUBMODULES.items():
+    _install_proxy(_alias, _target)

ins_pricing v2/modelling/cli_common.py

@@ -0,0 +1,141 @@
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
+try:
+    from .config import BayesOptConfig
+except ImportError:
+    from ins_pricing.modelling.config import BayesOptConfig
+
+
+
+PLOT_MODEL_LABELS: Dict[str, Tuple[str, str]] = {
+    "glm": ("GLM", "pred_glm"),
+    "xgb": ("Xgboost", "pred_xgb"),
+    "resn": ("ResNet", "pred_resn"),
+    "ft": ("FTTransformer", "pred_ft"),
+    "gnn": ("GNN", "pred_gnn"),
+}
+
+PYTORCH_TRAINERS = {"resn", "ft", "gnn"}
+
+
+def dedupe_preserve_order(items: Iterable[str]) -> List[str]:
+    seen = set()
+    unique: List[str] = []
+    for item in items:
+        if item not in seen:
+            unique.append(item)
+            seen.add(item)
+    return unique
+
+
+def build_model_names(prefixes: Sequence[str], suffixes: Sequence[str]) -> List[str]:
+    names: List[str] = []
+    for suffix in suffixes:
+        names.extend(f"{prefix}_{suffix}" for prefix in prefixes)
+    return names
+
+
+def parse_model_pairs(raw_pairs: List) -> List[Tuple[str, str]]:
+    pairs: List[Tuple[str, str]] = []
+    for pair in raw_pairs:
+        if isinstance(pair, (list, tuple)) and len(pair) == 2:
+            pairs.append((str(pair[0]), str(pair[1])))
+        elif isinstance(pair, str):
+            parts = [p.strip() for p in pair.split(",") if p.strip()]
+            if len(parts) == 2:
+                pairs.append((parts[0], parts[1]))
+    return pairs
+
+
+def resolve_path(value: Optional[str], base_dir: Path) -> Optional[Path]:
+    if value is None:
+        return None
+    if not isinstance(value, str) or not value.strip():
+        return None
+    p = Path(value)
+    if p.is_absolute():
+        return p
+    return (base_dir / p).resolve()
+
+
+def resolve_config_path(raw: str, script_dir: Path) -> Path:
+    candidate = Path(raw)
+    if candidate.exists():
+        return candidate.resolve()
+    candidate2 = (script_dir / raw)
+    if candidate2.exists():
+        return candidate2.resolve()
+    raise FileNotFoundError(
+        f"Config file not found: {raw}. Tried: {Path(raw).resolve()} and {candidate2.resolve()}"
+    )
+
+
+def load_config_json(path: Path, required_keys: Sequence[str]) -> BayesOptConfig:
+    cfg_dict = json.loads(path.read_text(encoding="utf-8"))
+    missing = [key for key in required_keys if key not in cfg_dict]
+    if missing:
+        raise ValueError(f"Missing required keys in {path}: {missing}")
+    return BayesOptConfig.from_legacy_dict(cfg_dict)
+
+
+def set_env(env_overrides: Dict[str, Any]) -> None:
+    """Apply environment variables from config.json.
+
+    Notes (DDP/Optuna hang debugging):
+    - You can add these keys into config.json's `env` to debug distributed hangs:
+      - `TORCH_DISTRIBUTED_DEBUG=DETAIL`
+      - `NCCL_DEBUG=INFO`
+      - `BAYESOPT_DDP_BARRIER_DEBUG=1`
+      - `BAYESOPT_DDP_BARRIER_TIMEOUT=300`
+      - `BAYESOPT_CUDA_SYNC=1` (optional; can slow down)
+      - `BAYESOPT_CUDA_IPC_COLLECT=1` (optional; can slow down)
+    - This function uses `os.environ.setdefault`, so a value already set in the
+      shell will take precedence over config.json.
+    """
+    for key, value in (env_overrides or {}).items():
+        os.environ.setdefault(str(key), str(value))
+
+
+def _looks_like_url(value: str) -> bool:
+    value = str(value)
+    return "://" in value
+
+
+def normalize_config_paths(cfg: Dict[str, Any], config_path: Path) -> Dict[str, Any]:
+    """Resolve relative paths against the config.json directory.
+
+    Fields handled:
+    - data_dir / output_dir / optuna_storage / gnn_graph_cache
+    - best_params_files (dict: model_key -> path)
+    """
+    base_dir = config_path.parent
+    out = dict(cfg)
+
+    for key in ("data_dir", "output_dir", "gnn_graph_cache"):
+        if key in out and isinstance(out.get(key), str):
+            resolved = resolve_path(out.get(key), base_dir)
+            if resolved is not None:
+                out[key] = str(resolved)
+
+    storage = out.get("optuna_storage")
+    if isinstance(storage, str) and storage.strip():
+        if not _looks_like_url(storage):
+            resolved = resolve_path(storage, base_dir)
+            if resolved is not None:
+                out["optuna_storage"] = str(resolved)
+
+    best_files = out.get("best_params_files")
+    if isinstance(best_files, dict):
+        resolved_map: Dict[str, str] = {}
+        for mk, path_str in best_files.items():
+            if not isinstance(path_str, str):
+                continue
+            resolved = resolve_path(path_str, base_dir)
+            resolved_map[str(mk)] = str(resolved) if resolved is not None else str(path_str)
+        out["best_params_files"] = resolved_map
+
+    return out