ins-pricing 0.2.8__py3-none-any.whl → 0.3.0__py3-none-any.whl

ins_pricing/modelling/core/bayesopt/utils/__init__.py

@@ -0,0 +1,86 @@
+"""Backward compatibility re-exports from refactored utils modules.
+
+This module ensures all existing imports continue to work:
+    from ins_pricing.modelling.core.bayesopt.utils import EPS, IOUtils, ...
+
+The utils.py file has been split into focused modules for better maintainability:
+- constants.py: EPS, set_global_seed, etc.
+- io_utils.py: IOUtils for file I/O
+- distributed_utils.py: DistributedUtils, TrainingUtils for DDP
+- torch_trainer_mixin.py: TorchTrainerMixin for PyTorch training
+- metrics_and_devices.py: Metrics, GPU/device management, CV strategies, plotting
+"""
+
+from __future__ import annotations
+
+# Constants and simple utilities
+from .constants import (
+    EPS,
+    set_global_seed,
+    ensure_parent_dir,
+    compute_batch_size,
+    tweedie_loss,
+    infer_factor_and_cate_list,
+)
+
+# I/O utilities
+from .io_utils import (
+    IOUtils,
+    csv_to_dict,
+)
+
+# Distributed training
+from .distributed_utils import (
+    DistributedUtils,
+    TrainingUtils,
+    free_cuda,
+)
+
+# PyTorch training mixin
+from .torch_trainer_mixin import (
+    TorchTrainerMixin,
+)
+
+# Metrics, devices, CV, and plotting
+from .metrics_and_devices import (
+    get_logger,
+    MetricFactory,
+    GPUMemoryManager,
+    DeviceManager,
+    CVStrategyResolver,
+    PlotUtils,
+    split_data,
+    plot_lift_list,
+    plot_dlift_list,
+    _OrderedSplitter,
+)
+
+__all__ = [
+    # Constants
+    'EPS',
+    'set_global_seed',
+    'ensure_parent_dir',
+    'compute_batch_size',
+    'tweedie_loss',
+    'infer_factor_and_cate_list',
+    # I/O
+    'IOUtils',
+    'csv_to_dict',
+    # Distributed
+    'DistributedUtils',
+    'TrainingUtils',
+    'free_cuda',
+    # PyTorch
+    'TorchTrainerMixin',
+    # Utilities
+    'get_logger',
+    'MetricFactory',
+    'GPUMemoryManager',
+    'DeviceManager',
+    'CVStrategyResolver',
+    'PlotUtils',
+    'split_data',
+    'plot_lift_list',
+    'plot_dlift_list',
+    '_OrderedSplitter',
+]

ins_pricing/modelling/core/bayesopt/utils/constants.py

@@ -0,0 +1,183 @@
+"""Core constants and simple utility functions.
+
+This module contains:
+- EPS constant for numerical stability
+- set_global_seed() for reproducibility
+- ensure_parent_dir() for file operations
+- compute_batch_size() for adaptive batching
+- tweedie_loss() for regression loss
+- infer_factor_and_cate_list() for automatic feature detection
+"""
+
+from __future__ import annotations
+
+import random
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+import numpy as np
+import pandas as pd
+import torch
+
+# Constants
+# =============================================================================
+EPS = 1e-8
+"""Small epsilon value for numerical stability."""
+
+
+# Simple utility functions
+# =============================================================================
+
+def set_global_seed(seed: int) -> None:
+    """Set random seed for reproducibility across all libraries.
+
+    Args:
+        seed: Random seed value
+    """
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    if torch.cuda.is_available():
+        torch.cuda.manual_seed_all(seed)
+
+
+def ensure_parent_dir(file_path: str) -> None:
+    """Create parent directories when missing.
+
+    Args:
+        file_path: Path to file whose parent directory should be created
+    """
+    directory = Path(file_path).parent
+    if directory and not directory.exists():
+        directory.mkdir(parents=True, exist_ok=True)
+
+
+def compute_batch_size(data_size: int, learning_rate: float,
+                      batch_num: int, minimum: int) -> int:
+    """Compute adaptive batch size based on data size and learning rate.
+
+    Args:
+        data_size: Total number of samples
+        learning_rate: Learning rate value
+        batch_num: Target number of batches
+        minimum: Minimum batch size
+
+    Returns:
+        Computed batch size
+    """
+    estimated = int((learning_rate / 1e-4) ** 0.5 *
+                   (data_size / max(batch_num, 1)))
+    return max(1, min(data_size, max(minimum, estimated)))
+
+
+def tweedie_loss(pred, target, p=1.5, eps=1e-6, max_clip=1e6):
+    """Compute Tweedie deviance loss for PyTorch.
+
+    Reference: https://scikit-learn.org/stable/modules/model_evaluation.html
+
+    Args:
+        pred: Predicted values (tensor)
+        target: True values (tensor)
+        p: Tweedie power parameter (1.0-2.0)
+        eps: Small epsilon for numerical stability
+        max_clip: Maximum value for clipping
+
+    Returns:
+        Tweedie negative log-likelihood (tensor)
+    """
+    # Clamp predictions to positive values for stability
+    pred_clamped = torch.clamp(pred, min=eps)
+
+    if p == 1:
+        # Poisson
+        term1 = target * torch.log(target / pred_clamped + eps)
+        term2 = -target + pred_clamped
+        term3 = 0
+    elif p == 0:
+        # Gaussian
+        term1 = 0.5 * torch.pow(target - pred_clamped, 2)
+        term2 = 0
+        term3 = 0
+    elif p == 2:
+        # Gamma
+        term1 = torch.log(pred_clamped / target + eps)
+        term2 = -target / pred_clamped + 1
+        term3 = 0
+    else:
+        # General Tweedie
+        term1 = torch.pow(target, 2 - p) / ((1 - p) * (2 - p))
+        term2 = target * torch.pow(pred_clamped, 1 - p) / (1 - p)
+        term3 = torch.pow(pred_clamped, 2 - p) / (2 - p)
+
+    return torch.nan_to_num(
+        2 * (term1 - term2 + term3),
+        nan=eps,
+        posinf=max_clip,
+        neginf=-max_clip
+    )
+
+
+def infer_factor_and_cate_list(
+    train_df: pd.DataFrame,
+    test_df: pd.DataFrame,
+    resp_nme: str,
+    weight_nme: str,
+    binary_resp_nme: Optional[str] = None,
+    factor_nmes: Optional[List[str]] = None,
+    cate_list: Optional[List[str]] = None,
+    infer_categorical_max_unique: int = 50,
+    infer_categorical_max_ratio: float = 0.05
+) -> Tuple[List[str], List[str]]:
+    """Infer factor_nmes/cate_list when feature names are not provided.
+
+    Rules:
+      - factor_nmes: start from shared train/test columns, exclude target/weight/(optional binary target).
+      - cate_list: object/category/bool plus low-cardinality integer columns.
+      - Always intersect with shared train/test columns to avoid mismatches.
+
+    Args:
+        train_df: Training DataFrame
+        test_df: Test DataFrame
+        resp_nme: Response/target column name
+        weight_nme: Sample weight column name
+        binary_resp_nme: Optional binary response column name
+        factor_nmes: Optional list of feature column names
+        cate_list: Optional list of categorical feature names
+        infer_categorical_max_unique: Max unique values for categorical inference
+        infer_categorical_max_ratio: Max ratio of unique/total for categorical inference
+
+    Returns:
+        Tuple of (factor_nmes, cate_list)
+    """
+    excluded = {resp_nme, weight_nme}
+    if binary_resp_nme:
+        excluded.add(binary_resp_nme)
+
+    common_cols = [c for c in train_df.columns if c in test_df.columns]
+    if factor_nmes is None:
+        factors = [c for c in common_cols if c not in excluded]
+    else:
+        factors = [
+            c for c in factor_nmes if c in common_cols and c not in excluded
+        ]
+
+    if cate_list is not None:
+        cats = [c for c in cate_list if c in factors]
+        return factors, cats
+
+    n_rows = max(1, len(train_df))
+    cats: List[str] = []
+    for col in factors:
+        s = train_df[col]
+        if (pd.api.types.is_bool_dtype(s) or
+            pd.api.types.is_object_dtype(s) or
+            isinstance(s.dtype, pd.CategoricalDtype)):
+            cats.append(col)
+            continue
+        if pd.api.types.is_integer_dtype(s):
+            nunique = int(s.nunique(dropna=True))
+            if (nunique <= infer_categorical_max_unique or
+                (nunique / n_rows) <= infer_categorical_max_ratio):
+                cats.append(col)
+
+    return factors, cats

ins_pricing/modelling/core/bayesopt/utils/distributed_utils.py

@@ -0,0 +1,186 @@
+"""Distributed training utilities for PyTorch DDP.
+
+This module contains:
+- DistributedUtils for DDP setup and process coordination
+- TrainingUtils for CUDA memory management
+- free_cuda() for legacy compatibility
+"""
+
+from __future__ import annotations
+
+import gc
+import os
+from datetime import timedelta
+from typing import Optional
+
+import torch
+import torch.distributed as dist
+
+
+def _select_ddp_backend() -> str:
+    """Select the appropriate DDP backend based on system capabilities.
+
+    Returns:
+        "nccl" if CUDA is available and NCCL is supported (non-Windows),
+        otherwise "gloo"
+    """
+    if not torch.cuda.is_available():
+        return "gloo"
+
+    if os.name == "nt":  # Windows doesn't support NCCL
+        return "gloo"
+
+    try:
+        nccl_available = getattr(dist, "is_nccl_available", lambda: False)()
+        return "nccl" if nccl_available else "gloo"
+    except Exception:
+        return "gloo"
+
+
+def _get_ddp_timeout() -> timedelta:
+    """Get the DDP timeout from environment variable.
+
+    Returns:
+        timedelta for DDP timeout (default: 1800 seconds)
+    """
+    timeout_seconds = int(os.environ.get("BAYESOPT_DDP_TIMEOUT_SECONDS", "1800"))
+    return timedelta(seconds=max(1, timeout_seconds))
+
+
+def _cache_ddp_state(local_rank: int, rank: int, world_size: int) -> tuple:
+    """Cache and return DDP state tuple."""
+    state = (True, local_rank, rank, world_size)
+    DistributedUtils._cached_state = state
+    return state
+
+
+class DistributedUtils:
+    """Utilities for distributed data parallel training.
+
+    This class provides methods for:
+    - Initializing DDP process groups
+    - Checking process rank and world size
+    - Cleanup after distributed training
+    """
+
+    _cached_state: Optional[tuple] = None
+
+    @staticmethod
+    def setup_ddp():
+        """Initialize the DDP process group for distributed training.
+
+        Returns:
+            Tuple of (success, local_rank, rank, world_size)
+        """
+        # Return cached state if already initialized
+        if dist.is_initialized():
+            if DistributedUtils._cached_state is None:
+                DistributedUtils._cached_state = _cache_ddp_state(
+                    int(os.environ.get("LOCAL_RANK", 0)),
+                    dist.get_rank(),
+                    dist.get_world_size(),
+                )
+            return DistributedUtils._cached_state
+
+        # Check for required environment variables
+        if 'RANK' not in os.environ or 'WORLD_SIZE' not in os.environ:
+            print(
+                f">>> DDP Setup Failed: RANK or WORLD_SIZE not found in env. "
+                f"Keys found: {list(os.environ.keys())}"
+            )
+            print(">>> Hint: launch with torchrun --nproc_per_node=<N> <script.py>")
+            return False, 0, 0, 1
+
+        rank = int(os.environ["RANK"])
+        world_size = int(os.environ["WORLD_SIZE"])
+        local_rank = int(os.environ.get("LOCAL_RANK", 0))
+
+        # Windows CUDA DDP is not supported
+        if os.name == "nt" and torch.cuda.is_available() and world_size > 1:
+            print(
+                ">>> DDP Setup Disabled: Windows CUDA DDP is not supported. "
+                "Falling back to single process."
+            )
+            return False, 0, 0, 1
+
+        # Set CUDA device for this process
+        if torch.cuda.is_available():
+            torch.cuda.set_device(local_rank)
+
+        # Initialize process group
+        backend = _select_ddp_backend()
+        timeout = _get_ddp_timeout()
+
+        dist.init_process_group(backend=backend, init_method="env://", timeout=timeout)
+        print(
+            f">>> DDP Initialized ({backend}, timeout={timeout.total_seconds():.0f}s): "
+            f"Rank {rank}/{world_size}, Local Rank {local_rank}"
+        )
+
+        return _cache_ddp_state(local_rank, rank, world_size)
+
+    @staticmethod
+    def cleanup_ddp():
+        """Destroy the DDP process group and clear cached state."""
+        if dist.is_initialized():
+            dist.destroy_process_group()
+        DistributedUtils._cached_state = None
+
+    @staticmethod
+    def is_main_process():
+        """Check if current process is rank 0 (main process).
+
+        Returns:
+            True if main process or DDP not initialized
+        """
+        return not dist.is_initialized() or dist.get_rank() == 0
+
+    @staticmethod
+    def world_size() -> int:
+        """Get the total number of processes in the distributed group.
+
+        Returns:
+            World size (1 if DDP not initialized)
+        """
+        return dist.get_world_size() if dist.is_initialized() else 1
+
+
+class TrainingUtils:
+    """General training utilities including CUDA management."""
+
+    @staticmethod
+    def free_cuda() -> None:
+        """Release CUDA memory and clear cache.
+
+        This performs aggressive cleanup:
+        1. Move all PyTorch models to CPU
+        2. Run garbage collection
+        3. Clear CUDA cache
+        """
+        print(">>> Moving all models to CPU...")
+        for obj in gc.get_objects():
+            try:
+                if hasattr(obj, "to") and callable(obj.to):
+                    obj.to("cpu")
+            except Exception:
+                pass
+
+        print(">>> Releasing tensor/optimizer/DataLoader references...")
+        gc.collect()
+
+        print(">>> Clearing CUDA cache...")
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+            torch.cuda.synchronize()
+            print(">>> CUDA memory released.")
+        else:
+            print(">>> CUDA not available; cleanup skipped.")
+
+
+# Backward compatibility function wrapper
+def free_cuda():
+    """Legacy function wrapper for CUDA memory cleanup.
+
+    This function calls TrainingUtils.free_cuda() for backward compatibility.
+    """
+    TrainingUtils.free_cuda()

ins_pricing/modelling/core/bayesopt/utils/io_utils.py

@@ -0,0 +1,126 @@
+"""File I/O and parameter loading utilities.
+
+This module contains:
+- IOUtils class for loading parameters from JSON/CSV/TSV files
+- csv_to_dict() for CSV file handling
+- File path sanitization utilities
+"""
+
+from __future__ import annotations
+
+import csv
+import json
+from pathlib import Path
+from typing import Any, Dict, List
+
+import pandas as pd
+
+
+class IOUtils:
+    """File and path utilities for model parameters and configs."""
+
+    @staticmethod
+    def csv_to_dict(file_path: str) -> List[Dict[str, Any]]:
+        """Load CSV file as list of dictionaries.
+
+        Args:
+            file_path: Path to CSV file
+
+        Returns:
+            List of dictionaries, one per row
+        """
+        with open(file_path, mode='r', encoding='utf-8') as file:
+            reader = csv.DictReader(file)
+            return [
+                dict(filter(lambda item: item[0] != '', row.items()))
+                for row in reader
+            ]
+
+    @staticmethod
+    def ensure_parent_dir(file_path: str) -> None:
+        """Create parent directories when missing.
+
+        Args:
+            file_path: Path to file whose parent directory should be created
+        """
+        directory = Path(file_path).parent
+        if directory and not directory.exists():
+            directory.mkdir(parents=True, exist_ok=True)
+
+    @staticmethod
+    def _sanitize_params_dict(params: Dict[str, Any]) -> Dict[str, Any]:
+        """Filter index-like columns such as "Unnamed: 0" from pandas I/O.
+
+        Args:
+            params: Parameter dictionary
+
+        Returns:
+            Sanitized parameter dictionary
+        """
+        return {
+            k: v
+            for k, v in (params or {}).items()
+            if k and not str(k).startswith("Unnamed")
+        }
+
+    @staticmethod
+    def load_params_file(path: str) -> Dict[str, Any]:
+        """Load parameter dict from JSON/CSV/TSV files.
+
+        Supported formats:
+        - JSON: accept dict or {"best_params": {...}} wrapper
+        - CSV/TSV: read the first row as params
+
+        Args:
+            path: Path to parameter file
+
+        Returns:
+            Parameter dictionary
+
+        Raises:
+            FileNotFoundError: If file doesn't exist
+            ValueError: If file format is unsupported or invalid
+        """
+        file_path = Path(path).expanduser().resolve()
+        if not file_path.exists():
+            raise FileNotFoundError(f"params file not found: {file_path}")
+
+        suffix = file_path.suffix.lower()
+
+        if suffix == ".json":
+            payload = json.loads(file_path.read_text(
+                encoding="utf-8", errors="replace"))
+            if isinstance(payload, dict) and "best_params" in payload:
+                payload = payload.get("best_params") or {}
+            if not isinstance(payload, dict):
+                raise ValueError(
+                    f"Invalid JSON params file (expect dict): {file_path}")
+            return IOUtils._sanitize_params_dict(dict(payload))
+
+        if suffix in (".csv", ".tsv"):
+            df = pd.read_csv(file_path, sep="\t" if suffix == ".tsv" else ",")
+            if df.empty:
+                raise ValueError(f"Empty params file: {file_path}")
+            params = df.iloc[0].to_dict()
+            return IOUtils._sanitize_params_dict(params)
+
+        raise ValueError(
+            f"Unsupported params file type '{suffix}': {file_path}")
+
+
+# Backward compatibility function wrapper
+def csv_to_dict(file_path: str) -> List[Dict[str, Any]]:
+    """Load CSV file as list of dictionaries (legacy function).
+
+    Args:
+        file_path: Path to CSV file
+
+    Returns:
+        List of dictionaries, one per row
+    """
+    return IOUtils.csv_to_dict(file_path)
+
+
+def ensure_parent_dir(file_path: str) -> None:
+    """Create parent directories when missing (legacy function)."""
+    IOUtils.ensure_parent_dir(file_path)