ins-pricing 0.5.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

ins_pricing/modelling/bayesopt/utils/distributed_utils.py

@@ -1,186 +1,193 @@
-"""Distributed training utilities for PyTorch DDP.
-
-This module contains:
-- DistributedUtils for DDP setup and process coordination
-- TrainingUtils for CUDA memory management
-- free_cuda() for legacy compatibility
-"""
-
-from __future__ import annotations
-
-import gc
-import os
-from datetime import timedelta
-from typing import Optional
-
-import torch
-import torch.distributed as dist
-
-
-def _select_ddp_backend() -> str:
-    """Select the appropriate DDP backend based on system capabilities.
-
-    Returns:
-        "nccl" if CUDA is available and NCCL is supported (non-Windows),
-        otherwise "gloo"
-    """
-    if not torch.cuda.is_available():
-        return "gloo"
-
-    if os.name == "nt":  # Windows doesn't support NCCL
-        return "gloo"
-
-    try:
-        nccl_available = getattr(dist, "is_nccl_available", lambda: False)()
-        return "nccl" if nccl_available else "gloo"
-    except Exception:
-        return "gloo"
-
-
-def _get_ddp_timeout() -> timedelta:
-    """Get the DDP timeout from environment variable.
-
-    Returns:
-        timedelta for DDP timeout (default: 1800 seconds)
-    """
-    timeout_seconds = int(os.environ.get("BAYESOPT_DDP_TIMEOUT_SECONDS", "1800"))
-    return timedelta(seconds=max(1, timeout_seconds))
-
-
-def _cache_ddp_state(local_rank: int, rank: int, world_size: int) -> tuple:
-    """Cache and return DDP state tuple."""
-    state = (True, local_rank, rank, world_size)
-    DistributedUtils._cached_state = state
-    return state
-
-
-class DistributedUtils:
-    """Utilities for distributed data parallel training.
-
-    This class provides methods for:
-    - Initializing DDP process groups
-    - Checking process rank and world size
-    - Cleanup after distributed training
-    """
-
-    _cached_state: Optional[tuple] = None
-
-    @staticmethod
-    def setup_ddp():
-        """Initialize the DDP process group for distributed training.
-
-        Returns:
-            Tuple of (success, local_rank, rank, world_size)
-        """
-        # Return cached state if already initialized
-        if dist.is_initialized():
-            if DistributedUtils._cached_state is None:
-                DistributedUtils._cached_state = _cache_ddp_state(
-                    int(os.environ.get("LOCAL_RANK", 0)),
-                    dist.get_rank(),
-                    dist.get_world_size(),
-                )
-            return DistributedUtils._cached_state
-
-        # Check for required environment variables
-        if 'RANK' not in os.environ or 'WORLD_SIZE' not in os.environ:
-            print(
-                f">>> DDP Setup Failed: RANK or WORLD_SIZE not found in env. "
-                f"Keys found: {list(os.environ.keys())}"
-            )
-            print(">>> Hint: launch with torchrun --nproc_per_node=<N> <script.py>")
-            return False, 0, 0, 1
-
-        rank = int(os.environ["RANK"])
-        world_size = int(os.environ["WORLD_SIZE"])
-        local_rank = int(os.environ.get("LOCAL_RANK", 0))
-
-        # Windows CUDA DDP is not supported
-        if os.name == "nt" and torch.cuda.is_available() and world_size > 1:
-            print(
-                ">>> DDP Setup Disabled: Windows CUDA DDP is not supported. "
-                "Falling back to single process."
-            )
-            return False, 0, 0, 1
-
-        # Set CUDA device for this process
-        if torch.cuda.is_available():
-            torch.cuda.set_device(local_rank)
-
-        # Initialize process group
-        backend = _select_ddp_backend()
-        timeout = _get_ddp_timeout()
-
-        dist.init_process_group(backend=backend, init_method="env://", timeout=timeout)
-        print(
-            f">>> DDP Initialized ({backend}, timeout={timeout.total_seconds():.0f}s): "
-            f"Rank {rank}/{world_size}, Local Rank {local_rank}"
-        )
-
-        return _cache_ddp_state(local_rank, rank, world_size)
-
-    @staticmethod
-    def cleanup_ddp():
-        """Destroy the DDP process group and clear cached state."""
-        if dist.is_initialized():
-            dist.destroy_process_group()
-        DistributedUtils._cached_state = None
-
-    @staticmethod
-    def is_main_process():
-        """Check if current process is rank 0 (main process).
-
-        Returns:
-            True if main process or DDP not initialized
-        """
-        return not dist.is_initialized() or dist.get_rank() == 0
-
-    @staticmethod
-    def world_size() -> int:
-        """Get the total number of processes in the distributed group.
-
-        Returns:
-            World size (1 if DDP not initialized)
-        """
-        return dist.get_world_size() if dist.is_initialized() else 1
-
-
-class TrainingUtils:
-    """General training utilities including CUDA management."""
-
-    @staticmethod
-    def free_cuda() -> None:
-        """Release CUDA memory and clear cache.
-
-        This performs aggressive cleanup:
-        1. Move all PyTorch models to CPU
-        2. Run garbage collection
-        3. Clear CUDA cache
-        """
-        print(">>> Moving all models to CPU...")
-        for obj in gc.get_objects():
-            try:
-                if hasattr(obj, "to") and callable(obj.to):
-                    obj.to("cpu")
-            except Exception:
-                pass
-
-        print(">>> Releasing tensor/optimizer/DataLoader references...")
-        gc.collect()
-
-        print(">>> Clearing CUDA cache...")
-        if torch.cuda.is_available():
-            torch.cuda.empty_cache()
-            torch.cuda.synchronize()
-            print(">>> CUDA memory released.")
-        else:
-            print(">>> CUDA not available; cleanup skipped.")
-
-
-# Backward compatibility function wrapper
-def free_cuda():
-    """Legacy function wrapper for CUDA memory cleanup.
-
-    This function calls TrainingUtils.free_cuda() for backward compatibility.
-    """
-    TrainingUtils.free_cuda()
+"""Distributed training utilities for PyTorch DDP.
+
+This module contains:
+- DistributedUtils for DDP setup and process coordination
+- TrainingUtils for CUDA memory management
+- free_cuda() for legacy compatibility
+"""
+
+from __future__ import annotations
+
+import gc
+import os
+from datetime import timedelta
+from typing import Optional
+
+import torch
+import torch.distributed as dist
+from ins_pricing.utils import get_logger, log_print
+
+_logger = get_logger("ins_pricing.modelling.bayesopt.utils.distributed_utils")
+
+
+def _log(*args, **kwargs) -> None:
+    log_print(_logger, *args, **kwargs)
+
+
+def _select_ddp_backend() -> str:
+    """Select the appropriate DDP backend based on system capabilities.
+
+    Returns:
+        "nccl" if CUDA is available and NCCL is supported (non-Windows),
+        otherwise "gloo"
+    """
+    if not torch.cuda.is_available():
+        return "gloo"
+
+    if os.name == "nt":  # Windows doesn't support NCCL
+        return "gloo"
+
+    try:
+        nccl_available = getattr(dist, "is_nccl_available", lambda: False)()
+        return "nccl" if nccl_available else "gloo"
+    except Exception:
+        return "gloo"
+
+
+def _get_ddp_timeout() -> timedelta:
+    """Get the DDP timeout from environment variable.
+
+    Returns:
+        timedelta for DDP timeout (default: 1800 seconds)
+    """
+    timeout_seconds = int(os.environ.get("BAYESOPT_DDP_TIMEOUT_SECONDS", "1800"))
+    return timedelta(seconds=max(1, timeout_seconds))
+
+
+def _cache_ddp_state(local_rank: int, rank: int, world_size: int) -> tuple:
+    """Cache and return DDP state tuple."""
+    state = (True, local_rank, rank, world_size)
+    DistributedUtils._cached_state = state
+    return state
+
+
+class DistributedUtils:
+    """Utilities for distributed data parallel training.
+
+    This class provides methods for:
+    - Initializing DDP process groups
+    - Checking process rank and world size
+    - Cleanup after distributed training
+    """
+
+    _cached_state: Optional[tuple] = None
+
+    @staticmethod
+    def setup_ddp():
+        """Initialize the DDP process group for distributed training.
+
+        Returns:
+            Tuple of (success, local_rank, rank, world_size)
+        """
+        # Return cached state if already initialized
+        if dist.is_initialized():
+            if DistributedUtils._cached_state is None:
+                DistributedUtils._cached_state = _cache_ddp_state(
+                    int(os.environ.get("LOCAL_RANK", 0)),
+                    dist.get_rank(),
+                    dist.get_world_size(),
+                )
+            return DistributedUtils._cached_state
+
+        # Check for required environment variables
+        if 'RANK' not in os.environ or 'WORLD_SIZE' not in os.environ:
+            _log(
+                f">>> DDP Setup Failed: RANK or WORLD_SIZE not found in env. "
+                f"Keys found: {list(os.environ.keys())}"
+            )
+            _log(">>> Hint: launch with torchrun --nproc_per_node=<N> <script.py>")
+            return False, 0, 0, 1
+
+        rank = int(os.environ["RANK"])
+        world_size = int(os.environ["WORLD_SIZE"])
+        local_rank = int(os.environ.get("LOCAL_RANK", 0))
+
+        # Windows CUDA DDP is not supported
+        if os.name == "nt" and torch.cuda.is_available() and world_size > 1:
+            _log(
+                ">>> DDP Setup Disabled: Windows CUDA DDP is not supported. "
+                "Falling back to single process."
+            )
+            return False, 0, 0, 1
+
+        # Set CUDA device for this process
+        if torch.cuda.is_available():
+            torch.cuda.set_device(local_rank)
+
+        # Initialize process group
+        backend = _select_ddp_backend()
+        timeout = _get_ddp_timeout()
+
+        dist.init_process_group(backend=backend, init_method="env://", timeout=timeout)
+        _log(
+            f">>> DDP Initialized ({backend}, timeout={timeout.total_seconds():.0f}s): "
+            f"Rank {rank}/{world_size}, Local Rank {local_rank}"
+        )
+
+        return _cache_ddp_state(local_rank, rank, world_size)
+
+    @staticmethod
+    def cleanup_ddp():
+        """Destroy the DDP process group and clear cached state."""
+        if dist.is_initialized():
+            dist.destroy_process_group()
+        DistributedUtils._cached_state = None
+
+    @staticmethod
+    def is_main_process():
+        """Check if current process is rank 0 (main process).
+
+        Returns:
+            True if main process or DDP not initialized
+        """
+        return not dist.is_initialized() or dist.get_rank() == 0
+
+    @staticmethod
+    def world_size() -> int:
+        """Get the total number of processes in the distributed group.
+
+        Returns:
+            World size (1 if DDP not initialized)
+        """
+        return dist.get_world_size() if dist.is_initialized() else 1
+
+
+class TrainingUtils:
+    """General training utilities including CUDA management."""
+
+    @staticmethod
+    def free_cuda() -> None:
+        """Release CUDA memory and clear cache.
+
+        This performs aggressive cleanup:
+        1. Move all PyTorch models to CPU
+        2. Run garbage collection
+        3. Clear CUDA cache
+        """
+        _log(">>> Moving all models to CPU...")
+        for obj in gc.get_objects():
+            try:
+                if hasattr(obj, "to") and callable(obj.to):
+                    obj.to("cpu")
+            except Exception:
+                pass
+
+        _log(">>> Releasing tensor/optimizer/DataLoader references...")
+        gc.collect()
+
+        _log(">>> Clearing CUDA cache...")
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+            torch.cuda.synchronize()
+            _log(">>> CUDA memory released.")
+        else:
+            _log(">>> CUDA not available; cleanup skipped.")
+
+
+# Backward compatibility function wrapper
+def free_cuda():
+    """Legacy function wrapper for CUDA memory cleanup.
+
+    This function calls TrainingUtils.free_cuda() for backward compatibility.
+    """
+    TrainingUtils.free_cuda()

ins_pricing/modelling/bayesopt/utils/torch_trainer_mixin.py

@@ -51,7 +51,14 @@ except Exception:
         plot_loss_curve_common = None
 
 # Import from other utils modules
-from ins_pricing.utils import EPS, compute_batch_size, tweedie_loss, ensure_parent_dir
+from ins_pricing.utils import (
+    EPS,
+    compute_batch_size,
+    tweedie_loss,
+    ensure_parent_dir,
+    get_logger,
+    log_print,
+)
 from ins_pricing.utils.losses import (
     infer_loss_name_from_model_name,
     loss_requires_positive,
@@ -60,13 +67,19 @@ from ins_pricing.utils.losses import (
 )
 from ins_pricing.modelling.bayesopt.utils.distributed_utils import DistributedUtils
 
+_logger = get_logger("ins_pricing.modelling.bayesopt.utils.torch_trainer_mixin")
+
+
+def _log(*args, **kwargs) -> None:
+    log_print(_logger, *args, **kwargs)
+
 
 def _plot_skip(label: str) -> None:
     """Print message when plot is skipped due to missing matplotlib."""
     if _MPL_IMPORT_ERROR is not None:
-        print(f"[Plot] Skip {label}: matplotlib unavailable ({_MPL_IMPORT_ERROR}).", flush=True)
+        _log(f"[Plot] Skip {label}: matplotlib unavailable ({_MPL_IMPORT_ERROR}).", flush=True)
     else:
-        print(f"[Plot] Skip {label}: matplotlib unavailable.", flush=True)
+        _log(f"[Plot] Skip {label}: matplotlib unavailable.", flush=True)
 
 
 class TorchTrainerMixin:
@@ -121,7 +134,7 @@ class TorchTrainerMixin:
         mps_available = bool(getattr(torch.backends, "mps", None) and torch.backends.mps.is_available())
         ddp_enabled = bool(getattr(self, "is_ddp_enabled", False))
         data_parallel = bool(getattr(self, "use_data_parallel", False))
-        print(
+        _log(
             f">>> Resource summary: device={device}, device_type={device_type}, "
             f"cpu_count={cpu_count}, cuda_count={cuda_count}, mps={mps_available}, "
             f"ddp={ddp_enabled}, data_parallel={data_parallel}, profile={profile}"
@@ -229,7 +242,7 @@ class TorchTrainerMixin:
             return batch_size
         max_batch = max(1, int(budget // per_sample))
         if max_batch < batch_size:
-            print(
+            _log(
                 f">>> Memory cap: batch_size {batch_size} -> {max_batch} "
                 f"(per_sample~{sample_bytes}B, budget~{budget // (1024**2)}MB)"
             )
@@ -289,7 +302,7 @@ class TorchTrainerMixin:
                 device_count = torch.cuda.device_count()
                 if device_count > 1:
                     min_bs = min_bs * device_count
-                    print(
+                    _log(
                         f">>> Multi-GPU detected: {device_count} devices. Adjusted min_bs to {min_bs}.")
 
             if data_size > large_threshold:
@@ -329,7 +342,7 @@ class TorchTrainerMixin:
         if workers > 0:
             prefetch_factor = 4 if profile == "throughput" else 2
         persistent = workers > 0 and profile != "memory_saving"
-        print(
+        _log(
             f">>> DataLoader config: Batch Size={batch_size}, Accum Steps={accum_steps}, "
             f"Workers={workers}, Prefetch={prefetch_factor or 'off'}, Profile={profile}")
         sampler = None
@@ -559,13 +572,13 @@ class TorchTrainerMixin:
             if should_log_epoch:
                 elapsed = int(time.time() - epoch_start_ts)
                 if val_weighted_loss is None:
-                    print(
+                    _log(
                         f"[Training] Epoch {epoch}/{getattr(self, 'epochs', 1)} "
                         f"train_loss={float(train_epoch_loss):.6f} elapsed={elapsed}s",
                         flush=True,
                     )
                 else:
-                    print(
+                    _log(
                         f"[Training] Epoch {epoch}/{getattr(self, 'epochs', 1)} "
                         f"train_loss={float(train_epoch_loss):.6f} "
                         f"val_loss={float(val_weighted_loss):.6f} elapsed={elapsed}s",
@@ -620,4 +633,4 @@ class TorchTrainerMixin:
             plt.tight_layout()
             plt.savefig(save_path, dpi=300)
             plt.close(fig)
-        print(f"[Training] Loss curve saved to {save_path}")
+        _log(f"[Training] Loss curve saved to {save_path}")

ins_pricing/pricing/factors.py

@@ -1,43 +1,46 @@
 from __future__ import annotations
 
-from functools import lru_cache
+from collections import OrderedDict
+import hashlib
 from typing import Optional, Tuple
 
 import numpy as np
 import pandas as pd
 
 
-@lru_cache(maxsize=128)
-def _compute_bins_cached(
-    data_hash: int,
-    n_bins: int,
-    method: str,
-    min_val: float,
-    max_val: float,
-    n_unique: int
-) -> Tuple[tuple, int]:
-    """Cache bin edge computation based on data characteristics.
+_BIN_CACHE_MAXSIZE = 128
+_BIN_CACHE: "OrderedDict[tuple, np.ndarray]" = OrderedDict()
+_BIN_CACHE_HITS = 0
+_BIN_CACHE_MISSES = 0
 
-    Args:
-        data_hash: Hash of sorted unique values for cache key
-        n_bins: Number of bins to create
-        method: Binning method ('quantile' or 'uniform')
-        min_val: Minimum value in data
-        max_val: Maximum value in data
-        n_unique: Number of unique values
 
-    Returns:
-        Tuple of (bin_edges_tuple, actual_bins)
+def _cache_key(series: pd.Series, n_bins: int, method: str) -> Optional[tuple]:
+    try:
+        values = series.dropna().to_numpy(dtype=float, copy=False)
+        if values.size == 0:
+            return None
+        values = np.sort(values)
+        digest = hashlib.blake2b(values.tobytes(), digest_size=16).hexdigest()
+        return (digest, int(values.size), int(n_bins), str(method))
+    except Exception:
+        return None
+
+
+def _cache_get(key: tuple) -> Optional[np.ndarray]:
+    global _BIN_CACHE_HITS, _BIN_CACHE_MISSES
+    if key in _BIN_CACHE:
+        _BIN_CACHE_HITS += 1
+        _BIN_CACHE.move_to_end(key)
+        return _BIN_CACHE[key].copy()
+    _BIN_CACHE_MISSES += 1
+    return None
 
-    Note:
-        This function caches bin computation for identical data distributions.
-        The cache key includes data_hash to ensure correctness while enabling
-        reuse when the same column is binned multiple times.
-    """
-    # This function is called after validation, so we can safely compute
-    # The actual binning is done in the calling function
-    # This just provides a cache key mechanism
-    return (data_hash, n_bins, method, min_val, max_val, n_unique), n_bins
+
+def _cache_set(key: tuple, edges: np.ndarray) -> None:
+    _BIN_CACHE[key] = np.asarray(edges, dtype=float)
+    _BIN_CACHE.move_to_end(key)
+    if len(_BIN_CACHE) > _BIN_CACHE_MAXSIZE:
+        _BIN_CACHE.popitem(last=False)
 
 
 def bin_numeric(
@@ -66,34 +69,40 @@ def bin_numeric(
         When use_cache=True, identical distributions will reuse cached bin edges,
         improving performance when the same column is binned multiple times.
     """
-    # Create cache key from data characteristics if caching enabled
-    if use_cache:
-        # Compute data characteristics for cache key
-        unique_vals = series.dropna().unique()
-        unique_sorted = np.sort(unique_vals)
-        data_hash = hash(unique_sorted.tobytes())
-        min_val = float(series.min())
-        max_val = float(series.max())
-        n_unique = len(unique_vals)
-
-        # Check cache (the function call acts as cache lookup)
-        try:
-            _compute_bins_cached(data_hash, bins, method, min_val, max_val, n_unique)
-        except Exception:
-            # If hashing fails, proceed without cache
-            pass
+    cache_key = _cache_key(series, bins, method) if use_cache else None
+    bin_edges_full: Optional[np.ndarray] = None
+
+    if cache_key is not None:
+        bin_edges_full = _cache_get(cache_key)
+
+    if bin_edges_full is not None:
+        binned = pd.cut(series, bins=bin_edges_full, include_lowest=True, labels=labels)
+        return binned, np.asarray(bin_edges_full[:-1], dtype=float)
 
     # Perform actual binning
     if method == "quantile":
-        binned = pd.qcut(series, q=bins, duplicates="drop", labels=labels)
-        bin_edges = binned.cat.categories.left.to_numpy()
+        binned, bin_edges_full = pd.qcut(
+            series,
+            q=bins,
+            duplicates="drop",
+            labels=labels,
+            retbins=True,
+        )
     elif method == "uniform":
-        binned = pd.cut(series, bins=bins, include_lowest=include_lowest, labels=labels)
-        bin_edges = binned.cat.categories.left.to_numpy()
+        binned, bin_edges_full = pd.cut(
+            series,
+            bins=bins,
+            include_lowest=include_lowest,
+            labels=labels,
+            retbins=True,
+        )
     else:
         raise ValueError("method must be one of: quantile, uniform.")
 
-    return binned, bin_edges
+    if cache_key is not None and bin_edges_full is not None:
+        _cache_set(cache_key, np.asarray(bin_edges_full, dtype=float))
+
+    return binned, np.asarray(bin_edges_full[:-1], dtype=float)
 
 
 def clear_binning_cache() -> None:
@@ -108,7 +117,10 @@ def clear_binning_cache() -> None:
         >>> # After processing many different columns
         >>> clear_binning_cache()
     """
-    _compute_bins_cached.cache_clear()
+    global _BIN_CACHE_HITS, _BIN_CACHE_MISSES
+    _BIN_CACHE.clear()
+    _BIN_CACHE_HITS = 0
+    _BIN_CACHE_MISSES = 0
 
 
 def get_cache_info() -> dict:
@@ -126,12 +138,11 @@ def get_cache_info() -> dict:
         >>> info = get_cache_info()
         >>> print(f"Cache hit rate: {info['hits'] / (info['hits'] + info['misses']):.2%}")
     """
-    cache_info = _compute_bins_cached.cache_info()
     return {
-        'hits': cache_info.hits,
-        'misses': cache_info.misses,
-        'maxsize': cache_info.maxsize,
-        'currsize': cache_info.currsize
+        "hits": _BIN_CACHE_HITS,
+        "misses": _BIN_CACHE_MISSES,
+        "maxsize": _BIN_CACHE_MAXSIZE,
+        "currsize": len(_BIN_CACHE),
     }
 
 

ins_pricing/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="ins_pricing",
-    version="0.5.0",
+    version="0.5.1",
     description="Reusable modelling, pricing, governance, and reporting utilities.",
     author="meishi125478",
     license="Proprietary",