ins-pricing 0.2.7__py3-none-any.whl → 0.2.9__py3-none-any.whl

ins_pricing/tests/production/test_scoring.py

@@ -0,0 +1,311 @@
+"""Tests for production scoring module."""
+
+import numpy as np
+import pandas as pd
+import pytest
+from unittest.mock import Mock, patch
+
+from ins_pricing.exceptions import DataValidationError
+
+
+@pytest.fixture
+def sample_predictions():
+    """Sample prediction data."""
+    return pd.DataFrame({
+        "actual": [100, 150, 200, 250, 300],
+        "predicted": [105, 145, 210, 240, 295],
+        "weight": [1.0, 1.0, 1.0, 1.0, 1.0]
+    })
+
+
+@pytest.fixture
+def classification_data():
+    """Sample classification data."""
+    return pd.DataFrame({
+        "actual": [0, 1, 1, 0, 1, 0, 1, 1, 0, 0],
+        "predicted_proba": [0.1, 0.9, 0.8, 0.2, 0.7, 0.3, 0.85, 0.6, 0.15, 0.25],
+        "predicted_class": [0, 1, 1, 0, 1, 0, 1, 1, 0, 0]
+    })
+
+
+class TestRegressionMetrics:
+    """Test regression scoring metrics."""
+
+    def test_weighted_mse(self, sample_predictions):
+        """Test weighted mean squared error calculation."""
+        from ins_pricing.production.scoring import weighted_mse
+
+        mse = weighted_mse(
+            sample_predictions['actual'],
+            sample_predictions['predicted'],
+            sample_predictions['weight']
+        )
+
+        assert isinstance(mse, (int, float, np.number))
+        assert mse >= 0
+
+    def test_weighted_mae(self, sample_predictions):
+        """Test weighted mean absolute error calculation."""
+        from ins_pricing.production.scoring import weighted_mae
+
+        mae = weighted_mae(
+            sample_predictions['actual'],
+            sample_predictions['predicted'],
+            sample_predictions['weight']
+        )
+
+        assert isinstance(mae, (int, float, np.number))
+        assert mae >= 0
+
+    def test_weighted_r2(self, sample_predictions):
+        """Test weighted R² score calculation."""
+        from ins_pricing.production.scoring import weighted_r2
+
+        r2 = weighted_r2(
+            sample_predictions['actual'],
+            sample_predictions['predicted'],
+            sample_predictions['weight']
+        )
+
+        assert isinstance(r2, (int, float, np.number))
+        assert r2 <= 1.0
+
+    def test_mape(self, sample_predictions):
+        """Test mean absolute percentage error."""
+        from ins_pricing.production.scoring import mape
+
+        mape_score = mape(
+            sample_predictions['actual'],
+            sample_predictions['predicted']
+        )
+
+        assert isinstance(mape_score, (int, float, np.number))
+        assert mape_score >= 0
+
+    def test_metrics_with_zero_actuals(self):
+        """Test metrics handling when actual values are zero."""
+        from ins_pricing.production.scoring import mape
+
+        data = pd.DataFrame({
+            "actual": [0, 100, 200],
+            "predicted": [10, 105, 195]
+        })
+
+        # MAPE should handle zeros gracefully
+        with pytest.raises((ValueError, ZeroDivisionError)):
+            mape(data['actual'], data['predicted'])
+
+
+class TestClassificationMetrics:
+    """Test classification scoring metrics."""
+
+    def test_accuracy(self, classification_data):
+        """Test accuracy calculation."""
+        from ins_pricing.production.scoring import accuracy
+
+        acc = accuracy(
+            classification_data['actual'],
+            classification_data['predicted_class']
+        )
+
+        assert 0 <= acc <= 1
+
+    def test_precision_recall(self, classification_data):
+        """Test precision and recall calculation."""
+        from ins_pricing.production.scoring import precision_recall
+
+        precision, recall = precision_recall(
+            classification_data['actual'],
+            classification_data['predicted_class']
+        )
+
+        assert 0 <= precision <= 1
+        assert 0 <= recall <= 1
+
+    def test_f1_score(self, classification_data):
+        """Test F1 score calculation."""
+        from ins_pricing.production.scoring import f1_score
+
+        f1 = f1_score(
+            classification_data['actual'],
+            classification_data['predicted_class']
+        )
+
+        assert 0 <= f1 <= 1
+
+    def test_roc_auc(self, classification_data):
+        """Test ROC AUC calculation."""
+        from ins_pricing.production.scoring import roc_auc
+
+        auc = roc_auc(
+            classification_data['actual'],
+            classification_data['predicted_proba']
+        )
+
+        assert 0 <= auc <= 1
+
+    def test_confusion_matrix(self, classification_data):
+        """Test confusion matrix generation."""
+        from ins_pricing.production.scoring import confusion_matrix
+
+        cm = confusion_matrix(
+            classification_data['actual'],
+            classification_data['predicted_class']
+        )
+
+        assert cm.shape == (2, 2)
+        assert np.all(cm >= 0)
+
+
+class TestInsuranceMetrics:
+    """Test insurance-specific metrics."""
+
+    def test_loss_ratio(self):
+        """Test loss ratio calculation."""
+        from ins_pricing.production.scoring import loss_ratio
+
+        data = pd.DataFrame({
+            "claims": [100, 200, 150],
+            "premiums": [120, 180, 160],
+            "exposure": [1.0, 1.0, 1.0]
+        })
+
+        lr = loss_ratio(data['claims'], data['premiums'], data['exposure'])
+
+        assert isinstance(lr, (int, float, np.number))
+        assert lr >= 0
+
+    def test_gini_coefficient(self, sample_predictions):
+        """Test Gini coefficient calculation."""
+        from ins_pricing.production.scoring import gini_coefficient
+
+        gini = gini_coefficient(
+            sample_predictions['actual'],
+            sample_predictions['predicted']
+        )
+
+        assert -1 <= gini <= 1
+
+    def test_lift_at_percentile(self, sample_predictions):
+        """Test lift calculation at specific percentile."""
+        from ins_pricing.production.scoring import lift_at_percentile
+
+        lift = lift_at_percentile(
+            sample_predictions['actual'],
+            sample_predictions['predicted'],
+            percentile=20
+        )
+
+        assert isinstance(lift, (int, float, np.number))
+
+
+class TestMetricValidation:
+    """Test metric input validation."""
+
+    def test_mismatched_lengths(self):
+        """Test error on mismatched array lengths."""
+        from ins_pricing.production.scoring import weighted_mse
+        from ins_pricing.utils.validation import validate_dataframe_not_empty
+
+        actual = np.array([1, 2, 3])
+        predicted = np.array([1, 2])  # Wrong length
+        weights = np.array([1, 1, 1])
+
+        with pytest.raises((ValueError, IndexError)):
+            weighted_mse(actual, predicted, weights)
+
+    def test_negative_weights(self):
+        """Test handling of negative weights."""
+        from ins_pricing.production.scoring import weighted_mse
+
+        actual = np.array([100, 200, 300])
+        predicted = np.array([105, 195, 310])
+        weights = np.array([1.0, -1.0, 1.0])  # Negative weight
+
+        with pytest.raises(ValueError):
+            weighted_mse(actual, predicted, weights)
+
+    def test_nan_values(self):
+        """Test handling of NaN values."""
+        from ins_pricing.production.scoring import weighted_mse
+
+        actual = np.array([100, np.nan, 300])
+        predicted = np.array([105, 195, 310])
+        weights = np.array([1.0, 1.0, 1.0])
+
+        with pytest.raises(ValueError):
+            weighted_mse(actual, predicted, weights)
+
+
+class TestScoringReport:
+    """Test scoring report generation."""
+
+    def test_generate_regression_report(self, sample_predictions):
+        """Test comprehensive regression scoring report."""
+        from ins_pricing.production.scoring import generate_scoring_report
+
+        report = generate_scoring_report(
+            actual=sample_predictions['actual'],
+            predicted=sample_predictions['predicted'],
+            weights=sample_predictions['weight'],
+            task_type='regression'
+        )
+
+        assert 'mse' in report
+        assert 'mae' in report
+        assert 'r2' in report
+        assert all(isinstance(v, (int, float, np.number)) for v in report.values())
+
+    def test_generate_classification_report(self, classification_data):
+        """Test comprehensive classification scoring report."""
+        from ins_pricing.production.scoring import generate_scoring_report
+
+        report = generate_scoring_report(
+            actual=classification_data['actual'],
+            predicted=classification_data['predicted_class'],
+            predicted_proba=classification_data['predicted_proba'],
+            task_type='classification'
+        )
+
+        assert 'accuracy' in report
+        assert 'precision' in report
+        assert 'recall' in report
+        assert 'f1' in report
+        assert 'roc_auc' in report
+
+    def test_save_report_to_file(self, sample_predictions, tmp_path):
+        """Test saving scoring report to file."""
+        from ins_pricing.production.scoring import generate_scoring_report, save_report
+
+        report = generate_scoring_report(
+            actual=sample_predictions['actual'],
+            predicted=sample_predictions['predicted'],
+            task_type='regression'
+        )
+
+        output_path = tmp_path / "scoring_report.json"
+        save_report(report, output_path)
+
+        assert output_path.exists()
+
+
+@pytest.mark.performance
+class TestScoringPerformance:
+    """Test scoring performance on large datasets."""
+
+    def test_large_dataset_scoring(self):
+        """Test scoring metrics on large dataset."""
+        from ins_pricing.production.scoring import weighted_mse
+
+        n = 1_000_000
+        actual = np.random.uniform(100, 500, n)
+        predicted = actual + np.random.normal(0, 20, n)
+        weights = np.ones(n)
+
+        import time
+        start = time.time()
+        mse = weighted_mse(actual, predicted, weights)
+        elapsed = time.time() - start
+
+        assert isinstance(mse, (int, float, np.number))
+        assert elapsed < 1.0  # Should complete in under 1 second

ins_pricing/utils/profiling.py

@@ -0,0 +1,377 @@
+"""Performance profiling and memory monitoring utilities.
+
+This module provides tools for tracking execution time, memory usage,
+and GPU resources during model training and data processing.
+
+Example:
+    >>> from ins_pricing.utils.profiling import profile_section
+    >>> with profile_section("Data Loading", logger=my_logger):
+    ...     data = load_large_dataset()
+    [Profile] Data Loading: 5.23s, RAM: +1250.3MB, GPU peak: 2048.5MB
+"""
+
+from __future__ import annotations
+
+import gc
+import logging
+import time
+from contextlib import contextmanager
+from typing import Optional
+
+try:
+    import psutil
+    HAS_PSUTIL = True
+except ImportError:
+    HAS_PSUTIL = False
+
+try:
+    import torch
+    HAS_TORCH = True
+except ImportError:
+    HAS_TORCH = False
+
+
+@contextmanager
+def profile_section(
+    name: str,
+    logger: Optional[logging.Logger] = None,
+    log_level: int = logging.INFO
+):
+    """Context manager for profiling code sections.
+
+    Tracks execution time, RAM usage, and GPU memory (if available).
+    Logs results when the context exits.
+
+    Args:
+        name: Name of the section being profiled
+        logger: Optional logger instance. If None, prints to stdout
+        log_level: Logging level (default: INFO)
+
+    Yields:
+        None
+
+    Example:
+        >>> with profile_section("Training Loop", logger):
+        ...     model.fit(X_train, y_train)
+        [Profile] Training Loop: 45.2s, RAM: +2100.5MB, GPU peak: 4096.0MB
+
+        >>> with profile_section("Preprocessing"):
+        ...     df = preprocess_data(raw_df)
+        [Profile] Preprocessing: 2.1s, RAM: +150.2MB
+    """
+    start_time = time.time()
+
+    # Track CPU memory
+    start_mem = None
+    if HAS_PSUTIL:
+        start_mem = psutil.Process().memory_info().rss / 1024 / 1024  # MB
+
+    # Track GPU memory
+    start_gpu_mem = None
+    if HAS_TORCH and torch.cuda.is_available():
+        torch.cuda.reset_peak_memory_stats()
+        start_gpu_mem = torch.cuda.memory_allocated() / 1024 / 1024  # MB
+
+    try:
+        yield
+    finally:
+        elapsed = time.time() - start_time
+
+        # Build profiling message
+        msg_parts = [f"[Profile] {name}: {elapsed:.2f}s"]
+
+        # Add RAM usage
+        if HAS_PSUTIL and start_mem is not None:
+            end_mem = psutil.Process().memory_info().rss / 1024 / 1024
+            mem_delta = end_mem - start_mem
+            msg_parts.append(f"RAM: {mem_delta:+.1f}MB")
+
+        # Add GPU memory
+        if HAS_TORCH and torch.cuda.is_available() and start_gpu_mem is not None:
+            peak_gpu = torch.cuda.max_memory_allocated() / 1024 / 1024
+            msg_parts.append(f"GPU peak: {peak_gpu:.1f}MB")
+
+        msg = ", ".join(msg_parts)
+
+        if logger:
+            logger.log(log_level, msg)
+        else:
+            print(msg)
+
+
+def get_memory_info() -> dict:
+    """Get current memory usage information.
+
+    Returns:
+        Dictionary with memory statistics:
+        - rss_mb: Resident Set Size in MB (physical memory)
+        - vms_mb: Virtual Memory Size in MB
+        - percent: Memory usage percentage
+        - available_mb: Available system memory in MB
+        - gpu_allocated_mb: GPU memory allocated (if CUDA available)
+        - gpu_cached_mb: GPU memory cached (if CUDA available)
+
+    Example:
+        >>> info = get_memory_info()
+        >>> print(f"Using {info['rss_mb']:.1f} MB RAM")
+        Using 2048.5 MB RAM
+    """
+    info = {}
+
+    if HAS_PSUTIL:
+        process = psutil.Process()
+        mem = process.memory_info()
+        info['rss_mb'] = mem.rss / 1024 / 1024
+        info['vms_mb'] = mem.vms / 1024 / 1024
+
+        vm = psutil.virtual_memory()
+        info['percent'] = vm.percent
+        info['available_mb'] = vm.available / 1024 / 1024
+    else:
+        info['warning'] = 'psutil not available'
+
+    if HAS_TORCH and torch.cuda.is_available():
+        info['gpu_allocated_mb'] = torch.cuda.memory_allocated() / 1024 / 1024
+        info['gpu_cached_mb'] = torch.cuda.memory_reserved() / 1024 / 1024
+        info['gpu_max_allocated_mb'] = torch.cuda.max_memory_allocated() / 1024 / 1024
+
+    return info
+
+
+def log_memory_usage(
+    logger: logging.Logger,
+    prefix: str = "",
+    level: int = logging.INFO
+) -> None:
+    """Log current memory usage.
+
+    Args:
+        logger: Logger instance
+        prefix: Optional prefix for log message
+        level: Logging level (default: INFO)
+
+    Example:
+        >>> log_memory_usage(logger, prefix="After epoch 10")
+        After epoch 10 - Memory: RSS=2048.5MB, GPU=1024.0MB
+    """
+    info = get_memory_info()
+
+    if 'warning' in info:
+        logger.log(level, f"{prefix} - Memory info not available (psutil missing)")
+        return
+
+    msg_parts = []
+    if prefix:
+        msg_parts.append(prefix)
+
+    ram_msg = f"RSS={info['rss_mb']:.1f}MB ({info['percent']:.1f}%)"
+    msg_parts.append(f"Memory: {ram_msg}")
+
+    if 'gpu_allocated_mb' in info:
+        gpu_msg = f"GPU={info['gpu_allocated_mb']:.1f}MB"
+        msg_parts.append(gpu_msg)
+
+    logger.log(level, " - ".join(msg_parts))
+
+
+def check_memory_threshold(
+    threshold_gb: float = 32.0,
+    logger: Optional[logging.Logger] = None
+) -> bool:
+    """Check if memory usage exceeds threshold.
+
+    Args:
+        threshold_gb: Memory threshold in GB (default: 32.0)
+        logger: Optional logger for warnings
+
+    Returns:
+        True if memory usage exceeds threshold, False otherwise
+
+    Example:
+        >>> if check_memory_threshold(threshold_gb=16.0, logger=logger):
+        ...     torch.cuda.empty_cache()
+        ...     gc.collect()
+    """
+    if not HAS_PSUTIL:
+        return False
+
+    mem = psutil.Process().memory_info()
+    rss_gb = mem.rss / 1024 / 1024 / 1024
+
+    if rss_gb > threshold_gb:
+        if logger:
+            logger.warning(
+                f"High memory usage detected: {rss_gb:.1f}GB "
+                f"(threshold: {threshold_gb:.1f}GB)"
+            )
+        return True
+
+    return False
+
+
+def cleanup_memory(logger: Optional[logging.Logger] = None) -> None:
+    """Force memory cleanup for CPU and GPU.
+
+    Args:
+        logger: Optional logger instance
+
+    Example:
+        >>> cleanup_memory(logger)
+        [Memory] Cleanup: freed 250.5MB RAM, 512.0MB GPU
+    """
+    if HAS_PSUTIL:
+        mem_before = psutil.Process().memory_info().rss / 1024 / 1024
+
+    gpu_before = None
+    if HAS_TORCH and torch.cuda.is_available():
+        gpu_before = torch.cuda.memory_allocated() / 1024 / 1024
+
+    # Perform cleanup
+    gc.collect()
+
+    if HAS_TORCH and torch.cuda.is_available():
+        torch.cuda.empty_cache()
+
+    # Calculate freed memory
+    msg_parts = ["[Memory] Cleanup:"]
+
+    if HAS_PSUTIL:
+        mem_after = psutil.Process().memory_info().rss / 1024 / 1024
+        mem_freed = mem_before - mem_after
+        msg_parts.append(f"freed {mem_freed:.1f}MB RAM")
+
+    if gpu_before is not None:
+        gpu_after = torch.cuda.memory_allocated() / 1024 / 1024
+        gpu_freed = gpu_before - gpu_after
+        msg_parts.append(f"{gpu_freed:.1f}MB GPU")
+
+    msg = ", ".join(msg_parts)
+
+    if logger:
+        logger.info(msg)
+    else:
+        print(msg)
+
+
+class MemoryMonitor:
+    """Memory monitoring context manager with automatic cleanup.
+
+    Monitors memory usage and optionally triggers cleanup if threshold exceeded.
+
+    Args:
+        name: Name of the monitored section
+        threshold_gb: Memory threshold for automatic cleanup (default: None, no cleanup)
+        logger: Optional logger instance
+
+    Example:
+        >>> with MemoryMonitor("Training", threshold_gb=16.0, logger=logger):
+        ...     for epoch in range(100):
+        ...         train_epoch(model, data)
+        [Memory] Training started: RAM=1024.5MB, GPU=512.0MB
+        [Memory] Training completed: RAM=2048.3MB (+1023.8MB), GPU=2048.0MB (+1536.0MB)
+    """
+
+    def __init__(
+        self,
+        name: str,
+        threshold_gb: Optional[float] = None,
+        logger: Optional[logging.Logger] = None
+    ):
+        self.name = name
+        self.threshold_gb = threshold_gb
+        self.logger = logger
+        self.start_mem = None
+        self.start_gpu = None
+
+    def __enter__(self):
+        if HAS_PSUTIL:
+            self.start_mem = psutil.Process().memory_info().rss / 1024 / 1024
+
+        if HAS_TORCH and torch.cuda.is_available():
+            self.start_gpu = torch.cuda.memory_allocated() / 1024 / 1024
+
+        # Log starting state
+        msg_parts = [f"[Memory] {self.name} started:"]
+        if self.start_mem is not None:
+            msg_parts.append(f"RAM={self.start_mem:.1f}MB")
+        if self.start_gpu is not None:
+            msg_parts.append(f"GPU={self.start_gpu:.1f}MB")
+
+        msg = ", ".join(msg_parts)
+        if self.logger:
+            self.logger.info(msg)
+        else:
+            print(msg)
+
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        # Calculate deltas
+        msg_parts = [f"[Memory] {self.name} completed:"]
+
+        if HAS_PSUTIL and self.start_mem is not None:
+            end_mem = psutil.Process().memory_info().rss / 1024 / 1024
+            delta_mem = end_mem - self.start_mem
+            msg_parts.append(f"RAM={end_mem:.1f}MB ({delta_mem:+.1f}MB)")
+
+        if HAS_TORCH and torch.cuda.is_available() and self.start_gpu is not None:
+            end_gpu = torch.cuda.memory_allocated() / 1024 / 1024
+            delta_gpu = end_gpu - self.start_gpu
+            msg_parts.append(f"GPU={end_gpu:.1f}MB ({delta_gpu:+.1f}MB)")
+
+        msg = ", ".join(msg_parts)
+        if self.logger:
+            self.logger.info(msg)
+        else:
+            print(msg)
+
+        # Check threshold and cleanup if needed
+        if self.threshold_gb is not None:
+            if check_memory_threshold(self.threshold_gb, self.logger):
+                cleanup_memory(self.logger)
+
+
+def profile_training_epoch(
+    epoch: int,
+    total_epochs: int,
+    logger: Optional[logging.Logger] = None,
+    cleanup_interval: int = 10
+) -> None:
+    """Log memory usage during training epochs with periodic cleanup.
+
+    Args:
+        epoch: Current epoch number
+        total_epochs: Total number of epochs
+        logger: Optional logger instance
+        cleanup_interval: Cleanup memory every N epochs (default: 10)
+
+    Example:
+        >>> for epoch in range(1, 101):
+        ...     train_one_epoch(model, data)
+        ...     profile_training_epoch(epoch, 100, logger, cleanup_interval=10)
+    """
+    log_memory_usage(
+        logger or logging.getLogger(__name__),
+        prefix=f"Epoch {epoch}/{total_epochs}",
+        level=logging.DEBUG
+    )
+
+    # Periodic cleanup
+    if epoch % cleanup_interval == 0:
+        if logger:
+            logger.info(f"Epoch {epoch}: Performing periodic memory cleanup")
+        cleanup_memory(logger)
+
+
+# Convenience function for backward compatibility
+def ensure_memory_cleanup(threshold_gb: float = 32.0) -> None:
+    """Check memory and cleanup if needed (simple function interface).
+
+    Args:
+        threshold_gb: Memory threshold in GB
+
+    Example:
+        >>> ensure_memory_cleanup(threshold_gb=16.0)
+    """
+    if check_memory_threshold(threshold_gb):
+        cleanup_memory()