gitlabds 2.0.0__tar.gz → 2.1.1__tar.gz

{gitlabds-2.0.0 → gitlabds-2.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gitlabds
-Version: 2.0.0
+Version: 2.1.1
 Summary: Gitlab Data Science and Modeling Tools
 Home-page: https://gitlab.com/gitlab-data/gitlabds
 Author: Kevin Dietz
@@ -593,13 +593,12 @@ config.to_yaml("churn_model_config.yaml")
 
 ### Model Evaluation
 
-#### Comprehensive Evaluation
 <details><summary> ModelEvaluator </summary>
 
 #### Description
 A comprehensive framework for evaluating machine learning models, supporting both classification (binary and multi-class) and regression models. It provides extensive evaluation metrics, visualizations, and feature importance analysis.
 
-`gitlabds.ModelEvaluator(model, x_train, y_train, x_test, y_test, x_oot=None, y_oot=None, classification=True, algo=None, f_score=0.50, decile_n=10, top_features_n=20, show_all_classes=True, show_plots=True, save_plots=True, plot_dir='plots', plot_save_format='png', plot_save_dpi=300)`
+`gitlabds.ModelEvaluator(model, x_train, y_train, x_test, y_test, x_oot=None, y_oot=None, classification=True, algo=None, f1_threshold=0.50, decile_n=10, top_features_n=20, show_all_classes=True, show_plots=True, save_plots=True, plot_dir='plots', plot_save_format='png', plot_save_dpi=300)`
 
 #### Parameters:
 - _**model**_ : The trained model to evaluate. Must have predict for regression and predict_proba method for classification
@@ -611,7 +610,7 @@ A comprehensive framework for evaluating machine learning models, supporting bot
 - _**y_oot**_ : Optional out-of-time validation labels.
 - _**classification**_ : Whether this is a classification model. If False, regression metrics will be used.
 - _**algo**_ : Algorithm type for feature importance calculation. Options: 'xgb', 'rf', 'mars'. For other algorithms, use `None`
-- _**f_score**_ : Threshold for binary classification.
+- _**f1_threshold**_ : Threshold for binary classification.
 - _**decile_n**_ : Number of n-tiles for lift calculation. Defaults to 10 for deciles
 - _**top_features_n**_ : Number of top features to display in visualizations.
 - _**show_all_classes**_ : Whether to show metrics for all classes in multi-class classification.
@@ -674,58 +673,8 @@ results.feature_importance.to_csv("feature_importance.csv")
 ```
 </details>
 
-#### Metric Functions
-<details><summary> Model Metrics </summary>
 
-#### Description
-Display a variety of model metrics for linear and logistic predictive models.
-
-`gitlabds.model_metrics(model, x_train, y_train, x_test, y_test, show_graphs=True, f_score=0.50, classification=True, algo=None, decile_n=10, top_features_n=20):`
-
-#### Parameters:
-- _**model**_ : model file from training
-- _**x_train**_ : Training features DataFrame.
-- _**y_train**_ : Training labels (Series or DataFrame).
-- _**x_test**_ : Test features DataFrame.
-- _**y_test**_ : Test labels (Series or DataFrame).
-- _**show_graphs**_ : Whether to display plots and graphs.
-- _**f_score**_ : Threshold for binary classification.
-- _**classification**_ : Whether this is a classification model. If False, regression metrics will be used.
-- _**algo**_ : Algorithm type for feature importance calculation. Options: 'xgb', 'rf', 'mars'.
-- _**decile_n**_ : Number of n-tiles for lift calculation.
-- _**top_features_n**_ : Number of top features to display in feature importance.
-
-#### Returns
-- For classification models: tuple of (metricx, lift, classification_metricx, top_features, decile_breaks)
-- For regression models: tuple of (metricx, top_features)
-
-#### Examples:
-```python
-# For a classification model
-import gitlabds
-metricx, lift, classification_metricx, top_features, decile_breaks = gitlabds.model_metrics(
-    model=my_classifier,
-    x_train=x_train,
-    y_train=y_train, 
-    x_test=x_test, 
-    y_test=y_test,
-    classification=True, 
-    algo='xgb'
-)
-
-# For a regression model
-metricx, top_features = gitlabds.model_metrics(
-    model=my_regressor,
-    x_train=x_train,
-    y_train=y_train, 
-    x_test=x_test, 
-    y_test=y_test,
-    classification=False
-)
-```
-</details>
-
-#### Insight Generation
+### Insight Generation
 <details><summary> Marginal Effects </summary>
 
 #### Description
@@ -808,9 +757,154 @@ results = gitlabds.prescriptions(
 ```
 </details>
 
+### Model Monitoring
+
+<details><summary> Generate Baseline Features </summary>
+
+#### Description
+Generate baseline feature distributions, importance scores, and drift thresholds in a single comprehensive artifact for model monitoring.
+
+`gitlabds.generate_baseline_features(training_data, feature_importance_df, importance_method="shapley_values", n_bins=10, psi_warning=0.1, psi_critical=0.2, ks_warning=0.2, ks_critical=0.3, js_warning=0.1, js_critical=0.2, output_path="baseline_features.json"):`
+
+#### Parameters:
+- **_training_data_** : Training feature data DataFrame
+- **_feature_importance_df_** : DataFrame with columns: feature, importance
+- **_importance_method_** : Method used to calculate importance (default: "shapley_values")
+- **_n_bins_** : Number of bins for numerical features (default: 10)
+- **_psi_warning, psi_critical_** : PSI thresholds for warning and critical drift detection
+- **_ks_warning, ks_critical_** : KS statistic thresholds for warning and critical drift detection
+- **_js_warning, js_critical_** : JS divergence thresholds for warning and critical drift detection
+- **_output_path_** : Path to save the JSON file
+
+#### Returns
+- None (saves baseline artifact to JSON file)
+
+#### Examples:
+```python
+# Generate baseline features with default thresholds
+import gitlabds
+gitlabds.generate_baseline_features(
+    training_data=train_df,
+    feature_importance_df=importance_df,
+    importance_method="shapley_values",
+    output_path="model_baseline_features.json"
+)
+```
+
+```python
+# Generate with custom drift thresholds
+gitlabds.generate_baseline_features(
+    training_data=train_df,
+    feature_importance_df=importance_df,
+    n_bins=15,
+    psi_warning=0.15,
+    psi_critical=0.25,
+    output_path="custom_baseline_features.json"
+)
+```
+</details>
+
+<details><summary> Generate Baseline Calibration </summary>
+
+#### Description
+Generate baseline calibration data with train + test curves, prediction statistics, and model configuration for monitoring model calibration drift.
+
+`gitlabds.generate_baseline_calibration(train_predictions, train_actuals, test_predictions, test_actuals, model_configuration, n_bins=10, output_path="baseline_calibration.json"):`
+
+#### Parameters:
+- **_train_predictions_** : Training set predicted probabilities
+- **_train_actuals_** : Training set actual binary outcomes (0/1)
+- **_test_predictions_** : Test set predicted probabilities
+- **_test_actuals_** : Test set actual binary outcomes (0/1)
+- **_model_configuration_** : Dictionary of model configuration parameters
+- **_n_bins_** : Number of bins for calibration curve (default: 10)
+- **_prediction_drift_warning**_ :  Warning level for prediction score drift (default: 0.10), 
+- **_prediction_drift_critical**_ : Critical level for prediction score drift (default: 0.20),  
+- **_output_path_** : Path to save the JSON file
+        
+#### Returns
+- None (saves baseline calibration to JSON file)
+
+#### Examples:
+```python
+# Generate baseline calibration
+import gitlabds
+model_config = {'f1_threshold': 0.15, 'random_state': 42}
+
+gitlabds.generate_baseline_calibration(
+    train_predictions=y_train_pred,
+    train_actuals=y_train,
+    test_predictions=y_test_pred,
+    test_actuals=y_test,
+    model_configuration=model_config,
+    prediction_drift_warning=0.15,
+    prediction_drift_critical=0.25
+    output_path="model_baseline_calibration.json"
+)
+```
+</details>
+
+<details><summary> Calculate Monitoring Metrics </summary>
+
+#### Description
+Calculate all monitoring metrics including feature drift, prediction drift, and health status for a model scoring run.
+
+`gitlabds.calculate_monitoring_metrics(run_id, model_name, sub_model, model_version, scoring_date, feature_df, predictions, baseline_metrics, importance_threshold_pct=0.05):`
+
+#### Parameters:
+- **_run_id_** : Unique identifier for this scoring run
+- **_model_name_** : Model name (e.g., "propensity_model")
+- **_sub_model_** : Sub model identifier
+- **_model_version_** : Model version (e.g., "2.1")
+- **_scoring_date_** : Date of scoring
+- **_feature_df_** : Feature data used for scoring
+- **_predictions_** : Model predictions (probabilities 0-1)
+- **_baseline_metrics_** : Dictionary containing baseline_features and baseline_calibration JSON data
+- **_importance_threshold_pct_** : Percentage of total importance required for a feature to be considered "important"
+
+#### Returns
+- Dictionary containing DataFrames for each table:
+  - 'scoring_summary': model_scoring_summary table
+  - 'feature_drift': model_feature_drift table
+
+#### Examples:
+```python
+# Calculate monitoring metrics for a scoring run
+import gitlabds
+from datetime import datetime
+
+# Load baseline metrics
+with open('baseline_features.json', 'r') as f:
+    baseline_features = json.load(f)
+with open('baseline_calibration.json', 'r') as f:
+    baseline_calibration = json.load(f)
+
+baseline_metrics = {
+    'baseline_features': baseline_features,
+    'baseline_calibration': baseline_calibration
+}
+
+# Calculate metrics
+results = gitlabds.calculate_monitoring_metrics(
+    run_id="scoring_run_123",
+    model_name="churn_prediction",
+    sub_model="high_value_customers",
+    model_version="2.1",
+    scoring_date=datetime.now(),
+    feature_df=current_features,
+    predictions=model_predictions,
+    baseline_metrics=baseline_metrics,
+    importance_threshold_pct=0.05
+)
+
+# Access results
+scoring_summary = results['scoring_summary']
+feature_drift = results['feature_drift']
+```
+</details>
+
 ### SQL and Trend Analysis
 
-#### SQL Generation
 <details><summary> SQL Trend Query Generator </summary>
 
 #### Description
@@ -884,7 +978,6 @@ sql = gitlabds.generate_sql_trend_query(
 ```
 </details>
 
-#### Trend Analysis
 <details><summary> Trend Analysis </summary>
 
 #### Description

{gitlabds-2.0.0 → gitlabds-2.1.1}/README.md

@@ -573,13 +573,12 @@ config.to_yaml("churn_model_config.yaml")
 
 ### Model Evaluation
 
-#### Comprehensive Evaluation
 <details><summary> ModelEvaluator </summary>
 
 #### Description
 A comprehensive framework for evaluating machine learning models, supporting both classification (binary and multi-class) and regression models. It provides extensive evaluation metrics, visualizations, and feature importance analysis.
 
-`gitlabds.ModelEvaluator(model, x_train, y_train, x_test, y_test, x_oot=None, y_oot=None, classification=True, algo=None, f_score=0.50, decile_n=10, top_features_n=20, show_all_classes=True, show_plots=True, save_plots=True, plot_dir='plots', plot_save_format='png', plot_save_dpi=300)`
+`gitlabds.ModelEvaluator(model, x_train, y_train, x_test, y_test, x_oot=None, y_oot=None, classification=True, algo=None, f1_threshold=0.50, decile_n=10, top_features_n=20, show_all_classes=True, show_plots=True, save_plots=True, plot_dir='plots', plot_save_format='png', plot_save_dpi=300)`
 
 #### Parameters:
 - _**model**_ : The trained model to evaluate. Must have predict for regression and predict_proba method for classification
@@ -591,7 +590,7 @@ A comprehensive framework for evaluating machine learning models, supporting bot
 - _**y_oot**_ : Optional out-of-time validation labels.
 - _**classification**_ : Whether this is a classification model. If False, regression metrics will be used.
 - _**algo**_ : Algorithm type for feature importance calculation. Options: 'xgb', 'rf', 'mars'. For other algorithms, use `None`
-- _**f_score**_ : Threshold for binary classification.
+- _**f1_threshold**_ : Threshold for binary classification.
 - _**decile_n**_ : Number of n-tiles for lift calculation. Defaults to 10 for deciles
 - _**top_features_n**_ : Number of top features to display in visualizations.
 - _**show_all_classes**_ : Whether to show metrics for all classes in multi-class classification.
@@ -654,58 +653,8 @@ results.feature_importance.to_csv("feature_importance.csv")
 ```
 </details>
 
-#### Metric Functions
-<details><summary> Model Metrics </summary>
 
-#### Description
-Display a variety of model metrics for linear and logistic predictive models.
-
-`gitlabds.model_metrics(model, x_train, y_train, x_test, y_test, show_graphs=True, f_score=0.50, classification=True, algo=None, decile_n=10, top_features_n=20):`
-
-#### Parameters:
-- _**model**_ : model file from training
-- _**x_train**_ : Training features DataFrame.
-- _**y_train**_ : Training labels (Series or DataFrame).
-- _**x_test**_ : Test features DataFrame.
-- _**y_test**_ : Test labels (Series or DataFrame).
-- _**show_graphs**_ : Whether to display plots and graphs.
-- _**f_score**_ : Threshold for binary classification.
-- _**classification**_ : Whether this is a classification model. If False, regression metrics will be used.
-- _**algo**_ : Algorithm type for feature importance calculation. Options: 'xgb', 'rf', 'mars'.
-- _**decile_n**_ : Number of n-tiles for lift calculation.
-- _**top_features_n**_ : Number of top features to display in feature importance.
-
-#### Returns
-- For classification models: tuple of (metricx, lift, classification_metricx, top_features, decile_breaks)
-- For regression models: tuple of (metricx, top_features)
-
-#### Examples:
-```python
-# For a classification model
-import gitlabds
-metricx, lift, classification_metricx, top_features, decile_breaks = gitlabds.model_metrics(
-    model=my_classifier,
-    x_train=x_train,
-    y_train=y_train, 
-    x_test=x_test, 
-    y_test=y_test,
-    classification=True, 
-    algo='xgb'
-)
-
-# For a regression model
-metricx, top_features = gitlabds.model_metrics(
-    model=my_regressor,
-    x_train=x_train,
-    y_train=y_train, 
-    x_test=x_test, 
-    y_test=y_test,
-    classification=False
-)
-```
-</details>
-
-#### Insight Generation
+### Insight Generation
 <details><summary> Marginal Effects </summary>
 
 #### Description
@@ -788,9 +737,154 @@ results = gitlabds.prescriptions(
 ```
 </details>
 
+### Model Monitoring
+
+<details><summary> Generate Baseline Features </summary>
+
+#### Description
+Generate baseline feature distributions, importance scores, and drift thresholds in a single comprehensive artifact for model monitoring.
+
+`gitlabds.generate_baseline_features(training_data, feature_importance_df, importance_method="shapley_values", n_bins=10, psi_warning=0.1, psi_critical=0.2, ks_warning=0.2, ks_critical=0.3, js_warning=0.1, js_critical=0.2, output_path="baseline_features.json"):`
+
+#### Parameters:
+- **_training_data_** : Training feature data DataFrame
+- **_feature_importance_df_** : DataFrame with columns: feature, importance
+- **_importance_method_** : Method used to calculate importance (default: "shapley_values")
+- **_n_bins_** : Number of bins for numerical features (default: 10)
+- **_psi_warning, psi_critical_** : PSI thresholds for warning and critical drift detection
+- **_ks_warning, ks_critical_** : KS statistic thresholds for warning and critical drift detection
+- **_js_warning, js_critical_** : JS divergence thresholds for warning and critical drift detection
+- **_output_path_** : Path to save the JSON file
+
+#### Returns
+- None (saves baseline artifact to JSON file)
+
+#### Examples:
+```python
+# Generate baseline features with default thresholds
+import gitlabds
+gitlabds.generate_baseline_features(
+    training_data=train_df,
+    feature_importance_df=importance_df,
+    importance_method="shapley_values",
+    output_path="model_baseline_features.json"
+)
+```
+
+```python
+# Generate with custom drift thresholds
+gitlabds.generate_baseline_features(
+    training_data=train_df,
+    feature_importance_df=importance_df,
+    n_bins=15,
+    psi_warning=0.15,
+    psi_critical=0.25,
+    output_path="custom_baseline_features.json"
+)
+```
+</details>
+
+<details><summary> Generate Baseline Calibration </summary>
+
+#### Description
+Generate baseline calibration data with train + test curves, prediction statistics, and model configuration for monitoring model calibration drift.
+
+`gitlabds.generate_baseline_calibration(train_predictions, train_actuals, test_predictions, test_actuals, model_configuration, n_bins=10, output_path="baseline_calibration.json"):`
+
+#### Parameters:
+- **_train_predictions_** : Training set predicted probabilities
+- **_train_actuals_** : Training set actual binary outcomes (0/1)
+- **_test_predictions_** : Test set predicted probabilities
+- **_test_actuals_** : Test set actual binary outcomes (0/1)
+- **_model_configuration_** : Dictionary of model configuration parameters
+- **_n_bins_** : Number of bins for calibration curve (default: 10)
+- **_prediction_drift_warning**_ :  Warning level for prediction score drift (default: 0.10), 
+- **_prediction_drift_critical**_ : Critical level for prediction score drift (default: 0.20),  
+- **_output_path_** : Path to save the JSON file
+        
+#### Returns
+- None (saves baseline calibration to JSON file)
+
+#### Examples:
+```python
+# Generate baseline calibration
+import gitlabds
+model_config = {'f1_threshold': 0.15, 'random_state': 42}
+
+gitlabds.generate_baseline_calibration(
+    train_predictions=y_train_pred,
+    train_actuals=y_train,
+    test_predictions=y_test_pred,
+    test_actuals=y_test,
+    model_configuration=model_config,
+    prediction_drift_warning=0.15,
+    prediction_drift_critical=0.25
+    output_path="model_baseline_calibration.json"
+)
+```
+</details>
+
+<details><summary> Calculate Monitoring Metrics </summary>
+
+#### Description
+Calculate all monitoring metrics including feature drift, prediction drift, and health status for a model scoring run.
+
+`gitlabds.calculate_monitoring_metrics(run_id, model_name, sub_model, model_version, scoring_date, feature_df, predictions, baseline_metrics, importance_threshold_pct=0.05):`
+
+#### Parameters:
+- **_run_id_** : Unique identifier for this scoring run
+- **_model_name_** : Model name (e.g., "propensity_model")
+- **_sub_model_** : Sub model identifier
+- **_model_version_** : Model version (e.g., "2.1")
+- **_scoring_date_** : Date of scoring
+- **_feature_df_** : Feature data used for scoring
+- **_predictions_** : Model predictions (probabilities 0-1)
+- **_baseline_metrics_** : Dictionary containing baseline_features and baseline_calibration JSON data
+- **_importance_threshold_pct_** : Percentage of total importance required for a feature to be considered "important"
+
+#### Returns
+- Dictionary containing DataFrames for each table:
+  - 'scoring_summary': model_scoring_summary table
+  - 'feature_drift': model_feature_drift table
+
+#### Examples:
+```python
+# Calculate monitoring metrics for a scoring run
+import gitlabds
+from datetime import datetime
+
+# Load baseline metrics
+with open('baseline_features.json', 'r') as f:
+    baseline_features = json.load(f)
+with open('baseline_calibration.json', 'r') as f:
+    baseline_calibration = json.load(f)
+
+baseline_metrics = {
+    'baseline_features': baseline_features,
+    'baseline_calibration': baseline_calibration
+}
+
+# Calculate metrics
+results = gitlabds.calculate_monitoring_metrics(
+    run_id="scoring_run_123",
+    model_name="churn_prediction",
+    sub_model="high_value_customers",
+    model_version="2.1",
+    scoring_date=datetime.now(),
+    feature_df=current_features,
+    predictions=model_predictions,
+    baseline_metrics=baseline_metrics,
+    importance_threshold_pct=0.05
+)
+
+# Access results
+scoring_summary = results['scoring_summary']
+feature_drift = results['feature_drift']
+```
+</details>
+
 ### SQL and Trend Analysis
 
-#### SQL Generation
 <details><summary> SQL Trend Query Generator </summary>
 
 #### Description
@@ -864,7 +958,6 @@ sql = gitlabds.generate_sql_trend_query(
 ```
 </details>
 
-#### Trend Analysis
 <details><summary> Trend Analysis </summary>
 
 #### Description

{gitlabds-2.0.0 → gitlabds-2.1.1}/gitlabds/__init__.py

@@ -1,3 +1,4 @@
+from .baselines import generate_baseline_features, generate_baseline_calibration
 from .config_generator import ConfigGenerator
 from .dummy import dummy_code, dummy_top, apply_dummy
 from .feature_reduction import drop_categorical, remove_low_variation, remove_outcome_proxies, correlation_reduction
@@ -5,7 +6,7 @@ from .insights import marginal_effects, prescriptions
 from .memory_optimization import reduce_memory_usage, sparse_encode_df, identify_categorical_candidates, compact_categorical, memory_profile_df, memory_optimization
 from .missing import missing_values, apply_missing_values
 from .model_evaluator import ModelEvaluator
-from .model_metrics import model_metrics
+from .monitoring_metrics import calculate_monitoring_metrics, calculate_baseline_prediction_distribution, calculate_prediction_distribution, calculate_distribution_shift, calculate_psi_numerical, calculate_psi_categorical
 from .outliers import mad_outliers, apply_outliers
 from .split_data import split_data
 from .trends import generate_sql_trend_query, trend_analysis