ins-pricing 0.1.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (169) hide show
  1. ins_pricing/README.md +60 -0
  2. ins_pricing/__init__.py +102 -0
  3. ins_pricing/governance/README.md +18 -0
  4. ins_pricing/governance/__init__.py +20 -0
  5. ins_pricing/governance/approval.py +93 -0
  6. ins_pricing/governance/audit.py +37 -0
  7. ins_pricing/governance/registry.py +99 -0
  8. ins_pricing/governance/release.py +159 -0
  9. ins_pricing/modelling/BayesOpt.py +146 -0
  10. ins_pricing/modelling/BayesOpt_USAGE.md +925 -0
  11. ins_pricing/modelling/BayesOpt_entry.py +575 -0
  12. ins_pricing/modelling/BayesOpt_incremental.py +731 -0
  13. ins_pricing/modelling/Explain_Run.py +36 -0
  14. ins_pricing/modelling/Explain_entry.py +539 -0
  15. ins_pricing/modelling/Pricing_Run.py +36 -0
  16. ins_pricing/modelling/README.md +33 -0
  17. ins_pricing/modelling/__init__.py +44 -0
  18. ins_pricing/modelling/bayesopt/__init__.py +98 -0
  19. ins_pricing/modelling/bayesopt/config_preprocess.py +303 -0
  20. ins_pricing/modelling/bayesopt/core.py +1476 -0
  21. ins_pricing/modelling/bayesopt/models.py +2196 -0
  22. ins_pricing/modelling/bayesopt/trainers.py +2446 -0
  23. ins_pricing/modelling/bayesopt/utils.py +1021 -0
  24. ins_pricing/modelling/cli_common.py +136 -0
  25. ins_pricing/modelling/explain/__init__.py +55 -0
  26. ins_pricing/modelling/explain/gradients.py +334 -0
  27. ins_pricing/modelling/explain/metrics.py +176 -0
  28. ins_pricing/modelling/explain/permutation.py +155 -0
  29. ins_pricing/modelling/explain/shap_utils.py +146 -0
  30. ins_pricing/modelling/notebook_utils.py +284 -0
  31. ins_pricing/modelling/plotting/__init__.py +45 -0
  32. ins_pricing/modelling/plotting/common.py +63 -0
  33. ins_pricing/modelling/plotting/curves.py +572 -0
  34. ins_pricing/modelling/plotting/diagnostics.py +139 -0
  35. ins_pricing/modelling/plotting/geo.py +362 -0
  36. ins_pricing/modelling/plotting/importance.py +121 -0
  37. ins_pricing/modelling/run_logging.py +133 -0
  38. ins_pricing/modelling/tests/conftest.py +8 -0
  39. ins_pricing/modelling/tests/test_cross_val_generic.py +66 -0
  40. ins_pricing/modelling/tests/test_distributed_utils.py +18 -0
  41. ins_pricing/modelling/tests/test_explain.py +56 -0
  42. ins_pricing/modelling/tests/test_geo_tokens_split.py +49 -0
  43. ins_pricing/modelling/tests/test_graph_cache.py +33 -0
  44. ins_pricing/modelling/tests/test_plotting.py +63 -0
  45. ins_pricing/modelling/tests/test_plotting_library.py +150 -0
  46. ins_pricing/modelling/tests/test_preprocessor.py +48 -0
  47. ins_pricing/modelling/watchdog_run.py +211 -0
  48. ins_pricing/pricing/README.md +44 -0
  49. ins_pricing/pricing/__init__.py +27 -0
  50. ins_pricing/pricing/calibration.py +39 -0
  51. ins_pricing/pricing/data_quality.py +117 -0
  52. ins_pricing/pricing/exposure.py +85 -0
  53. ins_pricing/pricing/factors.py +91 -0
  54. ins_pricing/pricing/monitoring.py +99 -0
  55. ins_pricing/pricing/rate_table.py +78 -0
  56. ins_pricing/production/__init__.py +21 -0
  57. ins_pricing/production/drift.py +30 -0
  58. ins_pricing/production/monitoring.py +143 -0
  59. ins_pricing/production/scoring.py +40 -0
  60. ins_pricing/reporting/README.md +20 -0
  61. ins_pricing/reporting/__init__.py +11 -0
  62. ins_pricing/reporting/report_builder.py +72 -0
  63. ins_pricing/reporting/scheduler.py +45 -0
  64. ins_pricing/setup.py +41 -0
  65. ins_pricing v2/__init__.py +23 -0
  66. ins_pricing v2/governance/__init__.py +20 -0
  67. ins_pricing v2/governance/approval.py +93 -0
  68. ins_pricing v2/governance/audit.py +37 -0
  69. ins_pricing v2/governance/registry.py +99 -0
  70. ins_pricing v2/governance/release.py +159 -0
  71. ins_pricing v2/modelling/Explain_Run.py +36 -0
  72. ins_pricing v2/modelling/Pricing_Run.py +36 -0
  73. ins_pricing v2/modelling/__init__.py +151 -0
  74. ins_pricing v2/modelling/cli_common.py +141 -0
  75. ins_pricing v2/modelling/config.py +249 -0
  76. ins_pricing v2/modelling/config_preprocess.py +254 -0
  77. ins_pricing v2/modelling/core.py +741 -0
  78. ins_pricing v2/modelling/data_container.py +42 -0
  79. ins_pricing v2/modelling/explain/__init__.py +55 -0
  80. ins_pricing v2/modelling/explain/gradients.py +334 -0
  81. ins_pricing v2/modelling/explain/metrics.py +176 -0
  82. ins_pricing v2/modelling/explain/permutation.py +155 -0
  83. ins_pricing v2/modelling/explain/shap_utils.py +146 -0
  84. ins_pricing v2/modelling/features.py +215 -0
  85. ins_pricing v2/modelling/model_manager.py +148 -0
  86. ins_pricing v2/modelling/model_plotting.py +463 -0
  87. ins_pricing v2/modelling/models.py +2203 -0
  88. ins_pricing v2/modelling/notebook_utils.py +294 -0
  89. ins_pricing v2/modelling/plotting/__init__.py +45 -0
  90. ins_pricing v2/modelling/plotting/common.py +63 -0
  91. ins_pricing v2/modelling/plotting/curves.py +572 -0
  92. ins_pricing v2/modelling/plotting/diagnostics.py +139 -0
  93. ins_pricing v2/modelling/plotting/geo.py +362 -0
  94. ins_pricing v2/modelling/plotting/importance.py +121 -0
  95. ins_pricing v2/modelling/run_logging.py +133 -0
  96. ins_pricing v2/modelling/tests/conftest.py +8 -0
  97. ins_pricing v2/modelling/tests/test_cross_val_generic.py +66 -0
  98. ins_pricing v2/modelling/tests/test_distributed_utils.py +18 -0
  99. ins_pricing v2/modelling/tests/test_explain.py +56 -0
  100. ins_pricing v2/modelling/tests/test_geo_tokens_split.py +49 -0
  101. ins_pricing v2/modelling/tests/test_graph_cache.py +33 -0
  102. ins_pricing v2/modelling/tests/test_plotting.py +63 -0
  103. ins_pricing v2/modelling/tests/test_plotting_library.py +150 -0
  104. ins_pricing v2/modelling/tests/test_preprocessor.py +48 -0
  105. ins_pricing v2/modelling/trainers.py +2447 -0
  106. ins_pricing v2/modelling/utils.py +1020 -0
  107. ins_pricing v2/modelling/watchdog_run.py +211 -0
  108. ins_pricing v2/pricing/__init__.py +27 -0
  109. ins_pricing v2/pricing/calibration.py +39 -0
  110. ins_pricing v2/pricing/data_quality.py +117 -0
  111. ins_pricing v2/pricing/exposure.py +85 -0
  112. ins_pricing v2/pricing/factors.py +91 -0
  113. ins_pricing v2/pricing/monitoring.py +99 -0
  114. ins_pricing v2/pricing/rate_table.py +78 -0
  115. ins_pricing v2/production/__init__.py +21 -0
  116. ins_pricing v2/production/drift.py +30 -0
  117. ins_pricing v2/production/monitoring.py +143 -0
  118. ins_pricing v2/production/scoring.py +40 -0
  119. ins_pricing v2/reporting/__init__.py +11 -0
  120. ins_pricing v2/reporting/report_builder.py +72 -0
  121. ins_pricing v2/reporting/scheduler.py +45 -0
  122. ins_pricing v2/scripts/BayesOpt_incremental.py +722 -0
  123. ins_pricing v2/scripts/Explain_entry.py +545 -0
  124. ins_pricing v2/scripts/__init__.py +1 -0
  125. ins_pricing v2/scripts/train.py +568 -0
  126. ins_pricing v2/setup.py +55 -0
  127. ins_pricing v2/smoke_test.py +28 -0
  128. ins_pricing-0.1.6.dist-info/METADATA +78 -0
  129. ins_pricing-0.1.6.dist-info/RECORD +169 -0
  130. ins_pricing-0.1.6.dist-info/WHEEL +5 -0
  131. ins_pricing-0.1.6.dist-info/top_level.txt +4 -0
  132. user_packages/__init__.py +105 -0
  133. user_packages legacy/BayesOpt.py +5659 -0
  134. user_packages legacy/BayesOpt_entry.py +513 -0
  135. user_packages legacy/BayesOpt_incremental.py +685 -0
  136. user_packages legacy/Pricing_Run.py +36 -0
  137. user_packages legacy/Try/BayesOpt Legacy251213.py +3719 -0
  138. user_packages legacy/Try/BayesOpt Legacy251215.py +3758 -0
  139. user_packages legacy/Try/BayesOpt lagecy251201.py +3506 -0
  140. user_packages legacy/Try/BayesOpt lagecy251218.py +3992 -0
  141. user_packages legacy/Try/BayesOpt legacy.py +3280 -0
  142. user_packages legacy/Try/BayesOpt.py +838 -0
  143. user_packages legacy/Try/BayesOptAll.py +1569 -0
  144. user_packages legacy/Try/BayesOptAllPlatform.py +909 -0
  145. user_packages legacy/Try/BayesOptCPUGPU.py +1877 -0
  146. user_packages legacy/Try/BayesOptSearch.py +830 -0
  147. user_packages legacy/Try/BayesOptSearchOrigin.py +829 -0
  148. user_packages legacy/Try/BayesOptV1.py +1911 -0
  149. user_packages legacy/Try/BayesOptV10.py +2973 -0
  150. user_packages legacy/Try/BayesOptV11.py +3001 -0
  151. user_packages legacy/Try/BayesOptV12.py +3001 -0
  152. user_packages legacy/Try/BayesOptV2.py +2065 -0
  153. user_packages legacy/Try/BayesOptV3.py +2209 -0
  154. user_packages legacy/Try/BayesOptV4.py +2342 -0
  155. user_packages legacy/Try/BayesOptV5.py +2372 -0
  156. user_packages legacy/Try/BayesOptV6.py +2759 -0
  157. user_packages legacy/Try/BayesOptV7.py +2832 -0
  158. user_packages legacy/Try/BayesOptV8Codex.py +2731 -0
  159. user_packages legacy/Try/BayesOptV8Gemini.py +2614 -0
  160. user_packages legacy/Try/BayesOptV9.py +2927 -0
  161. user_packages legacy/Try/BayesOpt_entry legacy.py +313 -0
  162. user_packages legacy/Try/ModelBayesOptSearch.py +359 -0
  163. user_packages legacy/Try/ResNetBayesOptSearch.py +249 -0
  164. user_packages legacy/Try/XgbBayesOptSearch.py +121 -0
  165. user_packages legacy/Try/xgbbayesopt.py +523 -0
  166. user_packages legacy/__init__.py +19 -0
  167. user_packages legacy/cli_common.py +124 -0
  168. user_packages legacy/notebook_utils.py +228 -0
  169. user_packages legacy/watchdog_run.py +202 -0
@@ -0,0 +1,568 @@
1
+ """
2
+ CLI entry point generated from BayesOpt_AutoPricing.ipynb so the workflow can
3
+ run non‑interactively (e.g., via torchrun).
4
+
5
+ Example:
6
+ python -m torch.distributed.run --standalone --nproc_per_node=2 \\
7
+ ins_pricing/scripts/train.py \\
8
+ --config-json ins_pricing/modelling/demo/config_template.json \\
9
+ --model-keys ft --max-evals 50 --use-ft-ddp
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import argparse
15
+ import os
16
+ from pathlib import Path
17
+ from typing import Dict, List
18
+
19
+ import pandas as pd
20
+ from sklearn.model_selection import train_test_split
21
+
22
+ try:
23
+ import ins_pricing.modelling as ropt
24
+ from ins_pricing.modelling.cli_common import (
25
+ PLOT_MODEL_LABELS,
26
+ PYTORCH_TRAINERS,
27
+ build_model_names,
28
+ dedupe_preserve_order,
29
+ load_config_json,
30
+ normalize_config_paths,
31
+ parse_model_pairs,
32
+ resolve_config_path,
33
+ resolve_path,
34
+ set_env,
35
+ )
36
+ except ImportError:
37
+ # Fallback for flat layout without installation
38
+ import sys
39
+ _pkg_root = Path(__file__).resolve().parent.parent
40
+ if str(_pkg_root) not in sys.path:
41
+ sys.path.insert(0, str(_pkg_root))
42
+
43
+ # Try importing directly from modelling folder (flat layout-ish but nested)
44
+ try:
45
+ from modelling import core as ropt
46
+ from modelling.cli_common import (
47
+ PLOT_MODEL_LABELS,
48
+ PYTORCH_TRAINERS,
49
+ build_model_names,
50
+ dedupe_preserve_order,
51
+ load_config_json,
52
+ normalize_config_paths,
53
+ parse_model_pairs,
54
+ resolve_config_path,
55
+ resolve_path,
56
+ set_env,
57
+ )
58
+ except ImportError:
59
+ # Last resort: try ins_pricing namespace again
60
+ import ins_pricing.modelling as ropt
61
+ from ins_pricing.cli_common import (
62
+ PLOT_MODEL_LABELS,
63
+ PYTORCH_TRAINERS,
64
+ build_model_names,
65
+ dedupe_preserve_order,
66
+ load_config_json,
67
+ normalize_config_paths,
68
+ parse_model_pairs,
69
+ resolve_config_path,
70
+ resolve_path,
71
+ set_env,
72
+ )
73
+
74
+ import matplotlib
75
+
76
+ if os.name != "nt" and not os.environ.get("DISPLAY") and not os.environ.get("MPLBACKEND"):
77
+ matplotlib.use("Agg")
78
+ import matplotlib.pyplot as plt
79
+
80
+ try:
81
+ from ins_pricing.run_logging import configure_run_logging
82
+ except Exception:
83
+ try:
84
+ from run_logging import configure_run_logging
85
+ except Exception:
86
+ configure_run_logging = None
87
+
88
+ try:
89
+ from ins_pricing.plotting.diagnostics import plot_loss_curve as plot_loss_curve_common
90
+ except Exception:
91
+ try:
92
+ from plotting.diagnostics import plot_loss_curve as plot_loss_curve_common
93
+ except Exception:
94
+ plot_loss_curve_common = None
95
+
96
+ def _parse_args() -> argparse.Namespace:
97
+ parser = argparse.ArgumentParser(
98
+ description="Batch trainer generated from BayesOpt_AutoPricing notebook."
99
+ )
100
+ parser.add_argument(
101
+ "--config-json",
102
+ required=True,
103
+ help="Path to the JSON config describing datasets and feature columns.",
104
+ )
105
+ parser.add_argument(
106
+ "--model-keys",
107
+ nargs="+",
108
+ default=["ft"],
109
+ choices=["glm", "xgb", "resn", "ft", "gnn", "all"],
110
+ help="Space-separated list of trainers to run (e.g., --model-keys glm xgb). Include 'all' to run every trainer.",
111
+ )
112
+ parser.add_argument(
113
+ "--stack-model-keys",
114
+ nargs="+",
115
+ default=None,
116
+ choices=["glm", "xgb", "resn", "ft", "gnn", "all"],
117
+ help=(
118
+ "Only used when ft_role != 'model' (FT runs as feature generator). "
119
+ "When provided (or when config defines stack_model_keys), these trainers run after FT features "
120
+ "are generated. Use 'all' to run every non-FT trainer."
121
+ ),
122
+ )
123
+ parser.add_argument(
124
+ "--max-evals",
125
+ type=int,
126
+ default=50,
127
+ help="Optuna trial count per dataset.",
128
+ )
129
+ parser.add_argument(
130
+ "--use-resn-ddp",
131
+ action="store_true",
132
+ help="Force ResNet trainer to use DistributedDataParallel.",
133
+ )
134
+ parser.add_argument(
135
+ "--use-ft-ddp",
136
+ action="store_true",
137
+ help="Force FT-Transformer trainer to use DistributedDataParallel.",
138
+ )
139
+ parser.add_argument(
140
+ "--use-resn-dp",
141
+ action="store_true",
142
+ help="Enable ResNet DataParallel fall-back regardless of config.",
143
+ )
144
+ parser.add_argument(
145
+ "--use-ft-dp",
146
+ action="store_true",
147
+ help="Enable FT-Transformer DataParallel fall-back regardless of config.",
148
+ )
149
+ parser.add_argument(
150
+ "--use-gnn-dp",
151
+ action="store_true",
152
+ help="Enable GNN DataParallel fall-back regardless of config.",
153
+ )
154
+ parser.add_argument(
155
+ "--use-gnn-ddp",
156
+ action="store_true",
157
+ help="Force GNN trainer to use DistributedDataParallel.",
158
+ )
159
+ parser.add_argument(
160
+ "--gnn-no-ann",
161
+ action="store_true",
162
+ help="Disable approximate k-NN for GNN graph construction and use exact search.",
163
+ )
164
+ parser.add_argument(
165
+ "--gnn-ann-threshold",
166
+ type=int,
167
+ default=None,
168
+ help="Row threshold above which approximate k-NN is preferred (overrides config).",
169
+ )
170
+ parser.add_argument(
171
+ "--gnn-graph-cache",
172
+ default=None,
173
+ help="Optional path to persist/load cached adjacency matrix for GNN.",
174
+ )
175
+ parser.add_argument(
176
+ "--gnn-max-gpu-nodes",
177
+ type=int,
178
+ default=None,
179
+ help="Overrides the maximum node count allowed for GPU k-NN graph construction.",
180
+ )
181
+ parser.add_argument(
182
+ "--gnn-gpu-mem-ratio",
183
+ type=float,
184
+ default=None,
185
+ help="Overrides the fraction of free GPU memory the k-NN builder may consume.",
186
+ )
187
+ parser.add_argument(
188
+ "--gnn-gpu-mem-overhead",
189
+ type=float,
190
+ default=None,
191
+ help="Overrides the temporary GPU memory overhead multiplier for k-NN estimation.",
192
+ )
193
+ parser.add_argument(
194
+ "--output-dir",
195
+ default=None,
196
+ help="Override output root for models/results/plots.",
197
+ )
198
+ parser.add_argument(
199
+ "--plot-curves",
200
+ action="store_true",
201
+ help="Enable lift/diagnostic plots after training (config file may also request plotting).",
202
+ )
203
+ parser.add_argument(
204
+ "--ft-as-feature",
205
+ action="store_true",
206
+ help="Alias for --ft-role embedding (keep tuning, export embeddings; skip FT plots/SHAP).",
207
+ )
208
+ parser.add_argument(
209
+ "--ft-role",
210
+ default=None,
211
+ choices=["model", "embedding", "unsupervised_embedding"],
212
+ help="How to use FT: model (default), embedding (export pooling embeddings), or unsupervised_embedding.",
213
+ )
214
+ parser.add_argument(
215
+ "--ft-feature-prefix",
216
+ default="ft_feat",
217
+ help="Prefix used for generated FT features (columns: pred_<prefix>_0.. or pred_<prefix>).",
218
+ )
219
+ parser.add_argument(
220
+ "--reuse-best-params",
221
+ action="store_true",
222
+ help="Skip Optuna and reuse best_params saved in Results/versions or bestparams CSV when available.",
223
+ )
224
+ return parser.parse_args()
225
+
226
+
227
+ def _plot_curves_for_model(model: ropt.BayesOptModel, trained_keys: List[str], cfg: Dict) -> None:
228
+ plot_cfg = cfg.get("plot", {})
229
+ legacy_lift_flags = {
230
+ "glm": cfg.get("plot_lift_glm", False),
231
+ "xgb": cfg.get("plot_lift_xgb", False),
232
+ "resn": cfg.get("plot_lift_resn", False),
233
+ "ft": cfg.get("plot_lift_ft", False),
234
+ }
235
+ plot_enabled = plot_cfg.get("enable", any(legacy_lift_flags.values()))
236
+ if not plot_enabled:
237
+ return
238
+
239
+ n_bins = int(plot_cfg.get("n_bins", 10))
240
+ oneway_enabled = plot_cfg.get("oneway", True)
241
+
242
+ available_models = dedupe_preserve_order(
243
+ [m for m in trained_keys if m in PLOT_MODEL_LABELS]
244
+ )
245
+
246
+ if oneway_enabled:
247
+ model.plot_oneway(n_bins=n_bins)
248
+
249
+ if not available_models:
250
+ return
251
+
252
+ lift_models = plot_cfg.get("lift_models")
253
+ if lift_models is None:
254
+ lift_models = [
255
+ m for m, enabled in legacy_lift_flags.items() if enabled]
256
+ if not lift_models:
257
+ lift_models = available_models
258
+ lift_models = dedupe_preserve_order(
259
+ [m for m in lift_models if m in available_models]
260
+ )
261
+
262
+ for model_key in lift_models:
263
+ label, pred_nme = PLOT_MODEL_LABELS[model_key]
264
+ model.plot_lift(model_label=label, pred_nme=pred_nme, n_bins=n_bins)
265
+
266
+ if not plot_cfg.get("double_lift", True) or len(available_models) < 2:
267
+ return
268
+
269
+ raw_pairs = plot_cfg.get("double_lift_pairs")
270
+ if raw_pairs:
271
+ pairs = [
272
+ (a, b)
273
+ for a, b in parse_model_pairs(raw_pairs)
274
+ if a in available_models and b in available_models and a != b
275
+ ]
276
+ else:
277
+ pairs = [(a, b) for i, a in enumerate(available_models) for b in available_models[i + 1 :]]
278
+
279
+ for first, second in pairs:
280
+ model.plot_dlift([first, second], n_bins=n_bins)
281
+
282
+
283
+ def _plot_loss_curve_for_trainer(model_name: str, trainer) -> None:
284
+ model_obj = getattr(trainer, "model", None)
285
+ history = None
286
+ if model_obj is not None:
287
+ history = getattr(model_obj, "training_history", None)
288
+ if not history:
289
+ history = getattr(trainer, "training_history", None)
290
+ if not history:
291
+ return
292
+ train_hist = list(history.get("train") or [])
293
+ val_hist = list(history.get("val") or [])
294
+ if not train_hist and not val_hist:
295
+ return
296
+ try:
297
+ plot_dir = trainer.output.plot_path(
298
+ f"loss_{model_name}_{trainer.model_name_prefix}.png"
299
+ )
300
+ except Exception:
301
+ default_dir = Path("plot")
302
+ default_dir.mkdir(parents=True, exist_ok=True)
303
+ plot_dir = str(
304
+ default_dir / f"loss_{model_name}_{trainer.model_name_prefix}.png")
305
+ if plot_loss_curve_common is not None:
306
+ plot_loss_curve_common(
307
+ history=history,
308
+ title=f"{trainer.model_name_prefix} Loss Curve ({model_name})",
309
+ save_path=plot_dir,
310
+ show=False,
311
+ )
312
+ else:
313
+ epochs = range(1, max(len(train_hist), len(val_hist)) + 1)
314
+ fig, ax = plt.subplots(figsize=(8, 4))
315
+ if train_hist:
316
+ ax.plot(range(1, len(train_hist) + 1),
317
+ train_hist, label="Train Loss", color="tab:blue")
318
+ if val_hist:
319
+ ax.plot(range(1, len(val_hist) + 1),
320
+ val_hist, label="Validation Loss", color="tab:orange")
321
+ ax.set_xlabel("Epoch")
322
+ ax.set_ylabel("Weighted Loss")
323
+ ax.set_title(
324
+ f"{trainer.model_name_prefix} Loss Curve ({model_name})")
325
+ ax.grid(True, linestyle="--", alpha=0.3)
326
+ ax.legend()
327
+ plt.tight_layout()
328
+ plt.savefig(plot_dir, dpi=300)
329
+ plt.close(fig)
330
+ print(
331
+ f"[Plot] Saved loss curve for {model_name}/{trainer.label} -> {plot_dir}")
332
+
333
+
334
+ def train_from_config(args: argparse.Namespace) -> None:
335
+ script_dir = Path(__file__).resolve().parent
336
+ config_path = resolve_config_path(args.config_json, script_dir)
337
+ cfg = load_config_json(
338
+ config_path,
339
+ required_keys=["data_dir", "model_list", "model_categories", "target", "weight"],
340
+ )
341
+ cfg = normalize_config_paths(cfg, config_path)
342
+
343
+ set_env(cfg.get("env", {}))
344
+ plot_requested = bool(args.plot_curves or cfg.get("plot_curves", False))
345
+
346
+ def _safe_int_env(key: str, default: int) -> int:
347
+ try:
348
+ return int(os.environ.get(key, default))
349
+ except (TypeError, ValueError):
350
+ return default
351
+
352
+ dist_world_size = _safe_int_env("WORLD_SIZE", 1)
353
+ dist_rank = _safe_int_env("RANK", 0)
354
+ dist_active = dist_world_size > 1
355
+
356
+ data_dir = Path(cfg["data_dir"])
357
+ data_dir.mkdir(parents=True, exist_ok=True)
358
+
359
+ prop_test = cfg.get("prop_test", 0.25)
360
+ rand_seed = cfg.get("rand_seed", 13)
361
+ epochs = cfg.get("epochs", 50)
362
+ output_dir = args.output_dir or cfg.get("output_dir")
363
+ if isinstance(output_dir, str) and output_dir.strip():
364
+ resolved = resolve_path(output_dir, config_path.parent)
365
+ if resolved is not None:
366
+ output_dir = str(resolved)
367
+ reuse_best_params = bool(args.reuse_best_params or cfg.get("reuse_best_params", False))
368
+ xgb_max_depth_max = int(cfg.get("xgb_max_depth_max", 25))
369
+ xgb_n_estimators_max = int(cfg.get("xgb_n_estimators_max", 500))
370
+ optuna_storage = cfg.get("optuna_storage")
371
+ optuna_study_prefix = cfg.get("optuna_study_prefix")
372
+ best_params_files = cfg.get("best_params_files")
373
+
374
+ model_names = build_model_names(
375
+ cfg["model_list"], cfg["model_categories"])
376
+ if not model_names:
377
+ raise ValueError(
378
+ "No model names generated from model_list/model_categories.")
379
+
380
+ results: Dict[str, ropt.BayesOptModel] = {}
381
+ trained_keys_by_model: Dict[str, List[str]] = {}
382
+
383
+ for model_name in model_names:
384
+ # Per-dataset training loop: load data, split train/test, and train requested models.
385
+ csv_path = data_dir / f"{model_name}.csv"
386
+ if not csv_path.exists():
387
+ raise FileNotFoundError(f"Missing dataset: {csv_path}")
388
+
389
+ print(f"\n=== Processing model {model_name} ===")
390
+ raw = pd.read_csv(csv_path, low_memory=False)
391
+ raw = raw.copy()
392
+ for col in raw.columns:
393
+ s = raw[col]
394
+ if pd.api.types.is_numeric_dtype(s):
395
+ raw[col] = pd.to_numeric(s, errors="coerce").fillna(0)
396
+ else:
397
+ raw[col] = s.astype("object").fillna("<NA>")
398
+
399
+ train_df, test_df = train_test_split(
400
+ raw, test_size=prop_test, random_state=rand_seed
401
+ )
402
+
403
+ use_resn_dp = args.use_resn_dp or cfg.get(
404
+ "use_resn_data_parallel", False)
405
+ use_ft_dp = args.use_ft_dp or cfg.get("use_ft_data_parallel", True)
406
+ use_resn_ddp = args.use_resn_ddp or cfg.get("use_resn_ddp", False)
407
+ use_ft_ddp = args.use_ft_ddp or cfg.get("use_ft_ddp", False)
408
+ use_gnn_dp = args.use_gnn_dp or cfg.get("use_gnn_data_parallel", False)
409
+ use_gnn_ddp = args.use_gnn_ddp or cfg.get("use_gnn_ddp", False)
410
+ gnn_use_ann = cfg.get("gnn_use_approx_knn", True)
411
+ if args.gnn_no_ann:
412
+ gnn_use_ann = False
413
+ gnn_threshold = args.gnn_ann_threshold if args.gnn_ann_threshold is not None else cfg.get(
414
+ "gnn_approx_knn_threshold", 50000)
415
+ gnn_graph_cache = args.gnn_graph_cache or cfg.get("gnn_graph_cache")
416
+ if isinstance(gnn_graph_cache, str) and gnn_graph_cache.strip():
417
+ resolved_cache = resolve_path(gnn_graph_cache, config_path.parent)
418
+ if resolved_cache is not None:
419
+ gnn_graph_cache = str(resolved_cache)
420
+ gnn_max_gpu_nodes = args.gnn_max_gpu_nodes if args.gnn_max_gpu_nodes is not None else cfg.get(
421
+ "gnn_max_gpu_knn_nodes", 200000)
422
+ gnn_gpu_mem_ratio = args.gnn_gpu_mem_ratio if args.gnn_gpu_mem_ratio is not None else cfg.get(
423
+ "gnn_knn_gpu_mem_ratio", 0.9)
424
+ gnn_gpu_mem_overhead = args.gnn_gpu_mem_overhead if args.gnn_gpu_mem_overhead is not None else cfg.get(
425
+ "gnn_knn_gpu_mem_overhead", 2.0)
426
+
427
+ binary_target = cfg.get("binary_target") or cfg.get("binary_resp_nme")
428
+ feature_list = cfg.get("feature_list")
429
+ categorical_features = cfg.get("categorical_features")
430
+
431
+ ft_role = args.ft_role or cfg.get("ft_role", "model")
432
+ if args.ft_as_feature and args.ft_role is None:
433
+ # Keep legacy behavior as a convenience alias only when the config
434
+ # didn't already request a non-default FT role.
435
+ if str(cfg.get("ft_role", "model")) == "model":
436
+ ft_role = "embedding"
437
+ ft_feature_prefix = str(cfg.get("ft_feature_prefix", args.ft_feature_prefix))
438
+ ft_num_numeric_tokens = cfg.get("ft_num_numeric_tokens")
439
+
440
+ model = ropt.BayesOptModel(
441
+ train_df,
442
+ test_df,
443
+ model_name,
444
+ cfg["target"],
445
+ cfg["weight"],
446
+ feature_list,
447
+ binary_resp_nme=binary_target,
448
+ cate_list=categorical_features,
449
+ prop_test=prop_test,
450
+ rand_seed=rand_seed,
451
+ epochs=epochs,
452
+ use_resn_data_parallel=use_resn_dp,
453
+ use_ft_data_parallel=use_ft_dp,
454
+ use_resn_ddp=use_resn_ddp,
455
+ use_ft_ddp=use_ft_ddp,
456
+ use_gnn_data_parallel=use_gnn_dp,
457
+ use_gnn_ddp=use_gnn_ddp,
458
+ output_dir=output_dir,
459
+ xgb_max_depth_max=xgb_max_depth_max,
460
+ xgb_n_estimators_max=xgb_n_estimators_max,
461
+ resn_weight_decay=cfg.get("resn_weight_decay"),
462
+ final_ensemble=bool(cfg.get("final_ensemble", False)),
463
+ final_ensemble_k=int(cfg.get("final_ensemble_k", 3)),
464
+ final_refit=bool(cfg.get("final_refit", True)),
465
+ optuna_storage=optuna_storage,
466
+ optuna_study_prefix=optuna_study_prefix,
467
+ best_params_files=best_params_files,
468
+ gnn_use_approx_knn=gnn_use_ann,
469
+ gnn_approx_knn_threshold=gnn_threshold,
470
+ gnn_graph_cache=gnn_graph_cache,
471
+ gnn_max_gpu_knn_nodes=gnn_max_gpu_nodes,
472
+ gnn_knn_gpu_mem_ratio=gnn_gpu_mem_ratio,
473
+ gnn_knn_gpu_mem_overhead=gnn_gpu_mem_overhead,
474
+ ft_role=ft_role,
475
+ ft_feature_prefix=ft_feature_prefix,
476
+ ft_num_numeric_tokens=ft_num_numeric_tokens,
477
+ infer_categorical_max_unique=int(cfg.get("infer_categorical_max_unique", 50)),
478
+ infer_categorical_max_ratio=float(cfg.get("infer_categorical_max_ratio", 0.05)),
479
+ reuse_best_params=reuse_best_params,
480
+ )
481
+
482
+ if "all" in args.model_keys:
483
+ requested_keys = ["glm", "xgb", "resn", "ft", "gnn"]
484
+ else:
485
+ requested_keys = args.model_keys
486
+ requested_keys = dedupe_preserve_order(requested_keys)
487
+
488
+ if ft_role != "model":
489
+ requested_keys = [k for k in requested_keys if k != "ft"]
490
+ if not requested_keys:
491
+ stack_keys = args.stack_model_keys or cfg.get("stack_model_keys")
492
+ if stack_keys:
493
+ if "all" in stack_keys:
494
+ requested_keys = ["glm", "xgb", "resn", "gnn"]
495
+ else:
496
+ requested_keys = [k for k in stack_keys if k != "ft"]
497
+ requested_keys = dedupe_preserve_order(requested_keys)
498
+ if dist_active:
499
+ ft_trainer = model.trainers.get("ft")
500
+ if ft_trainer is None:
501
+ raise ValueError("FT trainer is not available.")
502
+ ft_trainer_uses_ddp = bool(
503
+ getattr(ft_trainer, "enable_distributed_optuna", False))
504
+ if not ft_trainer_uses_ddp:
505
+ raise ValueError(
506
+ "FT embedding under torchrun requires enabling FT DDP (use --use-ft-ddp or set use_ft_ddp=true)."
507
+ )
508
+ missing = [key for key in requested_keys if key not in model.trainers]
509
+ if missing:
510
+ raise ValueError(
511
+ f"Trainer(s) {missing} not available for {model_name}")
512
+
513
+ executed_keys: List[str] = []
514
+ if ft_role != "model":
515
+ print(
516
+ f"Optimizing ft as {ft_role} for {model_name} (max_evals={args.max_evals})")
517
+ model.optimize_model("ft", max_evals=args.max_evals)
518
+ model.trainers["ft"].save()
519
+ if getattr(ropt, "torch", None) is not None and ropt.torch.cuda.is_available():
520
+ ropt.free_cuda()
521
+ for key in requested_keys:
522
+ trainer = model.trainers[key]
523
+ trainer_uses_ddp = bool(
524
+ getattr(trainer, "enable_distributed_optuna", False))
525
+ should_run = True
526
+ if dist_active and not trainer_uses_ddp:
527
+ should_run = dist_rank == 0
528
+ if not should_run:
529
+ print(
530
+ f"[Rank {dist_rank}] Skip {model_name}/{key} because trainer is not DDP-enabled."
531
+ )
532
+ continue
533
+
534
+ print(
535
+ f"Optimizing {key} for {model_name} (max_evals={args.max_evals})")
536
+ model.optimize_model(key, max_evals=args.max_evals)
537
+ model.trainers[key].save()
538
+ _plot_loss_curve_for_trainer(model_name, model.trainers[key])
539
+ if key in PYTORCH_TRAINERS:
540
+ ropt.free_cuda()
541
+ executed_keys.append(key)
542
+
543
+ if not executed_keys:
544
+ continue
545
+
546
+ results[model_name] = model
547
+ trained_keys_by_model[model_name] = executed_keys
548
+
549
+ if not plot_requested:
550
+ return
551
+
552
+ for name, model in results.items():
553
+ _plot_curves_for_model(
554
+ model,
555
+ trained_keys_by_model.get(name, []),
556
+ cfg,
557
+ )
558
+
559
+
560
+ def main() -> None:
561
+ if configure_run_logging:
562
+ configure_run_logging(prefix="bayesopt_entry")
563
+ args = _parse_args()
564
+ train_from_config(args)
565
+
566
+
567
+ if __name__ == "__main__":
568
+ main()
@@ -0,0 +1,55 @@
1
+ from setuptools import setup, find_packages
2
+ import os
3
+
4
+ # Custom package discovery to map root folders to ins_pricing namespace
5
+ # Because the root folder "ins_pricing v2" has a space and handled as "."
6
+ def start_package_discovery():
7
+ # We want 'modelling' -> 'ins_pricing.modelling'
8
+ # 'pricing' -> 'ins_pricing.pricing'
9
+ # etc.
10
+ # And '.' -> 'ins_pricing'
11
+
12
+ root_packages = ['modelling', 'pricing', 'production', 'governance', 'reporting', 'scripts']
13
+ packages = ['ins_pricing']
14
+
15
+ for root in root_packages:
16
+ # Find subpackages for each root package
17
+ found = find_packages(where='.', include=[root, f"{root}.*"])
18
+ for pkg in found:
19
+ packages.append(f"ins_pricing.{pkg}")
20
+
21
+ return packages
22
+
23
+ setup(
24
+ name="ins_pricing",
25
+ version="2.0.0",
26
+ description="Insurance Pricing Modelling Toolbox",
27
+ # Map 'ins_pricing' package to current directory '.'
28
+ package_dir={'ins_pricing': '.'},
29
+ packages=start_package_discovery(),
30
+ python_requires=">=3.8",
31
+ install_requires=[
32
+ "numpy",
33
+ "pandas",
34
+ "scikit-learn",
35
+ "statsmodels",
36
+ "pydantic", # Required for data validation
37
+ ],
38
+ extras_require={
39
+ "full": [
40
+ "torch",
41
+ "xgboost",
42
+ "optuna",
43
+ "shap",
44
+ "matplotlib",
45
+ ]
46
+ },
47
+ entry_points={
48
+ "console_scripts": [
49
+ "ins-pricing-train=ins_pricing.scripts.train:main",
50
+ "ins-pricing-incremental=ins_pricing.scripts.BayesOpt_incremental:main",
51
+ "ins-pricing-explain=ins_pricing.scripts.Explain_entry:main",
52
+ ]
53
+ },
54
+ include_package_data=True,
55
+ )
@@ -0,0 +1,28 @@
1
+
2
+ import sys
3
+ import os
4
+ from pathlib import Path
5
+
6
+ # Add current dir to sys.path to simulate running from root without install (if needed)
7
+ sys.path.insert(0, os.getcwd())
8
+
9
+ print("Testing direct import from modelling folder (simulated root usage)...")
10
+ try:
11
+ from modelling.core import BayesOptModel
12
+ print("SUCCESS: from modelling.core import BayesOptModel")
13
+ except ImportError as e:
14
+ print(f"FAILURE: from modelling.core import BayesOptModel: {e}")
15
+
16
+ print("\nTesting ins_pricing.modelling namespace import (requires install or setup.py magic)...")
17
+ try:
18
+ from ins_pricing.modelling.core import BayesOptModel
19
+ print("SUCCESS: from ins_pricing.modelling.core import BayesOptModel")
20
+ except ImportError as e:
21
+ print(f"FAILURE: from ins_pricing.modelling.core import BayesOptModel: {e}")
22
+
23
+ print("\nTesting plotting import...")
24
+ try:
25
+ from modelling.plotting import curves
26
+ print("SUCCESS: from modelling.plotting import curves")
27
+ except ImportError as e:
28
+ print(f"FAILURE: from modelling.plotting import curves: {e}")