ins-pricing 0.1.11__py3-none-any.whl → 0.2.0__py3-none-any.whl

ins_pricing/{modelling → cli}/Explain_entry.py

@@ -2,55 +2,97 @@
 
 from __future__ import annotations
 
+from pathlib import Path
+import sys
+
+if __package__ in {None, ""}:
+    repo_root = Path(__file__).resolve().parents[2]
+    if str(repo_root) not in sys.path:
+        sys.path.insert(0, str(repo_root))
+
 import argparse
 import json
-from pathlib import Path
 from typing import Any, Dict, List, Optional, Sequence
 
 import numpy as np
 import pandas as pd
-from sklearn.model_selection import train_test_split
 
 try:
-    from . import bayesopt as ropt  # type: ignore
-    from .cli_common import (  # type: ignore
+    from .. import bayesopt as ropt  # type: ignore
+    from .utils.cli_common import (  # type: ignore
         build_model_names,
         dedupe_preserve_order,
-        load_config_json,
-        normalize_config_paths,
-        resolve_config_path,
-        resolve_path,
-        set_env,
+        load_dataset,
+        resolve_data_path,
+        coerce_dataset_types,
+        split_train_test,
+    )
+    from .utils.cli_config import (  # type: ignore
+        add_config_json_arg,
+        add_output_dir_arg,
+        resolve_and_load_config,
+        resolve_data_config,
+        resolve_explain_output_overrides,
+        resolve_explain_save_dir,
+        resolve_explain_save_root,
+        resolve_model_path_value,
+        resolve_split_config,
+        resolve_runtime_config,
+        resolve_output_dirs,
     )
 except Exception:  # pragma: no cover
     try:
         import bayesopt as ropt  # type: ignore
-        from cli_common import (  # type: ignore
+        from utils.cli_common import (  # type: ignore
             build_model_names,
             dedupe_preserve_order,
-            load_config_json,
-            normalize_config_paths,
-            resolve_config_path,
-            resolve_path,
-            set_env,
+            load_dataset,
+            resolve_data_path,
+            coerce_dataset_types,
+            split_train_test,
+        )
+        from utils.cli_config import (  # type: ignore
+            add_config_json_arg,
+            add_output_dir_arg,
+            resolve_and_load_config,
+            resolve_data_config,
+            resolve_explain_output_overrides,
+            resolve_explain_save_dir,
+            resolve_explain_save_root,
+            resolve_model_path_value,
+            resolve_split_config,
+            resolve_runtime_config,
+            resolve_output_dirs,
         )
     except Exception:
-        import ins_pricing.bayesopt as ropt  # type: ignore
-        from ins_pricing.cli_common import (  # type: ignore
+        import ins_pricing.modelling.core.bayesopt as ropt  # type: ignore
+        from ins_pricing.cli.utils.cli_common import (  # type: ignore
             build_model_names,
             dedupe_preserve_order,
-            load_config_json,
-            normalize_config_paths,
-            resolve_config_path,
-            resolve_path,
-            set_env,
+            load_dataset,
+            resolve_data_path,
+            coerce_dataset_types,
+            split_train_test,
+        )
+        from ins_pricing.cli.utils.cli_config import (  # type: ignore
+            add_config_json_arg,
+            add_output_dir_arg,
+            resolve_and_load_config,
+            resolve_data_config,
+            resolve_explain_output_overrides,
+            resolve_explain_save_dir,
+            resolve_explain_save_root,
+            resolve_model_path_value,
+            resolve_split_config,
+            resolve_runtime_config,
+            resolve_output_dirs,
         )
 
 try:
-    from .run_logging import configure_run_logging  # type: ignore
+    from .utils.run_logging import configure_run_logging  # type: ignore
 except Exception:  # pragma: no cover
     try:
-        from run_logging import configure_run_logging  # type: ignore
+        from utils.run_logging import configure_run_logging  # type: ignore
     except Exception:  # pragma: no cover
         configure_run_logging = None  # type: ignore
 
@@ -67,44 +109,19 @@ def _safe_name(value: str) -> str:
     return "".join(ch if ch.isalnum() or ch in "-_." else "_" for ch in str(value))
 
 
-def _load_dataset(path: Path) -> pd.DataFrame:
-    raw = pd.read_csv(path, low_memory=False)
-    raw = raw.copy()
-    for col in raw.columns:
-        s = raw[col]
-        if pd.api.types.is_numeric_dtype(s):
-            raw[col] = pd.to_numeric(s, errors="coerce").fillna(0)
-        else:
-            raw[col] = s.astype("object").fillna("<NA>")
-    return raw
-
-
-def _resolve_path_value(
-    value: Any,
+def _load_dataset(
+    path: Path,
     *,
-    model_name: str,
-    base_dir: Path,
-    data_dir: Optional[Path] = None,
-) -> Optional[Path]:
-    if value is None:
-        return None
-    if isinstance(value, dict):
-        value = value.get(model_name)
-    if value is None:
-        return None
-    path_str = str(value)
-    try:
-        path_str = path_str.format(model_name=model_name)
-    except Exception:
-        pass
-    if data_dir is not None and not Path(path_str).is_absolute():
-        candidate = data_dir / path_str
-        if candidate.exists():
-            return candidate.resolve()
-    resolved = resolve_path(path_str, base_dir)
-    if resolved is None:
-        return None
-    return resolved
+    data_format: str,
+    dtype_map: Optional[Dict[str, Any]],
+) -> pd.DataFrame:
+    raw = load_dataset(
+        path,
+        data_format=data_format,
+        dtype_map=dtype_map,
+        low_memory=False,
+    )
+    return coerce_dataset_types(raw)
 
 
 def _normalize_methods(raw: Sequence[str]) -> List[str]:
@@ -144,10 +161,9 @@ def _parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(
         description="Run explainability (permutation/SHAP/IG) on trained models."
     )
-    parser.add_argument(
-        "--config-json",
-        required=True,
-        help="Path to config.json (same schema as training).",
+    add_config_json_arg(
+        parser,
+        help_text="Path to config.json (same schema as training).",
     )
     parser.add_argument(
         "--model-keys",
@@ -162,10 +178,9 @@ def _parse_args() -> argparse.Namespace:
         default=None,
         help="Explain methods: permutation, shap, integrated_gradients (default from config.explain.methods).",
     )
-    parser.add_argument(
-        "--output-dir",
-        default=None,
-        help="Override output root for loading models/results.",
+    add_output_dir_arg(
+        parser,
+        help_text="Override output root for loading models/results.",
     )
     parser.add_argument(
         "--eval-path",
@@ -364,27 +379,34 @@ def _explain_for_model(
 
 
 def explain_from_config(args: argparse.Namespace) -> None:
-    script_dir = Path(__file__).resolve().parent
-    config_path = resolve_config_path(args.config_json, script_dir)
-    cfg = load_config_json(
-        config_path,
+    script_dir = Path(__file__).resolve().parents[1]
+    config_path, cfg = resolve_and_load_config(
+        args.config_json,
+        script_dir,
         required_keys=["data_dir", "model_list", "model_categories", "target", "weight"],
     )
-    cfg = normalize_config_paths(cfg, config_path)
-
-    set_env(cfg.get("env", {}))
 
-    data_dir = Path(cfg["data_dir"])
-    data_dir.mkdir(parents=True, exist_ok=True)
+    data_dir, data_format, data_path_template, dtype_map = resolve_data_config(
+        cfg,
+        config_path,
+        create_data_dir=True,
+    )
 
-    output_dir = args.output_dir or cfg.get("output_dir")
-    if isinstance(output_dir, str) and output_dir.strip():
-        resolved = resolve_path(output_dir, config_path.parent)
-        if resolved is not None:
-            output_dir = str(resolved)
+    runtime_cfg = resolve_runtime_config(cfg)
+    output_cfg = resolve_output_dirs(
+        cfg,
+        config_path,
+        output_override=args.output_dir,
+    )
+    output_dir = output_cfg["output_dir"]
 
-    prop_test = cfg.get("prop_test", 0.25)
-    rand_seed = cfg.get("rand_seed", 13)
+    split_cfg = resolve_split_config(cfg)
+    prop_test = split_cfg["prop_test"]
+    rand_seed = runtime_cfg["rand_seed"]
+    split_strategy = split_cfg["split_strategy"]
+    split_group_col = split_cfg["split_group_col"]
+    split_time_col = split_cfg["split_time_col"]
+    split_time_ascending = split_cfg["split_time_ascending"]
 
     explain_cfg = dict(cfg.get("explain") or {})
 
@@ -402,51 +424,72 @@ def explain_from_config(args: argparse.Namespace) -> None:
     if not model_names:
         raise ValueError("No model names generated from model_list/model_categories.")
 
-    save_dir_raw = args.save_dir or explain_cfg.get("save_dir")
-    if save_dir_raw:
-        resolved = resolve_path(str(save_dir_raw), config_path.parent)
-        save_root = resolved if resolved is not None else Path(str(save_dir_raw))
-    else:
-        save_root = None
+    save_root = resolve_explain_save_root(
+        args.save_dir or explain_cfg.get("save_dir"),
+        config_path.parent,
+    )
 
     for model_name in model_names:
-        train_path = _resolve_path_value(
+        train_path = resolve_model_path_value(
             explain_cfg.get("train_path"),
             model_name=model_name,
             base_dir=config_path.parent,
             data_dir=data_dir,
         )
         if train_path is None:
-            train_path = data_dir / f"{model_name}.csv"
+            train_path = resolve_data_path(
+                data_dir,
+                model_name,
+                data_format=data_format,
+                path_template=data_path_template,
+            )
         if not train_path.exists():
             raise FileNotFoundError(f"Missing training dataset: {train_path}")
 
         validation_override = args.eval_path or explain_cfg.get("validation_path") or explain_cfg.get("eval_path")
-        validation_path = _resolve_path_value(
+        validation_path = resolve_model_path_value(
             validation_override,
             model_name=model_name,
             base_dir=config_path.parent,
             data_dir=data_dir,
         )
 
-        raw = _load_dataset(train_path)
+        raw = _load_dataset(
+            train_path,
+            data_format=data_format,
+            dtype_map=dtype_map,
+        )
         if validation_path is not None:
             if not validation_path.exists():
                 raise FileNotFoundError(f"Missing validation dataset: {validation_path}")
             train_df = raw
-            test_df = _load_dataset(validation_path)
+            test_df = _load_dataset(
+                validation_path,
+                data_format=data_format,
+                dtype_map=dtype_map,
+            )
         else:
             if float(prop_test) <= 0:
                 train_df = raw
                 test_df = raw.copy()
             else:
-                train_df, test_df = train_test_split(
-                    raw, test_size=prop_test, random_state=rand_seed
+                train_df, test_df = split_train_test(
+                    raw,
+                    holdout_ratio=prop_test,
+                    strategy=split_strategy,
+                    group_col=split_group_col,
+                    time_col=split_time_col,
+                    time_ascending=split_time_ascending,
+                    rand_seed=rand_seed,
+                    reset_index_mode="time_group",
+                    ratio_label="prop_test",
+                    include_strategy_in_ratio_error=True,
                 )
 
         binary_target = cfg.get("binary_target") or cfg.get("binary_resp_nme")
         feature_list = cfg.get("feature_list")
         categorical_features = cfg.get("categorical_features")
+        plot_path_style = runtime_cfg["plot_path_style"]
 
         model = ropt.BayesOptModel(
             train_df,
@@ -455,65 +498,67 @@ def explain_from_config(args: argparse.Namespace) -> None:
             cfg["target"],
             cfg["weight"],
             feature_list,
+            task_type=str(cfg.get("task_type", "regression")),
             binary_resp_nme=binary_target,
             cate_list=categorical_features,
             prop_test=prop_test,
             rand_seed=rand_seed,
-            epochs=int(cfg.get("epochs", 50)),
+            epochs=int(runtime_cfg["epochs"]),
             use_gpu=bool(cfg.get("use_gpu", True)),
             output_dir=output_dir,
-            xgb_max_depth_max=int(cfg.get("xgb_max_depth_max", 25)),
-            xgb_n_estimators_max=int(cfg.get("xgb_n_estimators_max", 500)),
+            xgb_max_depth_max=runtime_cfg["xgb_max_depth_max"],
+            xgb_n_estimators_max=runtime_cfg["xgb_n_estimators_max"],
             resn_weight_decay=cfg.get("resn_weight_decay"),
             final_ensemble=bool(cfg.get("final_ensemble", False)),
             final_ensemble_k=int(cfg.get("final_ensemble_k", 3)),
             final_refit=bool(cfg.get("final_refit", True)),
-            optuna_storage=cfg.get("optuna_storage"),
-            optuna_study_prefix=cfg.get("optuna_study_prefix"),
-            best_params_files=cfg.get("best_params_files"),
+            optuna_storage=runtime_cfg["optuna_storage"],
+            optuna_study_prefix=runtime_cfg["optuna_study_prefix"],
+            best_params_files=runtime_cfg["best_params_files"],
             gnn_use_approx_knn=cfg.get("gnn_use_approx_knn", True),
             gnn_approx_knn_threshold=cfg.get("gnn_approx_knn_threshold", 50000),
             gnn_graph_cache=cfg.get("gnn_graph_cache"),
             gnn_max_gpu_knn_nodes=cfg.get("gnn_max_gpu_knn_nodes", 200000),
             gnn_knn_gpu_mem_ratio=cfg.get("gnn_knn_gpu_mem_ratio", 0.9),
             gnn_knn_gpu_mem_overhead=cfg.get("gnn_knn_gpu_mem_overhead", 2.0),
+            region_province_col=cfg.get("region_province_col"),
+            region_city_col=cfg.get("region_city_col"),
+            region_effect_alpha=cfg.get("region_effect_alpha"),
+            geo_feature_nmes=cfg.get("geo_feature_nmes"),
+            geo_token_hidden_dim=cfg.get("geo_token_hidden_dim"),
+            geo_token_layers=cfg.get("geo_token_layers"),
+            geo_token_dropout=cfg.get("geo_token_dropout"),
+            geo_token_k_neighbors=cfg.get("geo_token_k_neighbors"),
+            geo_token_learning_rate=cfg.get("geo_token_learning_rate"),
+            geo_token_epochs=cfg.get("geo_token_epochs"),
             ft_role=str(cfg.get("ft_role", "model")),
             ft_feature_prefix=str(cfg.get("ft_feature_prefix", "ft_emb")),
             ft_num_numeric_tokens=cfg.get("ft_num_numeric_tokens"),
             infer_categorical_max_unique=int(cfg.get("infer_categorical_max_unique", 50)),
             infer_categorical_max_ratio=float(cfg.get("infer_categorical_max_ratio", 0.05)),
-            reuse_best_params=bool(cfg.get("reuse_best_params", False)),
+            reuse_best_params=runtime_cfg["reuse_best_params"],
+            plot_path_style=plot_path_style,
         )
 
-        model_dir_override = _resolve_path_value(
-            explain_cfg.get("model_dir"),
+        output_overrides = resolve_explain_output_overrides(
+            explain_cfg,
             model_name=model_name,
             base_dir=config_path.parent,
-            data_dir=None,
         )
+        model_dir_override = output_overrides.get("model_dir")
         if model_dir_override is not None:
             model.output_manager.model_dir = model_dir_override
-        result_dir_override = _resolve_path_value(
-            explain_cfg.get("result_dir") or explain_cfg.get("results_dir"),
-            model_name=model_name,
-            base_dir=config_path.parent,
-            data_dir=None,
-        )
+        result_dir_override = output_overrides.get("result_dir")
         if result_dir_override is not None:
             model.output_manager.result_dir = result_dir_override
-        plot_dir_override = _resolve_path_value(
-            explain_cfg.get("plot_dir"),
-            model_name=model_name,
-            base_dir=config_path.parent,
-            data_dir=None,
-        )
+        plot_dir_override = output_overrides.get("plot_dir")
         if plot_dir_override is not None:
             model.output_manager.plot_dir = plot_dir_override
 
-        if save_root is None:
-            save_dir = Path(model.output_manager.result_dir) / "explain"
-        else:
-            save_dir = Path(save_root)
+        save_dir = resolve_explain_save_dir(
+            save_root,
+            result_dir=model.output_manager.result_dir,
+        )
         save_dir.mkdir(parents=True, exist_ok=True)
 
         print(f"\n=== Explain model {model_name} ===")

ins_pricing/cli/Pricing_Run.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+try:
+    from .utils.notebook_utils import run_from_config, run_from_config_cli  # type: ignore
+except Exception:  # pragma: no cover
+    from utils.notebook_utils import run_from_config, run_from_config_cli  # type: ignore
+
+
+def run(config_json: str | Path) -> None:
+    """Unified entry point: run entry/incremental/watchdog/DDP based on config.json runner."""
+    run_from_config(config_json)
+
+
+def main(argv: Optional[list[str]] = None) -> None:
+    run_from_config_cli(
+        "Pricing_Run: run BayesOpt by config.json (entry/incremental/watchdog/DDP).",
+        argv,
+    )
+
+
+if __name__ == "__main__":
+    main()

ins_pricing/cli/__init__.py

@@ -0,0 +1 @@
+"""CLI entry points for ins_pricing modelling."""