ins-pricing 0.4.5__py3-none-any.whl → 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ins_pricing/README.md +48 -22
- ins_pricing/__init__.py +142 -90
- ins_pricing/cli/BayesOpt_entry.py +52 -50
- ins_pricing/cli/BayesOpt_incremental.py +39 -105
- ins_pricing/cli/Explain_Run.py +31 -23
- ins_pricing/cli/Explain_entry.py +532 -579
- ins_pricing/cli/Pricing_Run.py +31 -23
- ins_pricing/cli/bayesopt_entry_runner.py +11 -9
- ins_pricing/cli/utils/cli_common.py +256 -256
- ins_pricing/cli/utils/cli_config.py +375 -375
- ins_pricing/cli/utils/import_resolver.py +382 -365
- ins_pricing/cli/utils/notebook_utils.py +340 -340
- ins_pricing/cli/watchdog_run.py +209 -201
- ins_pricing/frontend/__init__.py +10 -10
- ins_pricing/frontend/example_workflows.py +1 -1
- ins_pricing/governance/__init__.py +20 -20
- ins_pricing/governance/release.py +159 -159
- ins_pricing/modelling/__init__.py +147 -92
- ins_pricing/modelling/{core/bayesopt → bayesopt}/README.md +2 -2
- ins_pricing/modelling/{core/bayesopt → bayesopt}/__init__.py +64 -102
- ins_pricing/modelling/{core/bayesopt → bayesopt}/config_preprocess.py +562 -562
- ins_pricing/modelling/{core/bayesopt → bayesopt}/core.py +965 -964
- ins_pricing/modelling/{core/bayesopt → bayesopt}/model_explain_mixin.py +296 -296
- ins_pricing/modelling/{core/bayesopt → bayesopt}/model_plotting_mixin.py +482 -548
- ins_pricing/modelling/{core/bayesopt → bayesopt}/models/__init__.py +27 -27
- ins_pricing/modelling/{core/bayesopt → bayesopt}/models/model_ft_trainer.py +915 -913
- ins_pricing/modelling/{core/bayesopt → bayesopt}/models/model_gnn.py +788 -785
- ins_pricing/modelling/{core/bayesopt → bayesopt}/models/model_resn.py +448 -446
- ins_pricing/modelling/bayesopt/trainers/__init__.py +19 -0
- ins_pricing/modelling/{core/bayesopt → bayesopt}/trainers/trainer_base.py +1308 -1308
- ins_pricing/modelling/{core/bayesopt → bayesopt}/trainers/trainer_ft.py +3 -3
- ins_pricing/modelling/{core/bayesopt → bayesopt}/trainers/trainer_glm.py +197 -198
- ins_pricing/modelling/{core/bayesopt → bayesopt}/trainers/trainer_gnn.py +344 -344
- ins_pricing/modelling/{core/bayesopt → bayesopt}/trainers/trainer_resn.py +283 -283
- ins_pricing/modelling/{core/bayesopt → bayesopt}/trainers/trainer_xgb.py +346 -347
- ins_pricing/modelling/bayesopt/utils/__init__.py +67 -0
- ins_pricing/modelling/bayesopt/utils/constants.py +21 -0
- ins_pricing/modelling/bayesopt/utils/io_utils.py +7 -0
- ins_pricing/modelling/bayesopt/utils/losses.py +27 -0
- ins_pricing/modelling/bayesopt/utils/metrics_and_devices.py +17 -0
- ins_pricing/modelling/{core/bayesopt → bayesopt}/utils/torch_trainer_mixin.py +623 -623
- ins_pricing/modelling/{core/evaluation.py → evaluation.py} +113 -104
- ins_pricing/modelling/explain/__init__.py +55 -55
- ins_pricing/modelling/explain/metrics.py +27 -174
- ins_pricing/modelling/explain/permutation.py +237 -237
- ins_pricing/modelling/plotting/__init__.py +40 -36
- ins_pricing/modelling/plotting/compat.py +228 -0
- ins_pricing/modelling/plotting/curves.py +572 -572
- ins_pricing/modelling/plotting/diagnostics.py +163 -163
- ins_pricing/modelling/plotting/geo.py +362 -362
- ins_pricing/modelling/plotting/importance.py +121 -121
- ins_pricing/pricing/__init__.py +27 -27
- ins_pricing/production/__init__.py +35 -25
- ins_pricing/production/{predict.py → inference.py} +140 -57
- ins_pricing/production/monitoring.py +8 -21
- ins_pricing/reporting/__init__.py +11 -11
- ins_pricing/setup.py +1 -1
- ins_pricing/tests/production/test_inference.py +90 -0
- ins_pricing/utils/__init__.py +116 -83
- ins_pricing/utils/device.py +255 -255
- ins_pricing/utils/features.py +53 -0
- ins_pricing/utils/io.py +72 -0
- ins_pricing/{modelling/core/bayesopt/utils → utils}/losses.py +125 -129
- ins_pricing/utils/metrics.py +158 -24
- ins_pricing/utils/numerics.py +76 -0
- ins_pricing/utils/paths.py +9 -1
- {ins_pricing-0.4.5.dist-info → ins_pricing-0.5.0.dist-info}/METADATA +182 -182
- ins_pricing-0.5.0.dist-info/RECORD +131 -0
- ins_pricing/modelling/core/BayesOpt.py +0 -146
- ins_pricing/modelling/core/__init__.py +0 -1
- ins_pricing/modelling/core/bayesopt/trainers/__init__.py +0 -19
- ins_pricing/modelling/core/bayesopt/utils/__init__.py +0 -86
- ins_pricing/modelling/core/bayesopt/utils/constants.py +0 -183
- ins_pricing/modelling/core/bayesopt/utils/io_utils.py +0 -126
- ins_pricing/modelling/core/bayesopt/utils/metrics_and_devices.py +0 -555
- ins_pricing/modelling/core/bayesopt/utils.py +0 -105
- ins_pricing/modelling/core/bayesopt/utils_backup.py +0 -1503
- ins_pricing/tests/production/test_predict.py +0 -233
- ins_pricing-0.4.5.dist-info/RECORD +0 -130
- /ins_pricing/modelling/{core/bayesopt → bayesopt}/config_components.py +0 -0
- /ins_pricing/modelling/{core/bayesopt → bayesopt}/models/model_ft_components.py +0 -0
- /ins_pricing/modelling/{core/bayesopt → bayesopt}/utils/distributed_utils.py +0 -0
- {ins_pricing-0.4.5.dist-info → ins_pricing-0.5.0.dist-info}/WHEEL +0 -0
- {ins_pricing-0.4.5.dist-info → ins_pricing-0.5.0.dist-info}/top_level.txt +0 -0
ins_pricing/cli/watchdog_run.py
CHANGED
|
@@ -1,211 +1,219 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
3
|
import argparse
|
|
4
4
|
import os
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import importlib.util
|
|
5
7
|
import subprocess
|
|
6
8
|
import sys
|
|
7
9
|
import threading
|
|
8
10
|
import time
|
|
9
11
|
from typing import List, Optional
|
|
10
12
|
|
|
13
|
+
if __package__ in {None, ""}:
|
|
14
|
+
if importlib.util.find_spec("ins_pricing") is None:
|
|
15
|
+
repo_root = Path(__file__).resolve().parents[2]
|
|
16
|
+
if str(repo_root) not in sys.path:
|
|
17
|
+
sys.path.insert(0, str(repo_root))
|
|
18
|
+
|
|
11
19
|
try:
|
|
12
|
-
from .utils.run_logging import configure_run_logging # type: ignore
|
|
20
|
+
from ins_pricing.cli.utils.run_logging import configure_run_logging # type: ignore
|
|
13
21
|
except Exception: # pragma: no cover
|
|
14
|
-
try:
|
|
15
|
-
from utils.run_logging import configure_run_logging # type: ignore
|
|
16
|
-
except Exception: # pragma: no cover
|
|
17
|
-
configure_run_logging = None # type: ignore
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
def _split_argv(argv: List[str]) -> tuple[List[str], List[str]]:
|
|
21
|
-
if "--" not in argv:
|
|
22
|
-
raise ValueError("Missing '--' separator before the command to run.")
|
|
23
|
-
idx = argv.index("--")
|
|
24
|
-
return argv[:idx], argv[idx + 1 :]
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
def _kill_process_tree(pid: int) -> None:
|
|
28
|
-
if pid <= 0:
|
|
29
|
-
return
|
|
30
|
-
if os.name == "nt":
|
|
31
|
-
subprocess.run(
|
|
32
|
-
["taskkill", "/PID", str(pid), "/T", "/F"],
|
|
33
|
-
stdout=subprocess.DEVNULL,
|
|
34
|
-
stderr=subprocess.DEVNULL,
|
|
35
|
-
check=False,
|
|
36
|
-
)
|
|
37
|
-
return
|
|
38
|
-
try:
|
|
39
|
-
os.killpg(pid, 15)
|
|
40
|
-
time.sleep(2)
|
|
41
|
-
os.killpg(pid, 9)
|
|
42
|
-
except Exception:
|
|
43
|
-
try:
|
|
44
|
-
os.kill(pid, 9)
|
|
45
|
-
except Exception:
|
|
46
|
-
pass
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
def _reader_thread(
|
|
50
|
-
proc: subprocess.Popen, last_output_ts: dict, prefix: str = ""
|
|
51
|
-
) -> None:
|
|
52
|
-
assert proc.stdout is not None
|
|
53
|
-
for line in proc.stdout:
|
|
54
|
-
last_output_ts["ts"] = time.time()
|
|
55
|
-
if prefix:
|
|
56
|
-
sys.stdout.write(prefix)
|
|
57
|
-
sys.stdout.write(line)
|
|
58
|
-
sys.stdout.flush()
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
def _parse_args(before_cmd: List[str], cmd: List[str]) -> argparse.Namespace:
|
|
62
|
-
parser = argparse.ArgumentParser(
|
|
63
|
-
description=(
|
|
64
|
-
"Run a command under a simple watchdog: if there is no stdout/stderr "
|
|
65
|
-
"output for N seconds, kill the whole process tree and restart. "
|
|
66
|
-
"Designed to pair with optuna_storage so BayesOpt can resume."
|
|
67
|
-
)
|
|
68
|
-
)
|
|
69
|
-
parser.add_argument(
|
|
70
|
-
"--idle-seconds",
|
|
71
|
-
type=int,
|
|
72
|
-
default=7200,
|
|
73
|
-
help="Restart if there is no output for this many seconds (default: 7200).",
|
|
74
|
-
)
|
|
75
|
-
parser.add_argument(
|
|
76
|
-
"--max-restarts",
|
|
77
|
-
type=int,
|
|
78
|
-
default=50,
|
|
79
|
-
help="Maximum restart attempts (default: 50).",
|
|
80
|
-
)
|
|
81
|
-
parser.add_argument(
|
|
82
|
-
"--restart-delay-seconds",
|
|
83
|
-
type=int,
|
|
84
|
-
default=10,
|
|
85
|
-
help="Delay between restarts (default: 10).",
|
|
86
|
-
)
|
|
87
|
-
parser.add_argument(
|
|
88
|
-
"--stop-on-nonzero-exit",
|
|
89
|
-
action="store_true",
|
|
90
|
-
help="If the command exits non-zero, stop instead of restarting.",
|
|
91
|
-
)
|
|
92
|
-
args = parser.parse_args(before_cmd)
|
|
93
|
-
if not cmd:
|
|
94
|
-
parser.error("Empty command after '--'.")
|
|
95
|
-
return args
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
def run_with_watchdog(
|
|
99
|
-
cmd: List[str],
|
|
100
|
-
idle_seconds: int,
|
|
101
|
-
max_restarts: int,
|
|
102
|
-
restart_delay_seconds: int,
|
|
103
|
-
stop_on_nonzero_exit: bool,
|
|
104
|
-
) -> int:
|
|
105
|
-
idle_seconds = max(1, int(idle_seconds))
|
|
106
|
-
max_restarts = max(0, int(max_restarts))
|
|
107
|
-
restart_delay_seconds = max(0, int(restart_delay_seconds))
|
|
108
|
-
|
|
109
|
-
attempt = 0
|
|
110
|
-
while True:
|
|
111
|
-
attempt += 1
|
|
112
|
-
print(
|
|
113
|
-
f"[watchdog] start attempt={attempt} idle_seconds={idle_seconds} cmd={cmd}",
|
|
114
|
-
flush=True,
|
|
115
|
-
)
|
|
116
|
-
|
|
117
|
-
creationflags = 0
|
|
118
|
-
start_new_session = False
|
|
119
|
-
if os.name == "nt":
|
|
120
|
-
creationflags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)
|
|
121
|
-
else:
|
|
122
|
-
start_new_session = True
|
|
123
|
-
|
|
124
|
-
proc = subprocess.Popen(
|
|
125
|
-
cmd,
|
|
126
|
-
stdout=subprocess.PIPE,
|
|
127
|
-
stderr=subprocess.STDOUT,
|
|
128
|
-
text=True,
|
|
129
|
-
bufsize=1,
|
|
130
|
-
universal_newlines=True,
|
|
131
|
-
creationflags=creationflags,
|
|
132
|
-
start_new_session=start_new_session,
|
|
133
|
-
)
|
|
134
|
-
|
|
135
|
-
last_output_ts: dict = {"ts": time.time()}
|
|
136
|
-
reader = threading.Thread(
|
|
137
|
-
target=_reader_thread,
|
|
138
|
-
args=(proc, last_output_ts),
|
|
139
|
-
kwargs={"prefix": ""},
|
|
140
|
-
daemon=True,
|
|
141
|
-
)
|
|
142
|
-
reader.start()
|
|
143
|
-
|
|
144
|
-
killed_for_idle = False
|
|
145
|
-
exit_code: Optional[int] = None
|
|
146
|
-
while True:
|
|
147
|
-
exit_code = proc.poll()
|
|
148
|
-
if exit_code is not None:
|
|
149
|
-
break
|
|
150
|
-
idle_for = time.time() - float(last_output_ts["ts"])
|
|
151
|
-
if idle_for > idle_seconds:
|
|
152
|
-
killed_for_idle = True
|
|
153
|
-
print(
|
|
154
|
-
f"[watchdog] idle>{idle_seconds}s (idle_for={int(idle_for)}s), killing pid={proc.pid}",
|
|
155
|
-
flush=True,
|
|
156
|
-
)
|
|
157
|
-
_kill_process_tree(proc.pid)
|
|
158
|
-
break
|
|
159
|
-
time.sleep(5)
|
|
160
|
-
|
|
161
|
-
try:
|
|
162
|
-
proc.wait(timeout=30)
|
|
163
|
-
except Exception:
|
|
164
|
-
_kill_process_tree(proc.pid)
|
|
165
|
-
|
|
166
|
-
if exit_code is None:
|
|
167
|
-
exit_code = proc.poll() or 1
|
|
168
|
-
|
|
169
|
-
if exit_code == 0:
|
|
170
|
-
print("[watchdog] finished with exit_code=0", flush=True)
|
|
171
|
-
return 0
|
|
172
|
-
|
|
173
|
-
if stop_on_nonzero_exit and not killed_for_idle:
|
|
174
|
-
print(
|
|
175
|
-
f"[watchdog] command exited non-zero (exit_code={exit_code}); stop.",
|
|
176
|
-
flush=True,
|
|
177
|
-
)
|
|
178
|
-
return int(exit_code)
|
|
179
|
-
|
|
180
|
-
if attempt > max_restarts + 1:
|
|
181
|
-
print(
|
|
182
|
-
f"[watchdog] exceeded max_restarts={max_restarts}; last exit_code={exit_code}",
|
|
183
|
-
flush=True,
|
|
184
|
-
)
|
|
185
|
-
return int(exit_code)
|
|
186
|
-
|
|
187
|
-
print(
|
|
188
|
-
f"[watchdog] restart in {restart_delay_seconds}s (exit_code={exit_code}, killed_for_idle={killed_for_idle})",
|
|
189
|
-
flush=True,
|
|
190
|
-
)
|
|
191
|
-
if restart_delay_seconds:
|
|
192
|
-
time.sleep(restart_delay_seconds)
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
def main(argv: Optional[List[str]] = None) -> int:
|
|
196
|
-
if configure_run_logging:
|
|
197
|
-
configure_run_logging(prefix="watchdog")
|
|
198
|
-
argv = list(sys.argv[1:] if argv is None else argv)
|
|
199
|
-
before_cmd, cmd = _split_argv(argv)
|
|
200
|
-
args = _parse_args(before_cmd, cmd)
|
|
201
|
-
return run_with_watchdog(
|
|
202
|
-
cmd=cmd,
|
|
203
|
-
idle_seconds=args.idle_seconds,
|
|
204
|
-
max_restarts=args.max_restarts,
|
|
205
|
-
restart_delay_seconds=args.restart_delay_seconds,
|
|
206
|
-
stop_on_nonzero_exit=bool(args.stop_on_nonzero_exit),
|
|
207
|
-
)
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
if __name__ == "__main__":
|
|
211
|
-
raise SystemExit(main())
|
|
22
|
+
try:
|
|
23
|
+
from utils.run_logging import configure_run_logging # type: ignore
|
|
24
|
+
except Exception: # pragma: no cover
|
|
25
|
+
configure_run_logging = None # type: ignore
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _split_argv(argv: List[str]) -> tuple[List[str], List[str]]:
|
|
29
|
+
if "--" not in argv:
|
|
30
|
+
raise ValueError("Missing '--' separator before the command to run.")
|
|
31
|
+
idx = argv.index("--")
|
|
32
|
+
return argv[:idx], argv[idx + 1 :]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _kill_process_tree(pid: int) -> None:
|
|
36
|
+
if pid <= 0:
|
|
37
|
+
return
|
|
38
|
+
if os.name == "nt":
|
|
39
|
+
subprocess.run(
|
|
40
|
+
["taskkill", "/PID", str(pid), "/T", "/F"],
|
|
41
|
+
stdout=subprocess.DEVNULL,
|
|
42
|
+
stderr=subprocess.DEVNULL,
|
|
43
|
+
check=False,
|
|
44
|
+
)
|
|
45
|
+
return
|
|
46
|
+
try:
|
|
47
|
+
os.killpg(pid, 15)
|
|
48
|
+
time.sleep(2)
|
|
49
|
+
os.killpg(pid, 9)
|
|
50
|
+
except Exception:
|
|
51
|
+
try:
|
|
52
|
+
os.kill(pid, 9)
|
|
53
|
+
except Exception:
|
|
54
|
+
pass
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _reader_thread(
|
|
58
|
+
proc: subprocess.Popen, last_output_ts: dict, prefix: str = ""
|
|
59
|
+
) -> None:
|
|
60
|
+
assert proc.stdout is not None
|
|
61
|
+
for line in proc.stdout:
|
|
62
|
+
last_output_ts["ts"] = time.time()
|
|
63
|
+
if prefix:
|
|
64
|
+
sys.stdout.write(prefix)
|
|
65
|
+
sys.stdout.write(line)
|
|
66
|
+
sys.stdout.flush()
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _parse_args(before_cmd: List[str], cmd: List[str]) -> argparse.Namespace:
|
|
70
|
+
parser = argparse.ArgumentParser(
|
|
71
|
+
description=(
|
|
72
|
+
"Run a command under a simple watchdog: if there is no stdout/stderr "
|
|
73
|
+
"output for N seconds, kill the whole process tree and restart. "
|
|
74
|
+
"Designed to pair with optuna_storage so BayesOpt can resume."
|
|
75
|
+
)
|
|
76
|
+
)
|
|
77
|
+
parser.add_argument(
|
|
78
|
+
"--idle-seconds",
|
|
79
|
+
type=int,
|
|
80
|
+
default=7200,
|
|
81
|
+
help="Restart if there is no output for this many seconds (default: 7200).",
|
|
82
|
+
)
|
|
83
|
+
parser.add_argument(
|
|
84
|
+
"--max-restarts",
|
|
85
|
+
type=int,
|
|
86
|
+
default=50,
|
|
87
|
+
help="Maximum restart attempts (default: 50).",
|
|
88
|
+
)
|
|
89
|
+
parser.add_argument(
|
|
90
|
+
"--restart-delay-seconds",
|
|
91
|
+
type=int,
|
|
92
|
+
default=10,
|
|
93
|
+
help="Delay between restarts (default: 10).",
|
|
94
|
+
)
|
|
95
|
+
parser.add_argument(
|
|
96
|
+
"--stop-on-nonzero-exit",
|
|
97
|
+
action="store_true",
|
|
98
|
+
help="If the command exits non-zero, stop instead of restarting.",
|
|
99
|
+
)
|
|
100
|
+
args = parser.parse_args(before_cmd)
|
|
101
|
+
if not cmd:
|
|
102
|
+
parser.error("Empty command after '--'.")
|
|
103
|
+
return args
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def run_with_watchdog(
|
|
107
|
+
cmd: List[str],
|
|
108
|
+
idle_seconds: int,
|
|
109
|
+
max_restarts: int,
|
|
110
|
+
restart_delay_seconds: int,
|
|
111
|
+
stop_on_nonzero_exit: bool,
|
|
112
|
+
) -> int:
|
|
113
|
+
idle_seconds = max(1, int(idle_seconds))
|
|
114
|
+
max_restarts = max(0, int(max_restarts))
|
|
115
|
+
restart_delay_seconds = max(0, int(restart_delay_seconds))
|
|
116
|
+
|
|
117
|
+
attempt = 0
|
|
118
|
+
while True:
|
|
119
|
+
attempt += 1
|
|
120
|
+
print(
|
|
121
|
+
f"[watchdog] start attempt={attempt} idle_seconds={idle_seconds} cmd={cmd}",
|
|
122
|
+
flush=True,
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
creationflags = 0
|
|
126
|
+
start_new_session = False
|
|
127
|
+
if os.name == "nt":
|
|
128
|
+
creationflags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)
|
|
129
|
+
else:
|
|
130
|
+
start_new_session = True
|
|
131
|
+
|
|
132
|
+
proc = subprocess.Popen(
|
|
133
|
+
cmd,
|
|
134
|
+
stdout=subprocess.PIPE,
|
|
135
|
+
stderr=subprocess.STDOUT,
|
|
136
|
+
text=True,
|
|
137
|
+
bufsize=1,
|
|
138
|
+
universal_newlines=True,
|
|
139
|
+
creationflags=creationflags,
|
|
140
|
+
start_new_session=start_new_session,
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
last_output_ts: dict = {"ts": time.time()}
|
|
144
|
+
reader = threading.Thread(
|
|
145
|
+
target=_reader_thread,
|
|
146
|
+
args=(proc, last_output_ts),
|
|
147
|
+
kwargs={"prefix": ""},
|
|
148
|
+
daemon=True,
|
|
149
|
+
)
|
|
150
|
+
reader.start()
|
|
151
|
+
|
|
152
|
+
killed_for_idle = False
|
|
153
|
+
exit_code: Optional[int] = None
|
|
154
|
+
while True:
|
|
155
|
+
exit_code = proc.poll()
|
|
156
|
+
if exit_code is not None:
|
|
157
|
+
break
|
|
158
|
+
idle_for = time.time() - float(last_output_ts["ts"])
|
|
159
|
+
if idle_for > idle_seconds:
|
|
160
|
+
killed_for_idle = True
|
|
161
|
+
print(
|
|
162
|
+
f"[watchdog] idle>{idle_seconds}s (idle_for={int(idle_for)}s), killing pid={proc.pid}",
|
|
163
|
+
flush=True,
|
|
164
|
+
)
|
|
165
|
+
_kill_process_tree(proc.pid)
|
|
166
|
+
break
|
|
167
|
+
time.sleep(5)
|
|
168
|
+
|
|
169
|
+
try:
|
|
170
|
+
proc.wait(timeout=30)
|
|
171
|
+
except Exception:
|
|
172
|
+
_kill_process_tree(proc.pid)
|
|
173
|
+
|
|
174
|
+
if exit_code is None:
|
|
175
|
+
exit_code = proc.poll() or 1
|
|
176
|
+
|
|
177
|
+
if exit_code == 0:
|
|
178
|
+
print("[watchdog] finished with exit_code=0", flush=True)
|
|
179
|
+
return 0
|
|
180
|
+
|
|
181
|
+
if stop_on_nonzero_exit and not killed_for_idle:
|
|
182
|
+
print(
|
|
183
|
+
f"[watchdog] command exited non-zero (exit_code={exit_code}); stop.",
|
|
184
|
+
flush=True,
|
|
185
|
+
)
|
|
186
|
+
return int(exit_code)
|
|
187
|
+
|
|
188
|
+
if attempt > max_restarts + 1:
|
|
189
|
+
print(
|
|
190
|
+
f"[watchdog] exceeded max_restarts={max_restarts}; last exit_code={exit_code}",
|
|
191
|
+
flush=True,
|
|
192
|
+
)
|
|
193
|
+
return int(exit_code)
|
|
194
|
+
|
|
195
|
+
print(
|
|
196
|
+
f"[watchdog] restart in {restart_delay_seconds}s (exit_code={exit_code}, killed_for_idle={killed_for_idle})",
|
|
197
|
+
flush=True,
|
|
198
|
+
)
|
|
199
|
+
if restart_delay_seconds:
|
|
200
|
+
time.sleep(restart_delay_seconds)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def main(argv: Optional[List[str]] = None) -> int:
|
|
204
|
+
if configure_run_logging:
|
|
205
|
+
configure_run_logging(prefix="watchdog")
|
|
206
|
+
argv = list(sys.argv[1:] if argv is None else argv)
|
|
207
|
+
before_cmd, cmd = _split_argv(argv)
|
|
208
|
+
args = _parse_args(before_cmd, cmd)
|
|
209
|
+
return run_with_watchdog(
|
|
210
|
+
cmd=cmd,
|
|
211
|
+
idle_seconds=args.idle_seconds,
|
|
212
|
+
max_restarts=args.max_restarts,
|
|
213
|
+
restart_delay_seconds=args.restart_delay_seconds,
|
|
214
|
+
stop_on_nonzero_exit=bool(args.stop_on_nonzero_exit),
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
if __name__ == "__main__":
|
|
219
|
+
raise SystemExit(main())
|
ins_pricing/frontend/__init__.py
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Insurance Pricing Frontend Package
|
|
3
|
-
Web-based interface for configuring and running insurance pricing model tasks.
|
|
4
|
-
"""
|
|
5
|
-
|
|
6
|
-
from .config_builder import ConfigBuilder
|
|
7
|
-
from .runner import TaskRunner, TrainingRunner
|
|
8
|
-
from .ft_workflow import FTWorkflowHelper
|
|
9
|
-
|
|
10
|
-
__all__ = ['ConfigBuilder', 'TaskRunner', 'TrainingRunner', 'FTWorkflowHelper']
|
|
1
|
+
"""
|
|
2
|
+
Insurance Pricing Frontend Package
|
|
3
|
+
Web-based interface for configuring and running insurance pricing model tasks.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from ins_pricing.frontend.config_builder import ConfigBuilder
|
|
7
|
+
from ins_pricing.frontend.runner import TaskRunner, TrainingRunner
|
|
8
|
+
from ins_pricing.frontend.ft_workflow import FTWorkflowHelper
|
|
9
|
+
|
|
10
|
+
__all__ = ['ConfigBuilder', 'TaskRunner', 'TrainingRunner', 'FTWorkflowHelper']
|
|
@@ -19,7 +19,7 @@ from ins_pricing.modelling.plotting import (
|
|
|
19
19
|
plot_oneway,
|
|
20
20
|
)
|
|
21
21
|
from ins_pricing.modelling.plotting.common import finalize_figure, plt
|
|
22
|
-
from ins_pricing.production.
|
|
22
|
+
from ins_pricing.production.inference import load_predictor_from_config
|
|
23
23
|
|
|
24
24
|
|
|
25
25
|
def _parse_csv_list(value: str) -> List[str]:
|
|
@@ -1,20 +1,20 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from .approval import ApprovalAction, ApprovalRequest, ApprovalStore
|
|
4
|
-
from .audit import AuditEvent, AuditLogger
|
|
5
|
-
from .registry import ModelArtifact, ModelRegistry, ModelVersion
|
|
6
|
-
from .release import DeploymentState, ModelRef, ReleaseManager
|
|
7
|
-
|
|
8
|
-
__all__ = [
|
|
9
|
-
"ApprovalAction",
|
|
10
|
-
"ApprovalRequest",
|
|
11
|
-
"ApprovalStore",
|
|
12
|
-
"AuditEvent",
|
|
13
|
-
"AuditLogger",
|
|
14
|
-
"ModelArtifact",
|
|
15
|
-
"ModelRegistry",
|
|
16
|
-
"ModelVersion",
|
|
17
|
-
"DeploymentState",
|
|
18
|
-
"ModelRef",
|
|
19
|
-
"ReleaseManager",
|
|
20
|
-
]
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from ins_pricing.governance.approval import ApprovalAction, ApprovalRequest, ApprovalStore
|
|
4
|
+
from ins_pricing.governance.audit import AuditEvent, AuditLogger
|
|
5
|
+
from ins_pricing.governance.registry import ModelArtifact, ModelRegistry, ModelVersion
|
|
6
|
+
from ins_pricing.governance.release import DeploymentState, ModelRef, ReleaseManager
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"ApprovalAction",
|
|
10
|
+
"ApprovalRequest",
|
|
11
|
+
"ApprovalStore",
|
|
12
|
+
"AuditEvent",
|
|
13
|
+
"AuditLogger",
|
|
14
|
+
"ModelArtifact",
|
|
15
|
+
"ModelRegistry",
|
|
16
|
+
"ModelVersion",
|
|
17
|
+
"DeploymentState",
|
|
18
|
+
"ModelRef",
|
|
19
|
+
"ReleaseManager",
|
|
20
|
+
]
|