ins-pricing 0.4.5__py3-none-any.whl → 0.5.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ins_pricing/README.md +48 -22
- ins_pricing/__init__.py +142 -90
- ins_pricing/cli/BayesOpt_entry.py +58 -46
- ins_pricing/cli/BayesOpt_incremental.py +77 -110
- ins_pricing/cli/Explain_Run.py +42 -23
- ins_pricing/cli/Explain_entry.py +551 -577
- ins_pricing/cli/Pricing_Run.py +42 -23
- ins_pricing/cli/bayesopt_entry_runner.py +51 -16
- ins_pricing/cli/utils/bootstrap.py +23 -0
- ins_pricing/cli/utils/cli_common.py +256 -256
- ins_pricing/cli/utils/cli_config.py +379 -360
- ins_pricing/cli/utils/import_resolver.py +375 -358
- ins_pricing/cli/utils/notebook_utils.py +256 -242
- ins_pricing/cli/watchdog_run.py +216 -198
- ins_pricing/frontend/__init__.py +10 -10
- ins_pricing/frontend/app.py +132 -61
- ins_pricing/frontend/config_builder.py +33 -0
- ins_pricing/frontend/example_config.json +11 -0
- ins_pricing/frontend/example_workflows.py +1 -1
- ins_pricing/frontend/runner.py +340 -388
- ins_pricing/governance/__init__.py +20 -20
- ins_pricing/governance/release.py +159 -159
- ins_pricing/modelling/README.md +1 -1
- ins_pricing/modelling/__init__.py +147 -92
- ins_pricing/modelling/{core/bayesopt → bayesopt}/README.md +31 -13
- ins_pricing/modelling/{core/bayesopt → bayesopt}/__init__.py +64 -102
- ins_pricing/modelling/{core/bayesopt → bayesopt}/config_components.py +12 -0
- ins_pricing/modelling/{core/bayesopt → bayesopt}/config_preprocess.py +589 -552
- ins_pricing/modelling/{core/bayesopt → bayesopt}/core.py +987 -958
- ins_pricing/modelling/{core/bayesopt → bayesopt}/model_explain_mixin.py +296 -296
- ins_pricing/modelling/{core/bayesopt → bayesopt}/model_plotting_mixin.py +488 -548
- ins_pricing/modelling/{core/bayesopt → bayesopt}/models/__init__.py +27 -27
- ins_pricing/modelling/{core/bayesopt → bayesopt}/models/model_ft_components.py +349 -342
- ins_pricing/modelling/{core/bayesopt → bayesopt}/models/model_ft_trainer.py +921 -913
- ins_pricing/modelling/{core/bayesopt → bayesopt}/models/model_gnn.py +794 -785
- ins_pricing/modelling/{core/bayesopt → bayesopt}/models/model_resn.py +454 -446
- ins_pricing/modelling/bayesopt/trainers/__init__.py +19 -0
- ins_pricing/modelling/{core/bayesopt → bayesopt}/trainers/trainer_base.py +1294 -1282
- ins_pricing/modelling/{core/bayesopt → bayesopt}/trainers/trainer_ft.py +64 -56
- ins_pricing/modelling/{core/bayesopt → bayesopt}/trainers/trainer_glm.py +203 -198
- ins_pricing/modelling/{core/bayesopt → bayesopt}/trainers/trainer_gnn.py +333 -325
- ins_pricing/modelling/{core/bayesopt → bayesopt}/trainers/trainer_resn.py +279 -267
- ins_pricing/modelling/{core/bayesopt → bayesopt}/trainers/trainer_xgb.py +515 -313
- ins_pricing/modelling/bayesopt/utils/__init__.py +67 -0
- ins_pricing/modelling/bayesopt/utils/constants.py +21 -0
- ins_pricing/modelling/{core/bayesopt → bayesopt}/utils/distributed_utils.py +193 -186
- ins_pricing/modelling/bayesopt/utils/io_utils.py +7 -0
- ins_pricing/modelling/bayesopt/utils/losses.py +27 -0
- ins_pricing/modelling/bayesopt/utils/metrics_and_devices.py +17 -0
- ins_pricing/modelling/{core/bayesopt → bayesopt}/utils/torch_trainer_mixin.py +636 -623
- ins_pricing/modelling/{core/evaluation.py → evaluation.py} +113 -104
- ins_pricing/modelling/explain/__init__.py +55 -55
- ins_pricing/modelling/explain/metrics.py +27 -174
- ins_pricing/modelling/explain/permutation.py +237 -237
- ins_pricing/modelling/plotting/__init__.py +40 -36
- ins_pricing/modelling/plotting/compat.py +228 -0
- ins_pricing/modelling/plotting/curves.py +572 -572
- ins_pricing/modelling/plotting/diagnostics.py +163 -163
- ins_pricing/modelling/plotting/geo.py +362 -362
- ins_pricing/modelling/plotting/importance.py +121 -121
- ins_pricing/pricing/__init__.py +27 -27
- ins_pricing/pricing/factors.py +67 -56
- ins_pricing/production/__init__.py +35 -25
- ins_pricing/production/{predict.py → inference.py} +140 -57
- ins_pricing/production/monitoring.py +8 -21
- ins_pricing/reporting/__init__.py +11 -11
- ins_pricing/setup.py +1 -1
- ins_pricing/tests/production/test_inference.py +90 -0
- ins_pricing/utils/__init__.py +112 -78
- ins_pricing/utils/device.py +258 -237
- ins_pricing/utils/features.py +53 -0
- ins_pricing/utils/io.py +72 -0
- ins_pricing/utils/logging.py +34 -1
- ins_pricing/{modelling/core/bayesopt/utils → utils}/losses.py +125 -129
- ins_pricing/utils/metrics.py +158 -24
- ins_pricing/utils/numerics.py +76 -0
- ins_pricing/utils/paths.py +9 -1
- ins_pricing/utils/profiling.py +8 -4
- {ins_pricing-0.4.5.dist-info → ins_pricing-0.5.1.dist-info}/METADATA +1 -1
- ins_pricing-0.5.1.dist-info/RECORD +132 -0
- ins_pricing/modelling/core/BayesOpt.py +0 -146
- ins_pricing/modelling/core/__init__.py +0 -1
- ins_pricing/modelling/core/bayesopt/trainers/__init__.py +0 -19
- ins_pricing/modelling/core/bayesopt/utils/__init__.py +0 -86
- ins_pricing/modelling/core/bayesopt/utils/constants.py +0 -183
- ins_pricing/modelling/core/bayesopt/utils/io_utils.py +0 -126
- ins_pricing/modelling/core/bayesopt/utils/metrics_and_devices.py +0 -555
- ins_pricing/modelling/core/bayesopt/utils.py +0 -105
- ins_pricing/modelling/core/bayesopt/utils_backup.py +0 -1503
- ins_pricing/tests/production/test_predict.py +0 -233
- ins_pricing-0.4.5.dist-info/RECORD +0 -130
- {ins_pricing-0.4.5.dist-info → ins_pricing-0.5.1.dist-info}/WHEEL +0 -0
- {ins_pricing-0.4.5.dist-info → ins_pricing-0.5.1.dist-info}/top_level.txt +0 -0
ins_pricing/cli/watchdog_run.py
CHANGED
|
@@ -1,211 +1,229 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
3
|
import argparse
|
|
4
4
|
import os
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import importlib.util
|
|
5
7
|
import subprocess
|
|
6
8
|
import sys
|
|
7
9
|
import threading
|
|
8
10
|
import time
|
|
9
11
|
from typing import List, Optional
|
|
10
12
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
except Exception: # pragma: no cover
|
|
14
|
-
try:
|
|
15
|
-
from utils.run_logging import configure_run_logging # type: ignore
|
|
16
|
-
except Exception: # pragma: no cover
|
|
17
|
-
configure_run_logging = None # type: ignore
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
def _split_argv(argv: List[str]) -> tuple[List[str], List[str]]:
|
|
21
|
-
if "--" not in argv:
|
|
22
|
-
raise ValueError("Missing '--' separator before the command to run.")
|
|
23
|
-
idx = argv.index("--")
|
|
24
|
-
return argv[:idx], argv[idx + 1 :]
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
def _kill_process_tree(pid: int) -> None:
|
|
28
|
-
if pid <= 0:
|
|
13
|
+
def _ensure_repo_root() -> None:
|
|
14
|
+
if __package__ not in {None, ""}:
|
|
29
15
|
return
|
|
30
|
-
if
|
|
31
|
-
subprocess.run(
|
|
32
|
-
["taskkill", "/PID", str(pid), "/T", "/F"],
|
|
33
|
-
stdout=subprocess.DEVNULL,
|
|
34
|
-
stderr=subprocess.DEVNULL,
|
|
35
|
-
check=False,
|
|
36
|
-
)
|
|
16
|
+
if importlib.util.find_spec("ins_pricing") is not None:
|
|
37
17
|
return
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
except Exception:
|
|
46
|
-
pass
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
def _reader_thread(
|
|
50
|
-
proc: subprocess.Popen, last_output_ts: dict, prefix: str = ""
|
|
51
|
-
) -> None:
|
|
52
|
-
assert proc.stdout is not None
|
|
53
|
-
for line in proc.stdout:
|
|
54
|
-
last_output_ts["ts"] = time.time()
|
|
55
|
-
if prefix:
|
|
56
|
-
sys.stdout.write(prefix)
|
|
57
|
-
sys.stdout.write(line)
|
|
58
|
-
sys.stdout.flush()
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
def _parse_args(before_cmd: List[str], cmd: List[str]) -> argparse.Namespace:
|
|
62
|
-
parser = argparse.ArgumentParser(
|
|
63
|
-
description=(
|
|
64
|
-
"Run a command under a simple watchdog: if there is no stdout/stderr "
|
|
65
|
-
"output for N seconds, kill the whole process tree and restart. "
|
|
66
|
-
"Designed to pair with optuna_storage so BayesOpt can resume."
|
|
67
|
-
)
|
|
68
|
-
)
|
|
69
|
-
parser.add_argument(
|
|
70
|
-
"--idle-seconds",
|
|
71
|
-
type=int,
|
|
72
|
-
default=7200,
|
|
73
|
-
help="Restart if there is no output for this many seconds (default: 7200).",
|
|
74
|
-
)
|
|
75
|
-
parser.add_argument(
|
|
76
|
-
"--max-restarts",
|
|
77
|
-
type=int,
|
|
78
|
-
default=50,
|
|
79
|
-
help="Maximum restart attempts (default: 50).",
|
|
80
|
-
)
|
|
81
|
-
parser.add_argument(
|
|
82
|
-
"--restart-delay-seconds",
|
|
83
|
-
type=int,
|
|
84
|
-
default=10,
|
|
85
|
-
help="Delay between restarts (default: 10).",
|
|
86
|
-
)
|
|
87
|
-
parser.add_argument(
|
|
88
|
-
"--stop-on-nonzero-exit",
|
|
89
|
-
action="store_true",
|
|
90
|
-
help="If the command exits non-zero, stop instead of restarting.",
|
|
91
|
-
)
|
|
92
|
-
args = parser.parse_args(before_cmd)
|
|
93
|
-
if not cmd:
|
|
94
|
-
parser.error("Empty command after '--'.")
|
|
95
|
-
return args
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
def run_with_watchdog(
|
|
99
|
-
cmd: List[str],
|
|
100
|
-
idle_seconds: int,
|
|
101
|
-
max_restarts: int,
|
|
102
|
-
restart_delay_seconds: int,
|
|
103
|
-
stop_on_nonzero_exit: bool,
|
|
104
|
-
) -> int:
|
|
105
|
-
idle_seconds = max(1, int(idle_seconds))
|
|
106
|
-
max_restarts = max(0, int(max_restarts))
|
|
107
|
-
restart_delay_seconds = max(0, int(restart_delay_seconds))
|
|
108
|
-
|
|
109
|
-
attempt = 0
|
|
110
|
-
while True:
|
|
111
|
-
attempt += 1
|
|
112
|
-
print(
|
|
113
|
-
f"[watchdog] start attempt={attempt} idle_seconds={idle_seconds} cmd={cmd}",
|
|
114
|
-
flush=True,
|
|
115
|
-
)
|
|
116
|
-
|
|
117
|
-
creationflags = 0
|
|
118
|
-
start_new_session = False
|
|
119
|
-
if os.name == "nt":
|
|
120
|
-
creationflags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)
|
|
121
|
-
else:
|
|
122
|
-
start_new_session = True
|
|
123
|
-
|
|
124
|
-
proc = subprocess.Popen(
|
|
125
|
-
cmd,
|
|
126
|
-
stdout=subprocess.PIPE,
|
|
127
|
-
stderr=subprocess.STDOUT,
|
|
128
|
-
text=True,
|
|
129
|
-
bufsize=1,
|
|
130
|
-
universal_newlines=True,
|
|
131
|
-
creationflags=creationflags,
|
|
132
|
-
start_new_session=start_new_session,
|
|
133
|
-
)
|
|
134
|
-
|
|
135
|
-
last_output_ts: dict = {"ts": time.time()}
|
|
136
|
-
reader = threading.Thread(
|
|
137
|
-
target=_reader_thread,
|
|
138
|
-
args=(proc, last_output_ts),
|
|
139
|
-
kwargs={"prefix": ""},
|
|
140
|
-
daemon=True,
|
|
141
|
-
)
|
|
142
|
-
reader.start()
|
|
143
|
-
|
|
144
|
-
killed_for_idle = False
|
|
145
|
-
exit_code: Optional[int] = None
|
|
146
|
-
while True:
|
|
147
|
-
exit_code = proc.poll()
|
|
148
|
-
if exit_code is not None:
|
|
149
|
-
break
|
|
150
|
-
idle_for = time.time() - float(last_output_ts["ts"])
|
|
151
|
-
if idle_for > idle_seconds:
|
|
152
|
-
killed_for_idle = True
|
|
153
|
-
print(
|
|
154
|
-
f"[watchdog] idle>{idle_seconds}s (idle_for={int(idle_for)}s), killing pid={proc.pid}",
|
|
155
|
-
flush=True,
|
|
156
|
-
)
|
|
157
|
-
_kill_process_tree(proc.pid)
|
|
158
|
-
break
|
|
159
|
-
time.sleep(5)
|
|
160
|
-
|
|
161
|
-
try:
|
|
162
|
-
proc.wait(timeout=30)
|
|
163
|
-
except Exception:
|
|
164
|
-
_kill_process_tree(proc.pid)
|
|
165
|
-
|
|
166
|
-
if exit_code is None:
|
|
167
|
-
exit_code = proc.poll() or 1
|
|
168
|
-
|
|
169
|
-
if exit_code == 0:
|
|
170
|
-
print("[watchdog] finished with exit_code=0", flush=True)
|
|
171
|
-
return 0
|
|
172
|
-
|
|
173
|
-
if stop_on_nonzero_exit and not killed_for_idle:
|
|
174
|
-
print(
|
|
175
|
-
f"[watchdog] command exited non-zero (exit_code={exit_code}); stop.",
|
|
176
|
-
flush=True,
|
|
177
|
-
)
|
|
178
|
-
return int(exit_code)
|
|
179
|
-
|
|
180
|
-
if attempt > max_restarts + 1:
|
|
181
|
-
print(
|
|
182
|
-
f"[watchdog] exceeded max_restarts={max_restarts}; last exit_code={exit_code}",
|
|
183
|
-
flush=True,
|
|
184
|
-
)
|
|
185
|
-
return int(exit_code)
|
|
186
|
-
|
|
187
|
-
print(
|
|
188
|
-
f"[watchdog] restart in {restart_delay_seconds}s (exit_code={exit_code}, killed_for_idle={killed_for_idle})",
|
|
189
|
-
flush=True,
|
|
190
|
-
)
|
|
191
|
-
if restart_delay_seconds:
|
|
192
|
-
time.sleep(restart_delay_seconds)
|
|
193
|
-
|
|
18
|
+
bootstrap_path = Path(__file__).resolve().parents[1] / "utils" / "bootstrap.py"
|
|
19
|
+
spec = importlib.util.spec_from_file_location("ins_pricing.cli.utils.bootstrap", bootstrap_path)
|
|
20
|
+
if spec is None or spec.loader is None:
|
|
21
|
+
return
|
|
22
|
+
module = importlib.util.module_from_spec(spec)
|
|
23
|
+
spec.loader.exec_module(module)
|
|
24
|
+
module.ensure_repo_root()
|
|
194
25
|
|
|
195
|
-
def main(argv: Optional[List[str]] = None) -> int:
|
|
196
|
-
if configure_run_logging:
|
|
197
|
-
configure_run_logging(prefix="watchdog")
|
|
198
|
-
argv = list(sys.argv[1:] if argv is None else argv)
|
|
199
|
-
before_cmd, cmd = _split_argv(argv)
|
|
200
|
-
args = _parse_args(before_cmd, cmd)
|
|
201
|
-
return run_with_watchdog(
|
|
202
|
-
cmd=cmd,
|
|
203
|
-
idle_seconds=args.idle_seconds,
|
|
204
|
-
max_restarts=args.max_restarts,
|
|
205
|
-
restart_delay_seconds=args.restart_delay_seconds,
|
|
206
|
-
stop_on_nonzero_exit=bool(args.stop_on_nonzero_exit),
|
|
207
|
-
)
|
|
208
26
|
|
|
27
|
+
_ensure_repo_root()
|
|
209
28
|
|
|
210
|
-
|
|
211
|
-
|
|
29
|
+
try:
|
|
30
|
+
from ins_pricing.cli.utils.run_logging import configure_run_logging # type: ignore
|
|
31
|
+
except Exception: # pragma: no cover
|
|
32
|
+
try:
|
|
33
|
+
from utils.run_logging import configure_run_logging # type: ignore
|
|
34
|
+
except Exception: # pragma: no cover
|
|
35
|
+
configure_run_logging = None # type: ignore
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _split_argv(argv: List[str]) -> tuple[List[str], List[str]]:
|
|
39
|
+
if "--" not in argv:
|
|
40
|
+
raise ValueError("Missing '--' separator before the command to run.")
|
|
41
|
+
idx = argv.index("--")
|
|
42
|
+
return argv[:idx], argv[idx + 1 :]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _kill_process_tree(pid: int) -> None:
|
|
46
|
+
if pid <= 0:
|
|
47
|
+
return
|
|
48
|
+
if os.name == "nt":
|
|
49
|
+
subprocess.run(
|
|
50
|
+
["taskkill", "/PID", str(pid), "/T", "/F"],
|
|
51
|
+
stdout=subprocess.DEVNULL,
|
|
52
|
+
stderr=subprocess.DEVNULL,
|
|
53
|
+
check=False,
|
|
54
|
+
)
|
|
55
|
+
return
|
|
56
|
+
try:
|
|
57
|
+
os.killpg(pid, 15)
|
|
58
|
+
time.sleep(2)
|
|
59
|
+
os.killpg(pid, 9)
|
|
60
|
+
except Exception:
|
|
61
|
+
try:
|
|
62
|
+
os.kill(pid, 9)
|
|
63
|
+
except Exception:
|
|
64
|
+
pass
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _reader_thread(
|
|
68
|
+
proc: subprocess.Popen, last_output_ts: dict, prefix: str = ""
|
|
69
|
+
) -> None:
|
|
70
|
+
assert proc.stdout is not None
|
|
71
|
+
for line in proc.stdout:
|
|
72
|
+
last_output_ts["ts"] = time.time()
|
|
73
|
+
if prefix:
|
|
74
|
+
sys.stdout.write(prefix)
|
|
75
|
+
sys.stdout.write(line)
|
|
76
|
+
sys.stdout.flush()
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _parse_args(before_cmd: List[str], cmd: List[str]) -> argparse.Namespace:
|
|
80
|
+
parser = argparse.ArgumentParser(
|
|
81
|
+
description=(
|
|
82
|
+
"Run a command under a simple watchdog: if there is no stdout/stderr "
|
|
83
|
+
"output for N seconds, kill the whole process tree and restart. "
|
|
84
|
+
"Designed to pair with optuna_storage so BayesOpt can resume."
|
|
85
|
+
)
|
|
86
|
+
)
|
|
87
|
+
parser.add_argument(
|
|
88
|
+
"--idle-seconds",
|
|
89
|
+
type=int,
|
|
90
|
+
default=7200,
|
|
91
|
+
help="Restart if there is no output for this many seconds (default: 7200).",
|
|
92
|
+
)
|
|
93
|
+
parser.add_argument(
|
|
94
|
+
"--max-restarts",
|
|
95
|
+
type=int,
|
|
96
|
+
default=50,
|
|
97
|
+
help="Maximum restart attempts (default: 50).",
|
|
98
|
+
)
|
|
99
|
+
parser.add_argument(
|
|
100
|
+
"--restart-delay-seconds",
|
|
101
|
+
type=int,
|
|
102
|
+
default=10,
|
|
103
|
+
help="Delay between restarts (default: 10).",
|
|
104
|
+
)
|
|
105
|
+
parser.add_argument(
|
|
106
|
+
"--stop-on-nonzero-exit",
|
|
107
|
+
action="store_true",
|
|
108
|
+
help="If the command exits non-zero, stop instead of restarting.",
|
|
109
|
+
)
|
|
110
|
+
args = parser.parse_args(before_cmd)
|
|
111
|
+
if not cmd:
|
|
112
|
+
parser.error("Empty command after '--'.")
|
|
113
|
+
return args
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def run_with_watchdog(
|
|
117
|
+
cmd: List[str],
|
|
118
|
+
idle_seconds: int,
|
|
119
|
+
max_restarts: int,
|
|
120
|
+
restart_delay_seconds: int,
|
|
121
|
+
stop_on_nonzero_exit: bool,
|
|
122
|
+
) -> int:
|
|
123
|
+
idle_seconds = max(1, int(idle_seconds))
|
|
124
|
+
max_restarts = max(0, int(max_restarts))
|
|
125
|
+
restart_delay_seconds = max(0, int(restart_delay_seconds))
|
|
126
|
+
|
|
127
|
+
attempt = 0
|
|
128
|
+
while True:
|
|
129
|
+
attempt += 1
|
|
130
|
+
print(
|
|
131
|
+
f"[watchdog] start attempt={attempt} idle_seconds={idle_seconds} cmd={cmd}",
|
|
132
|
+
flush=True,
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
creationflags = 0
|
|
136
|
+
start_new_session = False
|
|
137
|
+
if os.name == "nt":
|
|
138
|
+
creationflags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)
|
|
139
|
+
else:
|
|
140
|
+
start_new_session = True
|
|
141
|
+
|
|
142
|
+
proc = subprocess.Popen(
|
|
143
|
+
cmd,
|
|
144
|
+
stdout=subprocess.PIPE,
|
|
145
|
+
stderr=subprocess.STDOUT,
|
|
146
|
+
text=True,
|
|
147
|
+
bufsize=1,
|
|
148
|
+
universal_newlines=True,
|
|
149
|
+
creationflags=creationflags,
|
|
150
|
+
start_new_session=start_new_session,
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
last_output_ts: dict = {"ts": time.time()}
|
|
154
|
+
reader = threading.Thread(
|
|
155
|
+
target=_reader_thread,
|
|
156
|
+
args=(proc, last_output_ts),
|
|
157
|
+
kwargs={"prefix": ""},
|
|
158
|
+
daemon=True,
|
|
159
|
+
)
|
|
160
|
+
reader.start()
|
|
161
|
+
|
|
162
|
+
killed_for_idle = False
|
|
163
|
+
exit_code: Optional[int] = None
|
|
164
|
+
while True:
|
|
165
|
+
exit_code = proc.poll()
|
|
166
|
+
if exit_code is not None:
|
|
167
|
+
break
|
|
168
|
+
idle_for = time.time() - float(last_output_ts["ts"])
|
|
169
|
+
if idle_for > idle_seconds:
|
|
170
|
+
killed_for_idle = True
|
|
171
|
+
print(
|
|
172
|
+
f"[watchdog] idle>{idle_seconds}s (idle_for={int(idle_for)}s), killing pid={proc.pid}",
|
|
173
|
+
flush=True,
|
|
174
|
+
)
|
|
175
|
+
_kill_process_tree(proc.pid)
|
|
176
|
+
break
|
|
177
|
+
time.sleep(5)
|
|
178
|
+
|
|
179
|
+
try:
|
|
180
|
+
proc.wait(timeout=30)
|
|
181
|
+
except Exception:
|
|
182
|
+
_kill_process_tree(proc.pid)
|
|
183
|
+
|
|
184
|
+
if exit_code is None:
|
|
185
|
+
exit_code = proc.poll() or 1
|
|
186
|
+
|
|
187
|
+
if exit_code == 0:
|
|
188
|
+
print("[watchdog] finished with exit_code=0", flush=True)
|
|
189
|
+
return 0
|
|
190
|
+
|
|
191
|
+
if stop_on_nonzero_exit and not killed_for_idle:
|
|
192
|
+
print(
|
|
193
|
+
f"[watchdog] command exited non-zero (exit_code={exit_code}); stop.",
|
|
194
|
+
flush=True,
|
|
195
|
+
)
|
|
196
|
+
return int(exit_code)
|
|
197
|
+
|
|
198
|
+
if attempt > max_restarts + 1:
|
|
199
|
+
print(
|
|
200
|
+
f"[watchdog] exceeded max_restarts={max_restarts}; last exit_code={exit_code}",
|
|
201
|
+
flush=True,
|
|
202
|
+
)
|
|
203
|
+
return int(exit_code)
|
|
204
|
+
|
|
205
|
+
print(
|
|
206
|
+
f"[watchdog] restart in {restart_delay_seconds}s (exit_code={exit_code}, killed_for_idle={killed_for_idle})",
|
|
207
|
+
flush=True,
|
|
208
|
+
)
|
|
209
|
+
if restart_delay_seconds:
|
|
210
|
+
time.sleep(restart_delay_seconds)
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def main(argv: Optional[List[str]] = None) -> int:
|
|
214
|
+
if configure_run_logging:
|
|
215
|
+
configure_run_logging(prefix="watchdog")
|
|
216
|
+
argv = list(sys.argv[1:] if argv is None else argv)
|
|
217
|
+
before_cmd, cmd = _split_argv(argv)
|
|
218
|
+
args = _parse_args(before_cmd, cmd)
|
|
219
|
+
return run_with_watchdog(
|
|
220
|
+
cmd=cmd,
|
|
221
|
+
idle_seconds=args.idle_seconds,
|
|
222
|
+
max_restarts=args.max_restarts,
|
|
223
|
+
restart_delay_seconds=args.restart_delay_seconds,
|
|
224
|
+
stop_on_nonzero_exit=bool(args.stop_on_nonzero_exit),
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
if __name__ == "__main__":
|
|
229
|
+
raise SystemExit(main())
|
ins_pricing/frontend/__init__.py
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Insurance Pricing Frontend Package
|
|
3
|
-
Web-based interface for configuring and running insurance pricing model tasks.
|
|
4
|
-
"""
|
|
5
|
-
|
|
6
|
-
from .config_builder import ConfigBuilder
|
|
7
|
-
from .runner import TaskRunner, TrainingRunner
|
|
8
|
-
from .ft_workflow import FTWorkflowHelper
|
|
9
|
-
|
|
10
|
-
__all__ = ['ConfigBuilder', 'TaskRunner', 'TrainingRunner', 'FTWorkflowHelper']
|
|
1
|
+
"""
|
|
2
|
+
Insurance Pricing Frontend Package
|
|
3
|
+
Web-based interface for configuring and running insurance pricing model tasks.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from ins_pricing.frontend.config_builder import ConfigBuilder
|
|
7
|
+
from ins_pricing.frontend.runner import TaskRunner, TrainingRunner
|
|
8
|
+
from ins_pricing.frontend.ft_workflow import FTWorkflowHelper
|
|
9
|
+
|
|
10
|
+
__all__ = ['ConfigBuilder', 'TaskRunner', 'TrainingRunner', 'FTWorkflowHelper']
|