py-ib-controller 0.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ibcontroller/__init__.py +0 -0
- ibcontroller/actions.py +273 -0
- ibcontroller/agent_client.py +466 -0
- ibcontroller/app_dirs.py +44 -0
- ibcontroller/cli.py +211 -0
- ibcontroller/config.py +410 -0
- ibcontroller/control_loop.py +636 -0
- ibcontroller/data/builtin_settings.toml +37 -0
- ibcontroller/data/ibcontroller.toml.example +178 -0
- ibcontroller/data/ibkr_settings.toml.example +135 -0
- ibcontroller/data/labels.json +81 -0
- ibcontroller/data/labels.json.example +44 -0
- ibcontroller/diagnostics.py +119 -0
- ibcontroller/dispatch.py +288 -0
- ibcontroller/ibcontroller-agent.jar +0 -0
- ibcontroller/labels.py +242 -0
- ibcontroller/launcher.py +909 -0
- ibcontroller/logging_setup.py +277 -0
- ibcontroller/login.py +578 -0
- ibcontroller/main.py +156 -0
- ibcontroller/py.typed +0 -0
- ibcontroller/recognisers.py +448 -0
- ibcontroller/schedule.py +157 -0
- ibcontroller/settings.py +554 -0
- py_ib_controller-0.0.0.dist-info/METADATA +490 -0
- py_ib_controller-0.0.0.dist-info/RECORD +30 -0
- py_ib_controller-0.0.0.dist-info/WHEEL +5 -0
- py_ib_controller-0.0.0.dist-info/entry_points.txt +2 -0
- py_ib_controller-0.0.0.dist-info/licenses/LICENSE.txt +176 -0
- py_ib_controller-0.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,636 @@
|
|
|
1
|
+
"""control_loop.py -- the top-level orchestrator.
|
|
2
|
+
|
|
3
|
+
launch -> login (with the always-on background watcher running concurrently) ->
|
|
4
|
+
declarative settings (built-in entries always, a user's own ibkr_settings.toml
|
|
5
|
+
merged in when configured -- see settings.py's own "Two tiers, not one" section) ->
|
|
6
|
+
a main loop that keeps the instance alive -> clean shutdown, whatever the reason.
|
|
7
|
+
|
|
8
|
+
Ties together L1/L5/L6/L7 (`launcher.py`/`recognisers.py`/`login.py`/
|
|
9
|
+
`settings.py`) into one continuously-running instance session.
|
|
10
|
+
|
|
11
|
+
Not a state machine -- a fixed sequence (like Login and Settings) that falls
|
|
12
|
+
into an indefinite wait once there's nothing left to actively do. The real
|
|
13
|
+
work for as long as an instance runs happens in `watch_for_unprompted_windows`
|
|
14
|
+
(a background task) and `Dispatcher`'s own event-reader/command-worker tasks;
|
|
15
|
+
this module's "main loop" just waits for one of those to end, or for the
|
|
16
|
+
caller to cancel it.
|
|
17
|
+
|
|
18
|
+
Shutdown has six causes, raced together (`asyncio.wait(...,
|
|
19
|
+
return_when=FIRST_COMPLETED)`), all converging on one exit path:
|
|
20
|
+
|
|
21
|
+
- **REQUESTED**: caller cancels this coroutine's own task.
|
|
22
|
+
- **PROCESS_EXITED**: `launched.process.wait()` returns -- a manual close, or
|
|
23
|
+
a scheduled daily restart (confirmed live: it kills the whole JVM, not an
|
|
24
|
+
in-place UI restart). Raced against every phase of the cycle, not just the
|
|
25
|
+
post-login `READY` wait, so a process dying during login or settings
|
|
26
|
+
application is classified the same way instead of hanging on a login-side
|
|
27
|
+
timeout that's either unbounded or unrelated to the process being gone.
|
|
28
|
+
- **CONNECTION_LOST**: `Dispatcher.tasks` ends while the process is still
|
|
29
|
+
alive -- a distinct failure mode from process-exit, also raced across every
|
|
30
|
+
phase.
|
|
31
|
+
- **LOGIN_FAILED**: `watch_for_unprompted_windows` raises
|
|
32
|
+
(`recognisers.LoginFailedError`, or `login.LoginError` from the MFA-timeout
|
|
33
|
+
watchdog).
|
|
34
|
+
- **COLD_RESTART**: `Config.cold_restart_time` reached (TWS and Gateway
|
|
35
|
+
alike, see `schedule.py`) -- a self-scheduled tidy close-down followed by a
|
|
36
|
+
full fresh relogin via `launch_instance` (no restart hash, deliberately not
|
|
37
|
+
a silent relogin, and not IBC's native-launcher `File > Restart`
|
|
38
|
+
mechanism), forcing the weekly full reauth IBKR requires around Sunday
|
|
39
|
+
01:00 US/Eastern token invalidation.
|
|
40
|
+
- **TIDY_CLOSEDOWN**: `Config.closedown_at` reached (TWS and Gateway alike)
|
|
41
|
+
-- a self-scheduled tidy close-down with no relaunch; the control loop
|
|
42
|
+
stops for good, same as any other terminal cause.
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
from __future__ import annotations
|
|
46
|
+
|
|
47
|
+
import asyncio
|
|
48
|
+
import contextlib
|
|
49
|
+
import logging
|
|
50
|
+
from collections.abc import Coroutine, Sequence
|
|
51
|
+
from datetime import datetime
|
|
52
|
+
from enum import Enum, auto
|
|
53
|
+
from pathlib import Path
|
|
54
|
+
from typing import Any, Protocol
|
|
55
|
+
|
|
56
|
+
from ibcontroller.agent_client import AgentClientError
|
|
57
|
+
from ibcontroller.config import Config
|
|
58
|
+
from ibcontroller.diagnostics import watch_for_diagnostics
|
|
59
|
+
from ibcontroller.labels import Labels, load_labels
|
|
60
|
+
from ibcontroller.launcher import (
|
|
61
|
+
LaunchedInstance,
|
|
62
|
+
LauncherError,
|
|
63
|
+
clean_shutdown,
|
|
64
|
+
launch_instance,
|
|
65
|
+
)
|
|
66
|
+
from ibcontroller.logging_setup import stop_logging
|
|
67
|
+
from ibcontroller.login import (
|
|
68
|
+
LoginError,
|
|
69
|
+
LoginManager,
|
|
70
|
+
find_autorestart_hash,
|
|
71
|
+
is_restart,
|
|
72
|
+
)
|
|
73
|
+
from ibcontroller.recognisers import (
|
|
74
|
+
AcceptIncomingConnectionsRecognizer,
|
|
75
|
+
DeclarativeDismissRecognizer,
|
|
76
|
+
ExistingSessionRecognizer,
|
|
77
|
+
LoginFailedError,
|
|
78
|
+
LoginFailedRecognizer,
|
|
79
|
+
RecognizerRegistry,
|
|
80
|
+
TooManyFailedLoginAttemptsRecognizer,
|
|
81
|
+
watch_for_unprompted_windows,
|
|
82
|
+
)
|
|
83
|
+
from ibcontroller.schedule import ScheduledAction, next_scheduled_shutdown
|
|
84
|
+
from ibcontroller.settings import (
|
|
85
|
+
SettingsError,
|
|
86
|
+
SettingsFile,
|
|
87
|
+
apply_settings_from_file,
|
|
88
|
+
close_settings_dialog,
|
|
89
|
+
load_builtin_settings_file,
|
|
90
|
+
load_settings_file,
|
|
91
|
+
merge_settings_files,
|
|
92
|
+
open_settings_dialog,
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
logger = logging.getLogger(__name__)
|
|
96
|
+
|
|
97
|
+
# Known "operational" failure modes reachable from within a cycle (a real
|
|
98
|
+
# login/settings/launch failure) -- `cli.py` extends this with the config-load
|
|
99
|
+
# failures that happen before a cycle even starts (`ConfigError`/`RuntimeError`)
|
|
100
|
+
# and reports the whole set as a clean one-line message, not a traceback.
|
|
101
|
+
# Shared here so `_run_one_cycle`'s own logging can tell an anticipated
|
|
102
|
+
# failure apart from a genuine bug too.
|
|
103
|
+
OPERATIONAL_ERRORS = (
|
|
104
|
+
LoginError,
|
|
105
|
+
LoginFailedError,
|
|
106
|
+
SettingsError,
|
|
107
|
+
LauncherError,
|
|
108
|
+
AgentClientError,
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class ShutdownCause(Enum):
|
|
113
|
+
REQUESTED = auto()
|
|
114
|
+
PROCESS_EXITED = auto()
|
|
115
|
+
CONNECTION_LOST = auto()
|
|
116
|
+
LOGIN_FAILED = auto()
|
|
117
|
+
COLD_RESTART = auto()
|
|
118
|
+
TIDY_CLOSEDOWN = auto()
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
class StartupState(Enum):
|
|
122
|
+
"""One instance's coarse lifecycle step, logged at each transition.
|
|
123
|
+
Observer only -- has zero effect on control flow.
|
|
124
|
+
|
|
125
|
+
`LOGGING_IN` deliberately covers both a fresh login and a restart-skip
|
|
126
|
+
relogin -- `login.py`'s own `LoginManager.state` (a finer `LoginState`
|
|
127
|
+
enum) already distinguishes those; this coarser field doesn't need to
|
|
128
|
+
duplicate it."""
|
|
129
|
+
|
|
130
|
+
LAUNCHING = auto()
|
|
131
|
+
LOGGING_IN = auto()
|
|
132
|
+
APPLYING_SETTINGS = auto()
|
|
133
|
+
READY = auto()
|
|
134
|
+
SHUTTING_DOWN = auto()
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _log_transition(previous: StartupState, new: StartupState) -> None:
|
|
138
|
+
logger.info("IBController > startup state: %s -> %s", previous.name, new.name)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _build_registry(
|
|
142
|
+
config: Config, labels: Labels, manager: LoginManager
|
|
143
|
+
) -> RecognizerRegistry:
|
|
144
|
+
# Declarative rules first, hand-written built-ins as the fallback --
|
|
145
|
+
# matching RecognizerRegistry's own stated priority order (config
|
|
146
|
+
# rules/plugins checked first, built-ins checked last).
|
|
147
|
+
declarative: list[DeclarativeDismissRecognizer] = [
|
|
148
|
+
DeclarativeDismissRecognizer(rule) for rule in labels.dismiss_rules
|
|
149
|
+
]
|
|
150
|
+
return RecognizerRegistry(
|
|
151
|
+
[
|
|
152
|
+
*declarative,
|
|
153
|
+
ExistingSessionRecognizer(
|
|
154
|
+
labels.existing_session,
|
|
155
|
+
config.existing_session_action,
|
|
156
|
+
manager.is_logged_in,
|
|
157
|
+
),
|
|
158
|
+
AcceptIncomingConnectionsRecognizer(
|
|
159
|
+
labels.accept_incoming_connection,
|
|
160
|
+
config.accept_incoming_connections,
|
|
161
|
+
),
|
|
162
|
+
LoginFailedRecognizer(labels.login_failed),
|
|
163
|
+
TooManyFailedLoginAttemptsRecognizer(
|
|
164
|
+
labels.too_many_failed_login_attempts,
|
|
165
|
+
config.relogin_after_mfa_timeout,
|
|
166
|
+
manager.schedule_retry,
|
|
167
|
+
),
|
|
168
|
+
]
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
async def _apply_declarative_settings(
|
|
173
|
+
launched: LaunchedInstance,
|
|
174
|
+
labels: Labels,
|
|
175
|
+
config: Config,
|
|
176
|
+
main_window_id: str | None,
|
|
177
|
+
) -> None:
|
|
178
|
+
"""Applies the bundled built-in entries (`read_only_api`/
|
|
179
|
+
`auto_restart_time`, from `Config` alone -- see `settings.py`'s own "Two
|
|
180
|
+
tiers, not one" section), merged with a user's own `Config.settings_file`
|
|
181
|
+
when set.
|
|
182
|
+
|
|
183
|
+
`main_window_id` (`LoginManager.main_window_id`, from `_run_one_cycle`'s
|
|
184
|
+
own `manager`) scopes `open_settings_dialog`'s menu navigation to the
|
|
185
|
+
already-validated main window instead of a global guess -- #40.
|
|
186
|
+
|
|
187
|
+
`window_id` (from `open_settings_dialog`'s return value) scopes every
|
|
188
|
+
subsequent action to the exact dialog that opened, and `close_settings_dialog`
|
|
189
|
+
always runs in `finally` -- both entries applying cleanly and one raising
|
|
190
|
+
still leave the dialog closed.
|
|
191
|
+
|
|
192
|
+
A `config.settings_file` that fails to load (missing, invalid TOML, bad
|
|
193
|
+
structure -- `settings.SettingsError`) is logged and treated as absent
|
|
194
|
+
rather than aborting this whole function -- otherwise a typo'd path or a
|
|
195
|
+
bad edit would silently take the load-bearing built-in entries
|
|
196
|
+
(`read_only_api`, `auto_restart_time`) down with it, since this runs
|
|
197
|
+
before `open_settings_dialog`."""
|
|
198
|
+
builtin_settings = await load_builtin_settings_file()
|
|
199
|
+
user_settings = None
|
|
200
|
+
if config.settings_file is not None:
|
|
201
|
+
try:
|
|
202
|
+
user_settings = await load_settings_file(config.settings_file)
|
|
203
|
+
except SettingsError as exc:
|
|
204
|
+
logger.error(
|
|
205
|
+
"IBController > settings: could not load %s -- continuing "
|
|
206
|
+
"with built-in settings only: %s",
|
|
207
|
+
config.settings_file,
|
|
208
|
+
exc,
|
|
209
|
+
)
|
|
210
|
+
merged = merge_settings_files(builtin_settings, user_settings)
|
|
211
|
+
# open_settings_dialog -> apply_settings_from_file -> close_settings_dialog
|
|
212
|
+
window_id = await open_settings_dialog(
|
|
213
|
+
launched.dispatcher,
|
|
214
|
+
labels.settings,
|
|
215
|
+
program=config.program,
|
|
216
|
+
timeout=config.mfa_timeout,
|
|
217
|
+
main_window_id=main_window_id,
|
|
218
|
+
)
|
|
219
|
+
try:
|
|
220
|
+
await apply_settings_from_file(
|
|
221
|
+
launched.dispatcher,
|
|
222
|
+
labels,
|
|
223
|
+
config,
|
|
224
|
+
SettingsFile(settings=merged),
|
|
225
|
+
window_id=window_id,
|
|
226
|
+
)
|
|
227
|
+
finally:
|
|
228
|
+
await close_settings_dialog(
|
|
229
|
+
launched.dispatcher, labels.settings, window_id=window_id
|
|
230
|
+
)
|
|
231
|
+
if user_settings is not None:
|
|
232
|
+
logger.info(
|
|
233
|
+
"IBController > declarative settings applied (built-in + %s)",
|
|
234
|
+
config.settings_file,
|
|
235
|
+
)
|
|
236
|
+
else:
|
|
237
|
+
logger.info("IBController > declarative settings applied (built-in only)")
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
async def _apply_declarative_settings_or_log(
|
|
241
|
+
launched: LaunchedInstance,
|
|
242
|
+
labels: Labels,
|
|
243
|
+
config: Config,
|
|
244
|
+
main_window_id: str | None,
|
|
245
|
+
) -> None:
|
|
246
|
+
"""`_apply_declarative_settings`, with a local error boundary -- a
|
|
247
|
+
Settings-application failure must not tear down an otherwise-healthy,
|
|
248
|
+
already-logged-in session. Matches IBC's own `ConfigurationTask.java`,
|
|
249
|
+
which recovers from exactly this step (`catch (Exception e) {
|
|
250
|
+
Utils.logException(e); }`) while every other step stays fatal on an
|
|
251
|
+
unhandled exception."""
|
|
252
|
+
try:
|
|
253
|
+
await _apply_declarative_settings(launched, labels, config, main_window_id)
|
|
254
|
+
except Exception:
|
|
255
|
+
logger.exception(
|
|
256
|
+
"IBController > declarative settings application failed -- "
|
|
257
|
+
"continuing without them"
|
|
258
|
+
)
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
async def run_control_loop(
|
|
262
|
+
config: Config, agent_jar: str | Path, *, labels: Labels | None = None
|
|
263
|
+
) -> ShutdownCause:
|
|
264
|
+
"""The outer loop -- ports IBC's own `ibcstart.sh` shape (a `while` loop
|
|
265
|
+
wrapped around the entire JVM invocation) around one lap of
|
|
266
|
+
`_run_one_cycle`. Gateway/TWS restarts on a schedule daily and that
|
|
267
|
+
restart kills the whole JVM, so a naive single-lap loop would stop on the
|
|
268
|
+
first restart. On `PROCESS_EXITED`, checks the `autorestart` marker file
|
|
269
|
+
(`login.is_restart`) -- if present, Gateway expects a relaunch that skips
|
|
270
|
+
full authentication, so loop back to `launch_instance`; if absent, this
|
|
271
|
+
was a genuine unscheduled exit and the loop stops for good. Every other
|
|
272
|
+
cause returns immediately -- only a confirmed scheduled restart loops.
|
|
273
|
+
|
|
274
|
+
On a confirmed restart, extracts the marker's account hash
|
|
275
|
+
(`login.find_autorestart_hash`) and passes it to the next lap's
|
|
276
|
+
`launch_instance`, which adds `-Drestart=<hash>` to the JVM command line
|
|
277
|
+
-- matching IBC's own `ibcstart.sh` and ibctl; the marker's mere presence
|
|
278
|
+
is not sufficient on its own.
|
|
279
|
+
|
|
280
|
+
Cancelling this coroutine's own task requests a graceful stop at any
|
|
281
|
+
point, including mid-relaunch, and always returns
|
|
282
|
+
`ShutdownCause.REQUESTED` rather than propagating `CancelledError` --
|
|
283
|
+
this coroutine is the documented cancellation boundary, so nothing else
|
|
284
|
+
needs a `try`/`except CancelledError` around it."""
|
|
285
|
+
labels = labels if labels is not None else load_labels()
|
|
286
|
+
restart_hash: str | None = None
|
|
287
|
+
try:
|
|
288
|
+
while True:
|
|
289
|
+
cause, settings_dir = await _run_one_cycle(
|
|
290
|
+
config, agent_jar, labels, restart_hash=restart_hash
|
|
291
|
+
)
|
|
292
|
+
if cause is ShutdownCause.COLD_RESTART:
|
|
293
|
+
restart_hash = None
|
|
294
|
+
logger.warning(
|
|
295
|
+
"IBController > cold_restart_time reached -- relaunching "
|
|
296
|
+
"with a full fresh login (no restart hash)"
|
|
297
|
+
)
|
|
298
|
+
continue
|
|
299
|
+
if cause is ShutdownCause.PROCESS_EXITED and await _is_restart_with_grace(
|
|
300
|
+
settings_dir
|
|
301
|
+
):
|
|
302
|
+
restart_hash = await find_autorestart_hash(settings_dir)
|
|
303
|
+
logger.warning(
|
|
304
|
+
"Gateway restarted on its own schedule (autorestart marker "
|
|
305
|
+
"present at %s, hash=%s) -- relaunching",
|
|
306
|
+
settings_dir,
|
|
307
|
+
restart_hash,
|
|
308
|
+
)
|
|
309
|
+
continue
|
|
310
|
+
return cause
|
|
311
|
+
except asyncio.CancelledError:
|
|
312
|
+
logger.info("IBController > control loop stop requested (task cancelled)")
|
|
313
|
+
return ShutdownCause.REQUESTED
|
|
314
|
+
finally:
|
|
315
|
+
# Drain `logging_setup`'s listener threads on the way out -- whatever the
|
|
316
|
+
# last lap logged or traced is still in its queues until this joins them.
|
|
317
|
+
stop_logging()
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
class _HasReturncode(Protocol):
|
|
321
|
+
"""The only slice of `asyncio.subprocess.Process` this function actually
|
|
322
|
+
reads -- narrowed so tests can pass a lightweight stand-in instead of a
|
|
323
|
+
real subprocess. A property, not a plain attribute, to structurally match
|
|
324
|
+
`asyncio.subprocess.Process.returncode`, which is itself a property."""
|
|
325
|
+
|
|
326
|
+
@property
|
|
327
|
+
def returncode(self) -> int | None: ...
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
async def _wait_for_first_completion(
|
|
331
|
+
*,
|
|
332
|
+
main_tasks: Sequence[asyncio.Task],
|
|
333
|
+
process_done: asyncio.Task,
|
|
334
|
+
dispatcher_tasks: Sequence[asyncio.Task],
|
|
335
|
+
process: _HasReturncode,
|
|
336
|
+
grace_period: float = 2.0,
|
|
337
|
+
) -> asyncio.Task:
|
|
338
|
+
"""Races `main_tasks` (the phase-specific work -- login, settings
|
|
339
|
+
application, or the READY-state `watcher`/`scheduled_shutdown` pair)
|
|
340
|
+
against `process_done`/`dispatcher_tasks` and returns whichever finished
|
|
341
|
+
first. Shared by every phase of `_run_one_cycle` (directly at READY, via
|
|
342
|
+
`_run_phase_watching_process` before it) so a dead process or lost
|
|
343
|
+
connection is classified the same way regardless of the instance's
|
|
344
|
+
current phase.
|
|
345
|
+
|
|
346
|
+
A JVM closes its own sockets early in its shutdown sequence -- well
|
|
347
|
+
before the OS reaps the process and asyncio's child watcher sets
|
|
348
|
+
`process.returncode` -- so on a real process exit, a dispatcher task
|
|
349
|
+
noticing the dropped connection can be the one `asyncio.wait` reports as
|
|
350
|
+
first-completed, moments ahead of `process_done` itself (confirmed live,
|
|
351
|
+
2026-09-14: a real scheduled restart was misclassified `CONNECTION_LOST`
|
|
352
|
+
this way even after the Linux native-relaunch path was independently
|
|
353
|
+
closed off, i.e. the process really was exiting -- this race, not a
|
|
354
|
+
bypassed relaunch, was the cause).
|
|
355
|
+
|
|
356
|
+
When that happens (some other task finished first and `returncode` isn't
|
|
357
|
+
set yet), gives `process_done` a short bounded window to catch up before
|
|
358
|
+
returning -- same shape as `_is_restart_with_grace`'s own poll: bound the
|
|
359
|
+
race statistically rather than trusting an instant check or waiting
|
|
360
|
+
forever. Deliberately still returns the *original* first-finished task
|
|
361
|
+
(not `process_done`) either way -- callers already fall back to checking
|
|
362
|
+
`process.returncode` directly, so they pick up a `PROCESS_EXITED`
|
|
363
|
+
classification correctly regardless of which task object this returns,
|
|
364
|
+
as long as `returncode` itself is now set."""
|
|
365
|
+
done, _pending = await asyncio.wait(
|
|
366
|
+
[*main_tasks, process_done, *dispatcher_tasks],
|
|
367
|
+
return_when=asyncio.FIRST_COMPLETED,
|
|
368
|
+
)
|
|
369
|
+
finished = next(iter(done))
|
|
370
|
+
if finished not in (*main_tasks, process_done) and process.returncode is None:
|
|
371
|
+
with contextlib.suppress(asyncio.TimeoutError):
|
|
372
|
+
await asyncio.wait_for(asyncio.shield(process_done), timeout=grace_period)
|
|
373
|
+
return finished
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
async def _is_restart_with_grace(
|
|
377
|
+
settings_dir: str, *, grace_period: float = 5.0, poll_interval: float = 0.2
|
|
378
|
+
) -> bool:
|
|
379
|
+
"""`is_restart()`, retried for a short bounded window -- the `autorestart`
|
|
380
|
+
marker can show up on disk slightly after `process.wait()` returns, so a
|
|
381
|
+
single immediate check can miss a genuine restart. Same shape as
|
|
382
|
+
`settings._await_menu_ready`'s splash-close wait: poll a short bounded
|
|
383
|
+
window rather than trusting an instant check or waiting forever."""
|
|
384
|
+
for _ in range(int(grace_period / poll_interval) + 1):
|
|
385
|
+
if await is_restart(settings_dir):
|
|
386
|
+
return True
|
|
387
|
+
await asyncio.sleep(poll_interval)
|
|
388
|
+
return False
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
class _PhaseAborted(Exception):
|
|
392
|
+
"""Signals that `process_done`/a dispatcher task won the race inside
|
|
393
|
+
`_run_phase_watching_process`, carrying the `ShutdownCause` to report.
|
|
394
|
+
Caught by `_run_one_cycle`, never propagates further."""
|
|
395
|
+
|
|
396
|
+
def __init__(self, cause: ShutdownCause) -> None:
|
|
397
|
+
super().__init__(cause)
|
|
398
|
+
self.cause = cause
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
async def _run_phase_watching_process(
|
|
402
|
+
phase: Coroutine[Any, Any, None] | asyncio.Task[None],
|
|
403
|
+
*,
|
|
404
|
+
process_done: asyncio.Task,
|
|
405
|
+
dispatcher_tasks: Sequence[asyncio.Task],
|
|
406
|
+
process: _HasReturncode,
|
|
407
|
+
) -> None:
|
|
408
|
+
"""Runs `phase` (login or settings application) racing it against
|
|
409
|
+
`process_done`/`dispatcher_tasks` via `_wait_for_first_completion` --
|
|
410
|
+
the same mechanism the READY-state wait already uses for `watcher`/
|
|
411
|
+
`scheduled_shutdown`, extended to cover the phases that run before it.
|
|
412
|
+
|
|
413
|
+
If `phase` finishes first, awaits it to completion, propagating its
|
|
414
|
+
result or exception unchanged -- a login/settings failure that isn't
|
|
415
|
+
caused by the process dying is not a `ShutdownCause`. If the process or
|
|
416
|
+
a dispatcher task finishes first instead, cancels `phase` and raises
|
|
417
|
+
`_PhaseAborted` with the matching cause."""
|
|
418
|
+
phase_task = asyncio.ensure_future(phase)
|
|
419
|
+
try:
|
|
420
|
+
finished = await _wait_for_first_completion(
|
|
421
|
+
main_tasks=[phase_task],
|
|
422
|
+
process_done=process_done,
|
|
423
|
+
dispatcher_tasks=dispatcher_tasks,
|
|
424
|
+
process=process,
|
|
425
|
+
)
|
|
426
|
+
if finished is phase_task:
|
|
427
|
+
await phase_task
|
|
428
|
+
return
|
|
429
|
+
cause = (
|
|
430
|
+
ShutdownCause.PROCESS_EXITED
|
|
431
|
+
if process.returncode is not None
|
|
432
|
+
else ShutdownCause.CONNECTION_LOST
|
|
433
|
+
)
|
|
434
|
+
raise _PhaseAborted(cause)
|
|
435
|
+
finally:
|
|
436
|
+
if not phase_task.done():
|
|
437
|
+
phase_task.cancel()
|
|
438
|
+
with contextlib.suppress(asyncio.CancelledError):
|
|
439
|
+
await phase_task
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
async def _sleep_until_scheduled_shutdown(config: Config) -> ShutdownCause:
|
|
443
|
+
"""Sleeps until `schedule.next_scheduled_shutdown` says to act, then
|
|
444
|
+
returns the matching `ShutdownCause` -- raced alongside `watcher`/
|
|
445
|
+
`process_done` in `_run_one_cycle`'s own `asyncio.wait`. Sleeps forever
|
|
446
|
+
(until cancelled by that race's `finally`) if neither `cold_restart_time`
|
|
447
|
+
nor `closedown_at` applies, so this task never itself decides to end the
|
|
448
|
+
cycle in that case.
|
|
449
|
+
|
|
450
|
+
Uses `asyncio.sleep`, not `loop.call_at` -- matches every other scheduled
|
|
451
|
+
wait in this codebase (`login.py`'s `schedule_retry`,
|
|
452
|
+
`_is_restart_with_grace` above, `launcher._wait_for_ready`); `call_at`
|
|
453
|
+
schedules against the monotonic loop clock, the wrong primitive for a
|
|
454
|
+
wall-clock target like "next Sunday 07:05 local time". Recomputed fresh
|
|
455
|
+
each `_run_one_cycle` lap (never carried over from a previous cycle), so
|
|
456
|
+
this only matters within one already-running cycle, not indefinitely."""
|
|
457
|
+
scheduled = next_scheduled_shutdown(config, datetime.now())
|
|
458
|
+
if scheduled is None:
|
|
459
|
+
await asyncio.Event().wait()
|
|
460
|
+
raise AssertionError("unreachable: the Event above is never set")
|
|
461
|
+
delay = max((scheduled.at - datetime.now()).total_seconds(), 0.0)
|
|
462
|
+
logger.info(
|
|
463
|
+
"IBController > scheduled %s at %s (in %.0fs)",
|
|
464
|
+
scheduled.action.name,
|
|
465
|
+
scheduled.at,
|
|
466
|
+
delay,
|
|
467
|
+
)
|
|
468
|
+
await asyncio.sleep(delay)
|
|
469
|
+
return (
|
|
470
|
+
ShutdownCause.COLD_RESTART
|
|
471
|
+
if scheduled.action is ScheduledAction.COLD_RESTART
|
|
472
|
+
else ShutdownCause.TIDY_CLOSEDOWN
|
|
473
|
+
)
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
async def _run_one_cycle( # noqa: PLR0915
|
|
477
|
+
config: Config,
|
|
478
|
+
agent_jar: str | Path,
|
|
479
|
+
labels: Labels,
|
|
480
|
+
*,
|
|
481
|
+
restart_hash: str | None = None,
|
|
482
|
+
) -> tuple[ShutdownCause, str]:
|
|
483
|
+
"""One lap: launch -> login (+ background watcher) -> settings (if
|
|
484
|
+
configured) -> wait until something ends -> clean shutdown. Returns
|
|
485
|
+
`(cause, settings_dir)` so `run_control_loop` can check `is_restart`
|
|
486
|
+
against the same directory this lap actually used. Propagates any
|
|
487
|
+
exception from login itself (`LoginError`/`LoginFailedError`/
|
|
488
|
+
`TimeoutError`) unchanged -- a failed login is not a `ShutdownCause`.
|
|
489
|
+
|
|
490
|
+
`restart_hash`, when set, is passed to `launch_instance` (see
|
|
491
|
+
`run_control_loop`) and to `LoginManager` as `restart_expected` rather
|
|
492
|
+
than letting `login.py` re-derive the same fact from a later filesystem
|
|
493
|
+
check -- Gateway can delete the marker quickly as part of a successful
|
|
494
|
+
silent relogin, so a later check could wrongly find it gone and attempt
|
|
495
|
+
a full credential fill on top of an already-succeeding relogin."""
|
|
496
|
+
state = StartupState.LAUNCHING
|
|
497
|
+
launched = await launch_instance(config, agent_jar, restart_hash=restart_hash)
|
|
498
|
+
manager = LoginManager(
|
|
499
|
+
config,
|
|
500
|
+
labels,
|
|
501
|
+
launched.dispatcher,
|
|
502
|
+
settings_dir=launched.settings_dir,
|
|
503
|
+
restart_expected=restart_hash is not None,
|
|
504
|
+
)
|
|
505
|
+
registry = _build_registry(config, labels, manager)
|
|
506
|
+
watcher = asyncio.ensure_future(
|
|
507
|
+
watch_for_unprompted_windows(registry, launched.dispatcher)
|
|
508
|
+
)
|
|
509
|
+
diagnostics_watcher = asyncio.ensure_future(
|
|
510
|
+
watch_for_diagnostics(
|
|
511
|
+
registry,
|
|
512
|
+
launched.dispatcher,
|
|
513
|
+
config.diagnostic_scope,
|
|
514
|
+
config.diagnostic_when,
|
|
515
|
+
)
|
|
516
|
+
)
|
|
517
|
+
cause = ShutdownCause.REQUESTED
|
|
518
|
+
# Spans the whole cycle (not just the post-login READY wait) so
|
|
519
|
+
# `_run_phase_watching_process` can race login/settings against it too.
|
|
520
|
+
process_done = asyncio.ensure_future(launched.process.wait())
|
|
521
|
+
try:
|
|
522
|
+
try:
|
|
523
|
+
_log_transition(state, StartupState.LOGGING_IN)
|
|
524
|
+
state = StartupState.LOGGING_IN
|
|
525
|
+
await _run_phase_watching_process(
|
|
526
|
+
manager.run(),
|
|
527
|
+
process_done=process_done,
|
|
528
|
+
dispatcher_tasks=launched.dispatcher.tasks,
|
|
529
|
+
process=launched.process,
|
|
530
|
+
)
|
|
531
|
+
logger.info("IBController > login completed, state=%s", manager.state)
|
|
532
|
+
|
|
533
|
+
_log_transition(state, StartupState.APPLYING_SETTINGS)
|
|
534
|
+
state = StartupState.APPLYING_SETTINGS
|
|
535
|
+
await _run_phase_watching_process(
|
|
536
|
+
_apply_declarative_settings_or_log(
|
|
537
|
+
launched, labels, config, manager.main_window_id
|
|
538
|
+
),
|
|
539
|
+
process_done=process_done,
|
|
540
|
+
dispatcher_tasks=launched.dispatcher.tasks,
|
|
541
|
+
process=launched.process,
|
|
542
|
+
)
|
|
543
|
+
|
|
544
|
+
_log_transition(state, StartupState.READY)
|
|
545
|
+
state = StartupState.READY
|
|
546
|
+
|
|
547
|
+
scheduled_shutdown = asyncio.ensure_future(
|
|
548
|
+
_sleep_until_scheduled_shutdown(config)
|
|
549
|
+
)
|
|
550
|
+
try:
|
|
551
|
+
finished = await _wait_for_first_completion(
|
|
552
|
+
main_tasks=[watcher, scheduled_shutdown],
|
|
553
|
+
process_done=process_done,
|
|
554
|
+
dispatcher_tasks=launched.dispatcher.tasks,
|
|
555
|
+
process=launched.process,
|
|
556
|
+
)
|
|
557
|
+
finally:
|
|
558
|
+
scheduled_shutdown.cancel()
|
|
559
|
+
with contextlib.suppress(asyncio.CancelledError):
|
|
560
|
+
await scheduled_shutdown
|
|
561
|
+
|
|
562
|
+
if finished is watcher:
|
|
563
|
+
exc = watcher.exception()
|
|
564
|
+
cause = ShutdownCause.LOGIN_FAILED
|
|
565
|
+
logger.warning("background watcher ended, raising=%r", exc)
|
|
566
|
+
elif finished is scheduled_shutdown:
|
|
567
|
+
cause = scheduled_shutdown.result()
|
|
568
|
+
logger.warning(
|
|
569
|
+
"IBController > scheduled shutdown fired: %s", cause.name
|
|
570
|
+
)
|
|
571
|
+
elif finished is process_done or launched.process.returncode is not None:
|
|
572
|
+
# A restart kills the process and every socket it held at once,
|
|
573
|
+
# so `asyncio.wait`'s FIRST_COMPLETED race between
|
|
574
|
+
# `process_done` and the event-reader noticing EOF isn't a
|
|
575
|
+
# reliable "process died" vs "connection dropped" signal on its
|
|
576
|
+
# own -- `returncode` is ground truth, and
|
|
577
|
+
# `_wait_for_first_completion` already gave it a short grace
|
|
578
|
+
# period to catch up before returning.
|
|
579
|
+
cause = ShutdownCause.PROCESS_EXITED
|
|
580
|
+
logger.warning(
|
|
581
|
+
"agent process exited on its own (returncode=%s)",
|
|
582
|
+
launched.process.returncode,
|
|
583
|
+
)
|
|
584
|
+
else:
|
|
585
|
+
cause = ShutdownCause.CONNECTION_LOST
|
|
586
|
+
logger.warning(
|
|
587
|
+
"dispatcher background task ended unexpectedly, process still "
|
|
588
|
+
"alive: %r",
|
|
589
|
+
finished,
|
|
590
|
+
)
|
|
591
|
+
except _PhaseAborted as exc:
|
|
592
|
+
# Process/connection died during login or settings application --
|
|
593
|
+
# same causes the READY-state wait above already handles, just
|
|
594
|
+
# reached from an earlier phase.
|
|
595
|
+
cause = exc.cause
|
|
596
|
+
logger.warning(
|
|
597
|
+
"IBController > %s during %s -- aborting this cycle",
|
|
598
|
+
cause.name,
|
|
599
|
+
state.name,
|
|
600
|
+
)
|
|
601
|
+
except OPERATIONAL_ERRORS as exc:
|
|
602
|
+
# An anticipated failure (bad credentials, a settings/launch problem)
|
|
603
|
+
# that `cli.py` will already report as a clean one-line message --
|
|
604
|
+
# log it plainly here too, no traceback, so it doesn't read as a bug.
|
|
605
|
+
logger.warning("IBController > cycle aborted: %s", exc)
|
|
606
|
+
raise
|
|
607
|
+
except Exception:
|
|
608
|
+
# `cause` defaults to REQUESTED (placeholder, above) -- log here so a
|
|
609
|
+
# genuine mid-cycle failure isn't misreported as a clean requested
|
|
610
|
+
# stop by the unconditional log line after `finally`. `except
|
|
611
|
+
# Exception` (not `BaseException`) leaves `asyncio.CancelledError`
|
|
612
|
+
# untouched, so a real cancellation still propagates.
|
|
613
|
+
logger.exception(
|
|
614
|
+
"IBController > control loop cycle aborted by an unhandled exception"
|
|
615
|
+
)
|
|
616
|
+
raise
|
|
617
|
+
finally:
|
|
618
|
+
process_done.cancel()
|
|
619
|
+
with contextlib.suppress(asyncio.CancelledError):
|
|
620
|
+
await process_done
|
|
621
|
+
_log_transition(state, StartupState.SHUTTING_DOWN)
|
|
622
|
+
state = StartupState.SHUTTING_DOWN
|
|
623
|
+
diagnostics_watcher.cancel()
|
|
624
|
+
with contextlib.suppress(asyncio.CancelledError):
|
|
625
|
+
await diagnostics_watcher
|
|
626
|
+
await clean_shutdown(
|
|
627
|
+
launched,
|
|
628
|
+
program=config.program,
|
|
629
|
+
logged_in=manager.is_logged_in(),
|
|
630
|
+
labels=labels.shutdown,
|
|
631
|
+
)
|
|
632
|
+
watcher.cancel()
|
|
633
|
+
with contextlib.suppress(asyncio.CancelledError):
|
|
634
|
+
await watcher
|
|
635
|
+
logger.info("IBController > control loop cycle stopped: %s", cause.name)
|
|
636
|
+
return cause, launched.settings_dir
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# Built-in declarative settings -- always loaded (settings.load_builtin_settings_file),
|
|
2
|
+
# always applied (control_loop._apply_declarative_settings), regardless of whether
|
|
3
|
+
# Config.settings_file is set. Each entry below only actually fires when its own
|
|
4
|
+
# value_from_config resolves to something other than None (settings.py's own
|
|
5
|
+
# _resolve_value/skip-on-None rule) -- so this file is a safe no-op on any install
|
|
6
|
+
# that hasn't set the matching Config field.
|
|
7
|
+
#
|
|
8
|
+
# Overriding a built-in entry: if a future TWS/Gateway release moves a control (a
|
|
9
|
+
# different tree_path/label, say), add an entry to your own ibkr_settings.toml with
|
|
10
|
+
# the SAME value_from_config as the built-in you're replacing -- settings.py's merge
|
|
11
|
+
# rule matches on that field and uses your entry instead of this one, rather than
|
|
12
|
+
# applying both. See ibkr_settings.toml.example for a worked override example.
|
|
13
|
+
#
|
|
14
|
+
# See that same file for the entry-field reference (tree_path/action/label/
|
|
15
|
+
# label_ref/value/value_from_config/field_index) -- not repeated here.
|
|
16
|
+
|
|
17
|
+
# auto_logoff_time is applied BEFORE auto_restart_time -- both write the same
|
|
18
|
+
# Lock and Exit radio-button pair, so if a deployment sets both fields, the
|
|
19
|
+
# last one applied wins the shared radio group. Keeping auto_logoff_time
|
|
20
|
+
# first here makes auto_restart_time win when both are set, matching IBC's
|
|
21
|
+
# own stated precedence ("if values are supplied for both, only the
|
|
22
|
+
# auto-restart time is set"). Do not reorder these two entries.
|
|
23
|
+
[[settings]]
|
|
24
|
+
tree_path = "Lock and Exit"
|
|
25
|
+
action = "auto_logoff_time"
|
|
26
|
+
value_from_config = "auto_logoff_time"
|
|
27
|
+
|
|
28
|
+
[[settings]]
|
|
29
|
+
tree_path = "Lock and Exit"
|
|
30
|
+
action = "auto_restart_time"
|
|
31
|
+
value_from_config = "auto_restart_time"
|
|
32
|
+
|
|
33
|
+
[[settings]]
|
|
34
|
+
tree_path = "API/Settings"
|
|
35
|
+
action = "toggle"
|
|
36
|
+
label_ref = "read_only_api"
|
|
37
|
+
value_from_config = "read_only_api"
|