ser2netweb 3.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. ser2netweb/__init__.py +20 -0
  2. ser2netweb/__main__.py +6 -0
  3. ser2netweb/accounts.py +503 -0
  4. ser2netweb/cli.py +489 -0
  5. ser2netweb/clock.py +128 -0
  6. ser2netweb/config.py +4491 -0
  7. ser2netweb/console.py +181 -0
  8. ser2netweb/datalock.py +181 -0
  9. ser2netweb/engine/__init__.py +0 -0
  10. ser2netweb/engine/access.py +173 -0
  11. ser2netweb/engine/bindhint.py +115 -0
  12. ser2netweb/engine/bridge.py +1928 -0
  13. ser2netweb/engine/cluster.py +1140 -0
  14. ser2netweb/engine/modbus.py +530 -0
  15. ser2netweb/engine/modbus_gateway.py +1465 -0
  16. ser2netweb/engine/mqtt_pub.py +632 -0
  17. ser2netweb/engine/netguard.py +805 -0
  18. ser2netweb/engine/netinfo.py +199 -0
  19. ser2netweb/engine/portdiag.py +414 -0
  20. ser2netweb/engine/portlist.py +471 -0
  21. ser2netweb/engine/protocols/__init__.py +28 -0
  22. ser2netweb/engine/protocols/base.py +117 -0
  23. ser2netweb/engine/protocols/raw.py +12 -0
  24. ser2netweb/engine/protocols/rfc2217.py +586 -0
  25. ser2netweb/engine/protocols/telnet.py +172 -0
  26. ser2netweb/engine/serial_compat.py +598 -0
  27. ser2netweb/engine/serial_io.py +210 -0
  28. ser2netweb/engine/serial_link.py +414 -0
  29. ser2netweb/engine/sockopts.py +103 -0
  30. ser2netweb/engine/sparkplug.py +538 -0
  31. ser2netweb/engine/sparkplug_node.py +550 -0
  32. ser2netweb/engine/supervisor.py +533 -0
  33. ser2netweb/engine/tls.py +232 -0
  34. ser2netweb/envvars.py +57 -0
  35. ser2netweb/explain.py +117 -0
  36. ser2netweb/healthcheck.py +78 -0
  37. ser2netweb/i18n.py +110 -0
  38. ser2netweb/importer.py +550 -0
  39. ser2netweb/layout.py +118 -0
  40. ser2netweb/licences.py +86 -0
  41. ser2netweb/locale/tr.json +1569 -0
  42. ser2netweb/logsetup.py +253 -0
  43. ser2netweb/logtext.py +23 -0
  44. ser2netweb/notify.py +761 -0
  45. ser2netweb/optional.py +76 -0
  46. ser2netweb/reload.py +295 -0
  47. ser2netweb/runtime.py +762 -0
  48. ser2netweb/sdnotify.py +85 -0
  49. ser2netweb/ser2net_import.py +2001 -0
  50. ser2netweb/signins.py +98 -0
  51. ser2netweb/standalone.py +154 -0
  52. ser2netweb/state.py +1457 -0
  53. ser2netweb/storage.py +130 -0
  54. ser2netweb/textfold.py +51 -0
  55. ser2netweb/tlsreload.py +37 -0
  56. ser2netweb/watchdog.py +201 -0
  57. ser2netweb/web/__init__.py +0 -0
  58. ser2netweb/web/api.py +793 -0
  59. ser2netweb/web/auth.py +699 -0
  60. ser2netweb/web/certinfo.py +144 -0
  61. ser2netweb/web/httpguard.py +413 -0
  62. ser2netweb/web/ldap_auth.py +556 -0
  63. ser2netweb/web/oidc_auth.py +330 -0
  64. ser2netweb/web/proxy.py +398 -0
  65. ser2netweb/web/routes/__init__.py +78 -0
  66. ser2netweb/web/routes/cluster.py +669 -0
  67. ser2netweb/web/routes/common.py +359 -0
  68. ser2netweb/web/routes/console.py +451 -0
  69. ser2netweb/web/routes/forms.py +853 -0
  70. ser2netweb/web/routes/login.py +573 -0
  71. ser2netweb/web/routes/mappings.py +470 -0
  72. ser2netweb/web/routes/ops.py +374 -0
  73. ser2netweb/web/routes/settings.py +2135 -0
  74. ser2netweb/web/routes/system.py +308 -0
  75. ser2netweb/web/routes/totp.py +282 -0
  76. ser2netweb/web/selfsigned.py +129 -0
  77. ser2netweb/web/server.py +483 -0
  78. ser2netweb/web/static/app.css +327 -0
  79. ser2netweb/web/static/app.js +1102 -0
  80. ser2netweb/web/static/htmx.min.js +1 -0
  81. ser2netweb/web/static/points.js +218 -0
  82. ser2netweb/web/static/theme.js +7 -0
  83. ser2netweb/web/static/xterm-addon-fit.js +2 -0
  84. ser2netweb/web/static/xterm.css +218 -0
  85. ser2netweb/web/static/xterm.js +2 -0
  86. ser2netweb/web/templates/_cluster_body.html +109 -0
  87. ser2netweb/web/templates/_console.html +21 -0
  88. ser2netweb/web/templates/_footer.html +13 -0
  89. ser2netweb/web/templates/_login_notice.html +8 -0
  90. ser2netweb/web/templates/_macros.html +21 -0
  91. ser2netweb/web/templates/_mapping_form.html +477 -0
  92. ser2netweb/web/templates/_mapping_log.html +28 -0
  93. ser2netweb/web/templates/_mapping_traces.html +44 -0
  94. ser2netweb/web/templates/_mappings_body.html +149 -0
  95. ser2netweb/web/templates/_modbus_read.html +18 -0
  96. ser2netweb/web/templates/_ports_table.html +37 -0
  97. ser2netweb/web/templates/_serial_port_select.html +48 -0
  98. ser2netweb/web/templates/_shell.html +41 -0
  99. ser2netweb/web/templates/audit.html +55 -0
  100. ser2netweb/web/templates/base.html +77 -0
  101. ser2netweb/web/templates/dashboard.html +72 -0
  102. ser2netweb/web/templates/licenses.html +18 -0
  103. ser2netweb/web/templates/login.html +32 -0
  104. ser2netweb/web/templates/login_totp.html +23 -0
  105. ser2netweb/web/templates/settings.html +830 -0
  106. ser2netweb/web/templates/setup.html +43 -0
  107. ser2netweb/web/templates/totp.html +41 -0
  108. ser2netweb/web/totp.py +94 -0
  109. ser2netweb/winservice.py +1793 -0
  110. ser2netweb-3.0.0.dist-info/METADATA +120 -0
  111. ser2netweb-3.0.0.dist-info/RECORD +121 -0
  112. ser2netweb-3.0.0.dist-info/WHEEL +5 -0
  113. ser2netweb-3.0.0.dist-info/entry_points.txt +2 -0
  114. ser2netweb-3.0.0.dist-info/licenses/COMMERCIAL_LICENSE.md +94 -0
  115. ser2netweb-3.0.0.dist-info/licenses/LICENSE +662 -0
  116. ser2netweb-3.0.0.dist-info/licenses/NOTICE +20 -0
  117. ser2netweb-3.0.0.dist-info/licenses/THIRD-PARTY-NOTICES.md +105 -0
  118. ser2netweb-3.0.0.dist-info/licenses/tools/licenses/npm/htmx.org.txt +13 -0
  119. ser2netweb-3.0.0.dist-info/licenses/tools/licenses/npm/xterm-addon-fit.txt +19 -0
  120. ser2netweb-3.0.0.dist-info/licenses/tools/licenses/npm/xterm.txt +21 -0
  121. ser2netweb-3.0.0.dist-info/top_level.txt +1 -0
ser2netweb/config.py ADDED
@@ -0,0 +1,4491 @@
1
+ """Configuration data model + atomic JSON persistence for ser2netweb.
2
+
3
+ The model is deliberately split into explicit typed fields (borrowed from
4
+ the classic ser2net's connection model) rather than gensio stack-strings, so the web UI can
5
+ present every option directly and validate it.
6
+
7
+ Persistence is plain stdlib JSON with an atomic write (tempfile -> fsync ->
8
+ os.replace -> parent-dir fsync) so a crash mid-write can never truncate dozens of
9
+ mappings. config.json is the single source of truth; earlier versions are kept in
10
+ backups/ (ConfigStore) but never read back unless an admin restores one, so deleting
11
+ config.json still resets all config.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import calendar
16
+ import contextlib
17
+ import dataclasses
18
+ import functools
19
+ import hashlib
20
+ import hmac
21
+ import ipaddress
22
+ import json
23
+ import math
24
+ import ntpath
25
+ import os
26
+ import posixpath
27
+ import re
28
+ import secrets
29
+ import socket
30
+ import tempfile
31
+ import time
32
+ import typing
33
+ import unicodedata
34
+ import urllib.parse
35
+ import uuid
36
+ from dataclasses import dataclass, field, asdict
37
+ from typing import Any, Iterable, Optional, Union
38
+ from . import __version__, envvars, i18n
39
+ from .i18n import LANGUAGES, _
40
+ from .textfold import fold # how names compare; still importable as ser2netweb.config.fold
41
+
42
+ # ---------------------------------------------------------------------------
43
+ # Allowed value sets (mirror pyserial; kept here so config validation has no
44
+ # hard import dependency on pyserial).
45
+ # ---------------------------------------------------------------------------
46
+ BYTESIZES = (5, 6, 7, 8)
47
+ PARITIES = ("N", "E", "O", "M", "S")
48
+ PARITY_LABELS = {"N": "None", "E": "Even", "O": "Odd", "M": "Mark", "S": "Space"}
49
+ STOPBITS = (1, 1.5, 2)
50
+ FLOWCONTROLS = ("none", "rtscts", "xonxoff", "dsrdtr")
51
+ LINE_STATES = ("on", "off", "keep")
52
+ PROTOCOLS = ("raw", "telnet", "rfc2217", "modbus")
53
+ COMMON_BAUDRATES = (
54
+ 300, 1200, 2400, 4800, 9600, 19200, 38400, 57600,
55
+ 115200, 230400, 460800, 921600,
56
+ )
57
+ MAX_TIMER_S = 365 * 86400 # upper bound for the per-connection timers (one year)
58
+ # a mapping's no-data alarm (MappingOptions.no_data_alarm_s): looked at every 2 s
59
+ NO_DATA_MIN_S = 10
60
+ NO_DATA_MAX_S = 7 * 86400
61
+ # Modbus gateway response timeout: the default, and what an edit may set
62
+ MODBUS_TIMEOUT_DEFAULT_S = 1.0
63
+ MODBUS_TIMEOUT_RANGE_S = (0.01, 60.0)
64
+ # web-UI sign-in session lifetime (AppConfig.session_timeout_s)
65
+ SESSION_TIMEOUT_DEFAULT_S = 8 * 3600
66
+ SESSION_TIMEOUT_MIN_S = 5 * 60
67
+ SESSION_TIMEOUT_MAX_S = 30 * 86400
68
+ # Strict-Transport-Security max-age with admin TLS (AdminUI.hsts_max_age): one year, as
69
+ # before it could be set; 0 sends max-age=0 (HSTS off, browsers drop what they stored)
70
+ HSTS_MAX_AGE_DEFAULT = 365 * 86400
71
+ HSTS_MAX_AGE_MAX = 2 * 365 * 86400
72
+ # LDAP/OIDC accounts not signed in for this many days are removed (AppConfig
73
+ # .external_users_expire_days, ser2netweb/signins.py); 0 = never, the default: an admin turns it
74
+ # on. At least the longest session lifetime, so no live session ends that way.
75
+ EXPIRE_DAYS_DEFAULT = 0
76
+ EXPIRE_DAYS_MIN = SESSION_TIMEOUT_MAX_S // 86400
77
+ EXPIRE_DAYS_MAX = 3650
78
+ # trace files not written for this many days are deleted by the hourly maintenance
79
+ # (AppConfig.trace_max_age_days, AppState.run_trace_maintenance); 0 = kept, the default
80
+ TRACE_DAYS_MAX = 3650
81
+ # log and audit records and configuration backups older than this many days are deleted by
82
+ # the hourly maintenance (AppConfig.log_max_age_days, AppState.run_retention); 0 = only
83
+ # their size and count limits, the default
84
+ LOG_DAYS_MAX = 3650
85
+
86
+
87
+ class ConfigError(ValueError):
88
+ """Raised when a config object fails validation. Message is user-facing."""
89
+
90
+
91
+ class ConfigWriteError(ConfigError):
92
+ """A change that validates but can't be written to config.json (a full or read-only
93
+ disk, a file another program holds): AppState.asave raises it for the OSError, so
94
+ whoever shows a refused save shows this one too. Message is user-facing."""
95
+
96
+
97
+ def _new_id() -> str:
98
+ return uuid.uuid4().hex[:12]
99
+
100
+
101
+ # A mapping id names the mapping's own files (logs/<id>.log, traces/<id>-<name>) and is
102
+ # part of URLs, so it is 1-64 ASCII letters, digits or "_": never a path separator, ".",
103
+ # ":" or NUL, and no "-" either (a mapping's traces are found by their "<id>-" prefix,
104
+ # so an id "a" must not also match the traces of a mapping "a-b"). Generated ids
105
+ # (_new_id) are 12 hex digits.
106
+ _MAPPING_ID = re.compile(r"[A-Za-z0-9_]{1,64}")
107
+ # device names Windows reserves, with any extension too (logs/NUL.log is the NUL device)
108
+ _WINDOWS_DEVICE_NAME = re.compile(r"(?i:con|prn|aux|nul|com[0-9]|lpt[0-9])")
109
+
110
+
111
+ def valid_mapping_id(value: Any) -> bool:
112
+ """True for a mapping id that is safe as a file name and in a URL (see _MAPPING_ID)."""
113
+ return (isinstance(value, str) and _MAPPING_ID.fullmatch(value) is not None
114
+ and _WINDOWS_DEVICE_NAME.fullmatch(value) is None)
115
+
116
+
117
+ def _mapping_id_from(value: Any) -> str:
118
+ """The id of a mapping read from a config file, an import or a form: the value when it
119
+ is a valid id, a new one when there is none, else one derived from the value. The
120
+ derived id is the same on every load, so the mapping keeps its log and traces even
121
+ when the config is not saved again."""
122
+ if valid_mapping_id(value):
123
+ return value
124
+ if not value:
125
+ return _new_id()
126
+ return hashlib.sha256(repr(value).encode("utf-8", "backslashreplace")).hexdigest()[:12]
127
+
128
+
129
+ def mapping_file(directory: str, mapping_id: Any, suffix: str = "") -> str:
130
+ """<directory>/<mapping id><suffix>: a mapping's own file, its log (logs/<id>.log) or a
131
+ trace (traces/<id>-<name>). Raises ValueError for an invalid mapping id, or when the
132
+ result would not be a file directly in `directory`, so no id or name can point
133
+ anywhere else."""
134
+ if not valid_mapping_id(mapping_id):
135
+ raise ValueError(f"not a valid mapping id: {mapping_id!r}")
136
+ name = f"{mapping_id}{suffix}"
137
+ path = os.path.join(directory, name)
138
+ if (os.path.basename(name) != name
139
+ or os.path.dirname(os.path.abspath(path)) != os.path.abspath(directory)):
140
+ raise ValueError(f"not a file in {directory}: {name!r}")
141
+ return path
142
+
143
+
144
+ def lock_down_dir(directory: str) -> None:
145
+ """Restrict a state directory to its owner so the secrets it holds (secret_key,
146
+ password hash in config.json) and captured serial logs are not readable by other
147
+ local users.
148
+
149
+ POSIX: chmod 0700. Windows: ``icacls`` removes inherited ACEs and grants Full
150
+ only to the current user, SYSTEM and the Administrators group, with inheritance
151
+ so child files (config.json, the atomic temp files, per-mapping logs) pick up the
152
+ same restriction. Best-effort: silently degrades if it cannot be applied (the
153
+ posix path was the only one protected before — Windows was left world-readable
154
+ despite the docs claiming 0600). A service's data directory as --install-service
155
+ leaves it (only SYSTEM and Administrators can use it, see winservice.service_folder)
156
+ is left as it is: an elevated run (--set-password) must not give its own account's
157
+ unelevated programs a way to change what the service runs. Any other folder gets
158
+ the grant for the current user, also one an elevated administrator created private
159
+ to itself, so that account can still use it unelevated.
160
+ """
161
+ if os.name == "posix":
162
+ with contextlib.suppress(OSError):
163
+ os.chmod(directory, 0o700)
164
+ return
165
+ if os.name == "nt":
166
+ user = os.environ.get("USERNAME") or ""
167
+ if not user or _service_folder(directory):
168
+ return
169
+ # Set the ACL on the directory only (NOT /T): /inheritance:r drops inherited
170
+ # ACEs, and the (OI)(CI) grants both protect the dir and propagate to child
171
+ # files/dirs, so config.json + the atomic temp files + per-mapping logs are
172
+ # created owner-private. Applying (OI)(CI) ACEs to existing *files* via /T
173
+ # corrupts their ACL (inheritance flags are meaningless on a leaf and left the
174
+ # owner unable to read config.json), so we rely on inheritance instead.
175
+ # SIDs are locale-independent: *S-1-5-18 = SYSTEM, *S-1-5-32-544 = Administrators.
176
+ _icacls_private(directory, [f"{user}:(OI)(CI)F", "*S-1-5-18:(OI)(CI)F",
177
+ "*S-1-5-32-544:(OI)(CI)F"])
178
+
179
+
180
+ def lock_down_file(path: str) -> None:
181
+ """Restrict one file to its owner, for config.json in a folder lock_down_dir() is not
182
+ applied to (ConfigStore: a folder that isn't ser2netweb's own). POSIX: chmod 0600.
183
+ Windows: inherited ACEs removed, Full for the current user, SYSTEM and Administrators.
184
+ Best effort, as lock_down_dir(); a file a service's --install-service left admin-only
185
+ is left as it is."""
186
+ if os.name == "posix":
187
+ with contextlib.suppress(OSError):
188
+ os.chmod(path, 0o600)
189
+ return
190
+ if os.name == "nt":
191
+ user = os.environ.get("USERNAME") or ""
192
+ if not user or _service_folder(path):
193
+ return
194
+ _icacls_private(path, [f"{user}:F", "*S-1-5-18:F", "*S-1-5-32-544:F"])
195
+
196
+
197
+ def _icacls_private(path: str, grants: list[str]) -> None:
198
+ """icacls <path> /inheritance:r with these grants (Windows; errors are ignored)."""
199
+ import subprocess
200
+
201
+ # by full path: Windows searches an unqualified name in the program's own
202
+ # folder and the current directory before System32, so an icacls.exe planted
203
+ # next to ser2netweb.exe (e.g. in Downloads) would otherwise run instead
204
+ root = os.environ.get("SystemRoot") or os.environ.get("WINDIR") or r"C:\Windows"
205
+ icacls = os.path.join(root, "System32", "icacls.exe")
206
+ cmd = [icacls if os.path.isfile(icacls) else "icacls", path, "/inheritance:r"]
207
+ for g in grants:
208
+ cmd += ["/grant:r", g]
209
+ cmd += ["/C", "/Q"]
210
+ with contextlib.suppress(Exception):
211
+ subprocess.run(cmd, capture_output=True, timeout=15, check=False)
212
+
213
+
214
+ # What ser2netweb itself puts in its data directory (the folder of config.json, which may have
215
+ # another name with --config): all.log and audit.log with their rotated copies, logs/,
216
+ # traces/, backups/, tls/ (a generated certificate), admin-requests/ (ser2netweb/accounts.py), the
217
+ # instance lock (ser2netweb/datalock.py), setup-code.txt (ser2netweb/state.py), the Sparkplug B bdSeq
218
+ # file (ser2netweb/engine/sparkplug_node.py), hang.log of the loop watchdog (ser2netweb/watchdog.py), the
219
+ # Windows service's service.log (ser2netweb/winservice.py), each with a kept older copy, and the
220
+ # temporary files of its atomic writes and log trims; and what a file system or a desktop
221
+ # adds to any folder.
222
+ OWN_NAMES = frozenset({"backups", "logs", "traces", "tls", "admin-requests", "ser2net.lock",
223
+ "setup-code.txt", "sparkplug-bdseq.json", "hang.log", "service.log",
224
+ "lost+found", ".DS_Store", "Thumbs.db", "desktop.ini"})
225
+ _OWN_NAME = re.compile(r"(?:all|audit)\.log(?:\.\d+|\.tmp)?|(?:hang|service)\.log\.1|"
226
+ r"\.cfg\..+\.tmp|setup-code\.txt\.tmp|sparkplug-bdseq\.json\.tmp")
227
+
228
+
229
+ def foreign_entries(directory: str, config_name: str = "config.json") -> Optional[list[str]]:
230
+ """The names in `directory` that ser2netweb doesn't put there itself (sorted; [] for a data
231
+ directory of its own), or None when it can't be listed."""
232
+ try:
233
+ names = os.listdir(directory)
234
+ except OSError:
235
+ return None
236
+ return sorted(n for n in names
237
+ if n != config_name and n not in OWN_NAMES and not _OWN_NAME.fullmatch(n))
238
+
239
+
240
+ def others_can_use(directory: str) -> bool:
241
+ """Accounts other than this one (and, on Windows, SYSTEM / Administrators) have some
242
+ access to `directory` (True when that can't be told)."""
243
+ if os.name == "posix":
244
+ try:
245
+ return bool(os.stat(directory).st_mode & 0o077)
246
+ except OSError:
247
+ return True
248
+ from . import winservice # the Windows API through ctypes
249
+
250
+ try:
251
+ sec = winservice.read_security(directory)
252
+ me = winservice._current_user_sid()
253
+ except Exception:
254
+ return True
255
+ if sec is None:
256
+ return True
257
+ return any(sid != me for sid in winservice.acl_offenders(sec[1], None,
258
+ rights=winservice.ANY_ACCESS))
259
+
260
+
261
+ def _service_folder(directory: str) -> bool:
262
+ """Windows: `directory` is a service's data directory as --install-service leaves it,
263
+ or inherits from one (False when that can't be told)."""
264
+ from . import winservice # the Windows API through ctypes
265
+
266
+ try:
267
+ sec = winservice.read_security(directory)
268
+ except Exception: # best effort, like the rest of lock_down_dir
269
+ return False
270
+ return sec is not None and winservice.service_folder(*sec)
271
+
272
+
273
+ def _choices(values) -> str:
274
+ """Allowed values for a message: "5, 6, 7, 8", not the tuple's "(5, 6, 7, 8)"."""
275
+ return ", ".join(str(v) for v in values)
276
+
277
+
278
+ def _is_int(value: Any) -> bool:
279
+ return isinstance(value, int) and not isinstance(value, bool)
280
+
281
+
282
+ def _is_number(value: Any) -> bool:
283
+ return isinstance(value, (int, float)) and not isinstance(value, bool)
284
+
285
+
286
+ def _legacy_seconds(value: Any) -> Any:
287
+ """Normalize a timer that predates validation (idle_timeout_s). Before v2.7 the
288
+ REST API / import stored it unchecked and anything <= 0 meant "off", so an old
289
+ config may hold -1 or a numeric string. Map those onto the validated range rather
290
+ than refuse to start: a headless/Docker boot re-saves (= validates) the config."""
291
+ if isinstance(value, str):
292
+ try:
293
+ value = float(value.strip())
294
+ except ValueError:
295
+ return 0
296
+ if value.is_integer():
297
+ value = int(value)
298
+ if not _is_number(value) or not math.isfinite(value) or value < 0:
299
+ return 0
300
+ return min(value, MAX_TIMER_S)
301
+
302
+
303
+ # ---------------------------------------------------------------------------
304
+ # Reading JSON (config.json, the REST API, an import, a form) into the typed model
305
+ # ---------------------------------------------------------------------------
306
+ @dataclass(frozen=True)
307
+ class FieldNote:
308
+ """What reading a JSON object into the model did with one of its values (_read_fields),
309
+ by its path in that object ("serial.baudrate", "modbus_poll.points[0].scale"):
310
+ unknown - no setting of this version: left out (an edit warns about it);
311
+ converted - another JSON type, read as the field's type ("false" as false, "9600" as
312
+ 9600): an edit takes it as meant, loading config.json notes it;
313
+ invalid - can't be read as the field's type: `used` instead, and an edit refuses
314
+ it (`expected`: what it must be, a key of _EXPECTED);
315
+ replaced - not supported together with the other settings: `used` instead, with a
316
+ warning (`why` in English for the log, `warning` in the language of the
317
+ request that sent it);
318
+ switch_text - a protective switch (_PROTECTIVE) given as text: loading reads it as
319
+ before 2.9.0 (`used`: any text but "" is on), an edit refuses it.
320
+ A switch that is on only for JSON true (_ONLY_TRUE) given anything else is invalid
321
+ (`used`: false)."""
322
+ path: str
323
+ kind: str
324
+ value: Any = None
325
+ used: Any = None
326
+ expected: str = ""
327
+ why: str = ""
328
+ warning: str = ""
329
+
330
+ def text(self) -> str:
331
+ """The note in English, for the log (AppConfig.load_notes)."""
332
+ if self.kind == "unknown":
333
+ return f"{self.path} is not a setting of this version and is ignored"
334
+ if self.kind == "invalid":
335
+ return (f"{self.path} {_shown(self.value)} is not {_EXPECTED[self.expected]}: "
336
+ f"{_shown(self.used)} is used")
337
+ if self.kind == "converted":
338
+ text = f"{self.path} {_shown(self.value)} is read as {_shown(self.used)}"
339
+ if self.used is False and isinstance(self.value, str) and self.value.strip():
340
+ text += " (versions before 2.9.0 took any text but \"\" as true)"
341
+ return text
342
+ return self.why
343
+
344
+ def error(self) -> str:
345
+ """An edit's refusal of an invalid value, in the UI's language."""
346
+ if self.kind == "switch_text":
347
+ return _("{0}: use true or false (JSON booleans), not a string.").format(self.path)
348
+ return _("{0}: {1} is not {2}.").format(self.path, _shown(self.value),
349
+ _expected(self.expected))
350
+
351
+
352
+ # what a field of each kind must be (FieldNote.expected), for the log ...
353
+ _EXPECTED = {"bool": "true or false", "int": "a whole number", "number": "a finite number",
354
+ "str": "text", "object": "a JSON object"}
355
+
356
+
357
+ def _expected(kind: str) -> str:
358
+ """... and in the UI's language."""
359
+ return {"bool": _("true or false"), "int": _("a whole number"),
360
+ "number": _("a finite number"), "str": _("text"),
361
+ "object": _("a JSON object")}[kind]
362
+
363
+
364
+ def _shown(value: Any, limit: int = 60) -> str:
365
+ """A value as JSON writes it, cut to `limit` characters, for messages (escaped where
366
+ it holds text UTF-8 can't carry, an unpaired surrogate: the message must reach a page,
367
+ a JSON answer and the log)."""
368
+ try:
369
+ text = json.dumps(value, ensure_ascii=False)
370
+ text.encode("utf-8")
371
+ except UnicodeEncodeError:
372
+ text = json.dumps(value)
373
+ except (TypeError, ValueError):
374
+ text = repr(value).encode("utf-8", "backslashreplace").decode("utf-8")
375
+ return text if len(text) <= limit else text[:limit - 1] + "…"
376
+
377
+
378
+ _TRUE_WORDS = ("true", "1", "on", "yes")
379
+ _FALSE_WORDS = ("false", "0", "off", "no", "")
380
+ # Field metadata of a switch that protects something (login, TLS, certificate checks,
381
+ # read-only). Loading reads text in it as before 2.9.0 — on unless empty — so an upgrade
382
+ # never weakens a hand-edited config whose "false" in quotes kept it on; an edit refuses
383
+ # text in it, so a client sending "false" learns it was not taken (_read_fields)
384
+ _PROTECTIVE = {"protective": True}
385
+ # Field metadata of a switch that is on only for JSON true: what this release added and
386
+ # checked that way from the start (`is True`: a listener that takes both families, MQTT
387
+ # writes to the device, HSTS through a proxy ...). Any other value reads as false, with a
388
+ # note, and an edit refuses it, so no text or number turns one on
389
+ _ONLY_TRUE = {"only_true": True}
390
+
391
+
392
+ def _text_number(text: str) -> Any:
393
+ """The number a text spells (an int when it is one), else None."""
394
+ t = text.strip()
395
+ try:
396
+ return int(t)
397
+ except ValueError:
398
+ pass
399
+ try:
400
+ return float(t)
401
+ except ValueError:
402
+ return None
403
+
404
+
405
+ def _coerce(value: Any, hint: Any) -> tuple[bool, Any]:
406
+ """`value` read as a field of type `hint`: (True, the value) — converted from another
407
+ JSON type when that needs no guess — or (False, None) when it can't be. null is an
408
+ unset value: false, "", [] or {} (a number: _read_fields takes the field's default).
409
+ A list of text takes any value: a text is a one-entry list, anything else becomes
410
+ text, so no entry of an allow-list is ever dropped (unusable ones are reported where
411
+ they are used: client_networks)."""
412
+ if hint is bool:
413
+ if value is None:
414
+ return True, False
415
+ if isinstance(value, bool):
416
+ return True, value
417
+ if _is_int(value) and value in (0, 1):
418
+ return True, bool(value)
419
+ if isinstance(value, str):
420
+ word = value.strip().lower()
421
+ if word in _TRUE_WORDS:
422
+ return True, True
423
+ if word in _FALSE_WORDS:
424
+ return True, False
425
+ return False, None
426
+ if hint is int or hint is float:
427
+ if isinstance(value, str):
428
+ value = _text_number(value)
429
+ if isinstance(value, bool) or not _is_number(value):
430
+ return False, None
431
+ if isinstance(value, float):
432
+ if not math.isfinite(value):
433
+ return False, None
434
+ if hint is int and value.is_integer():
435
+ return True, int(value)
436
+ elif hint is float:
437
+ try:
438
+ float(value)
439
+ except OverflowError: # an int too large for any float
440
+ return False, None
441
+ return True, value
442
+ if hint is str:
443
+ if isinstance(value, str):
444
+ return True, value
445
+ if value is None:
446
+ return True, ""
447
+ if _is_number(value):
448
+ return True, str(value)
449
+ return False, None
450
+ origin = typing.get_origin(hint)
451
+ if origin is list and typing.get_args(hint) == (str,):
452
+ if value is None:
453
+ return True, []
454
+ if isinstance(value, str):
455
+ return True, [value] if value.strip() else []
456
+ items = value if isinstance(value, list) else [value]
457
+ return True, [v if isinstance(v, str) else _shown(v, 200) for v in items]
458
+ if origin is dict:
459
+ if value is None:
460
+ return True, {}
461
+ return (True, value) if isinstance(value, dict) else (False, None)
462
+ return True, value
463
+
464
+
465
+ def _kind_of(hint: Any) -> str:
466
+ return {bool: "bool", int: "int", float: "number", str: "str"}.get(hint, "object")
467
+
468
+
469
+ def _same(value: Any, read: Any) -> bool:
470
+ """True when reading `value` changed nothing worth a note (null for an empty value,
471
+ 4001.0 for 4001)."""
472
+ if value is None:
473
+ return read is False or (not isinstance(read, bool) and read in ("", [], {}))
474
+ if _is_number(value) and _is_number(read):
475
+ return value == read
476
+ return type(value) is type(read) and value == read
477
+
478
+
479
+ @functools.cache
480
+ def _hints(cls) -> dict[str, Any]:
481
+ return typing.get_type_hints(cls)
482
+
483
+
484
+ def _field_default(f: dataclasses.Field) -> Any:
485
+ if f.default is not dataclasses.MISSING:
486
+ return f.default
487
+ return f.default_factory() # type: ignore[operator]
488
+
489
+
490
+ def _read_fields(cls, d: dict[str, Any], notes: list[FieldNote], path: str = "", *,
491
+ skip: tuple = (), keep: tuple = (),
492
+ fallback: Optional[dict[str, Any]] = None) -> dict[str, Any]:
493
+ """Keyword arguments for cls() from the JSON object `d`: each field `d` has, read as the
494
+ field's type (_coerce), except `skip` (the caller reads those). A value that can't be
495
+ takes the field's default — a true/false field the value's truth instead, as earlier
496
+ versions took it, or the field's `fallback`. Keys that are no field (or `keep`, read
497
+ elsewhere) are left out. What it did is added to `notes`, paths prefixed with `path`."""
498
+ hints = _hints(cls)
499
+ out: dict[str, Any] = {}
500
+ names = set()
501
+ for f in dataclasses.fields(cls):
502
+ names.add(f.name)
503
+ if f.name in skip or f.name not in d:
504
+ continue
505
+ value, hint = d[f.name], hints[f.name]
506
+ if value is None and hint in (int, float): # a number left unset
507
+ out[f.name] = _field_default(f)
508
+ continue
509
+ if f.metadata.get("only_true") and value is not None and not isinstance(value, bool):
510
+ notes.append(FieldNote(path + f.name, "invalid", value, False, "bool"))
511
+ out[f.name] = False
512
+ continue
513
+ if isinstance(value, str) and f.metadata.get("protective"):
514
+ used = bool(value)
515
+ notes.append(FieldNote(
516
+ path + f.name, "switch_text", value, used,
517
+ why=f"{path}{f.name} {_shown(value)} is text, so it is read as "
518
+ f"{_shown(used)}, as before: this protective switch takes true or false "
519
+ "(without quotes)"))
520
+ out[f.name] = used
521
+ continue
522
+ ok, read = _coerce(value, hint)
523
+ if not ok:
524
+ if fallback and f.name in fallback:
525
+ read = fallback[f.name]
526
+ else:
527
+ read = bool(value) if hint is bool else _field_default(f)
528
+ notes.append(FieldNote(path + f.name, "invalid", value, read, _kind_of(hint)))
529
+ elif not _same(value, read):
530
+ notes.append(FieldNote(path + f.name, "converted", value, read))
531
+ out[f.name] = read
532
+ for key, value in d.items():
533
+ if key not in names and key not in keep:
534
+ notes.append(FieldNote(f"{path}{key}", "unknown", value))
535
+ return out
536
+
537
+
538
+ def _section(d: dict[str, Any], key: str, path: str = "") -> dict[str, Any]:
539
+ """The JSON object d[key]; an absent or empty one is {} (as before). Anything else
540
+ raises ConfigError: config.json then fails to load, as it did before, rather than
541
+ lose that part of it."""
542
+ value = d.get(key)
543
+ if not value:
544
+ return {}
545
+ if not isinstance(value, dict):
546
+ raise ConfigError(_("{0} must be a JSON object, not {1}.").format(path + key,
547
+ _shown(value)))
548
+ return value
549
+
550
+
551
+ def _objects(d: dict[str, Any], key: str, path: str = "") -> list[dict[str, Any]]:
552
+ """The JSON list of objects d[key]; an absent or empty one is []. Raises ConfigError
553
+ for anything else (see _section)."""
554
+ value = d.get(key)
555
+ if not value:
556
+ return []
557
+ if not isinstance(value, list):
558
+ raise ConfigError(_("{0} must be a JSON list, not {1}.").format(path + key,
559
+ _shown(value)))
560
+ for i, item in enumerate(value):
561
+ if not isinstance(item, dict):
562
+ raise ConfigError(_("{0} must be a JSON object, not {1}.").format(
563
+ f"{path}{key}[{i}]", _shown(item)))
564
+ return value
565
+
566
+
567
+ TRACE_DEFAULT_NAME = "trace.log"
568
+ # letters/digits of any script, "_", "." and "-"; path separators and the characters
569
+ # Windows forbids become "_" (the <id>- prefix also rules out reserved names like CON)
570
+ _TRACE_NAME_BAD = re.compile(r"[^\w.-]+")
571
+
572
+
573
+ def trace_file_name(value: Any) -> str:
574
+ """The file name a mapping's trace is written to, inside the data directory's
575
+ traces/ folder. Before v2.8 ``trace_both`` was a free path that any operator could
576
+ point at any file the service could write (config.json, a startup script...);
577
+ such a value now only contributes its last path component, sanitised. '' = off."""
578
+ raw = str(value or "").strip()
579
+ if not raw:
580
+ return ""
581
+ name = re.split(r"[\\/]", raw)[-1]
582
+ name = _TRACE_NAME_BAD.sub("_", name).strip(". ")[:80].rstrip(". ")
583
+ return name or TRACE_DEFAULT_NAME
584
+
585
+
586
+ # The data directory (the folder of config.json, set by ConfigStore): where a relative
587
+ # TLS file path in the configuration points. Before 2.9 such a path was looked up from the
588
+ # working directory (/app in Docker, the service's folder, System32 for the Windows
589
+ # service), so a data directory that moved or was restored elsewhere lost its files.
590
+ _data_dir: Optional[str] = None
591
+
592
+
593
+ def set_data_dir(path: Optional[str]) -> None:
594
+ global _data_dir
595
+ _data_dir = os.path.abspath(path) if path else None
596
+
597
+
598
+ def data_path(path: Any, data_dir: Optional[str] = None) -> str:
599
+ """The file a TLS path from the configuration names: an absolute path as it is, a
600
+ relative one in the data directory (``data_dir``, else ConfigStore's). A relative path
601
+ found only from the working directory, where releases before 2.9 looked for it, is
602
+ still used from there."""
603
+ path = str(path or "")
604
+ base = data_dir or _data_dir
605
+ if not path or os.path.isabs(path) or ntpath.isabs(path) or not base:
606
+ return path # (a Windows path is no file in the data directory elsewhere either)
607
+ inside = os.path.join(base, path)
608
+ if os.name == "nt": # "tls/cert.pem" -> <data>\tls\cert.pem (the way Windows reads it)
609
+ inside = os.path.normpath(inside)
610
+ if not os.path.exists(inside) and os.path.exists(path):
611
+ return path
612
+ return inside
613
+
614
+
615
+ def moved_tls_file(path: Any, data_dir: str) -> str:
616
+ """Where a TLS file of a data directory that was moved, or restored on another machine,
617
+ is now: for an absolute ``path`` that is not there and lies in a folder named "tls"
618
+ (where Settings → Admin TLS generates its certificate: <data dir>/tls/), the file of
619
+ that name in ``data_dir``'s tls folder, if it is there; else "". A path written on
620
+ Windows is read on any OS, and one written elsewhere on Windows."""
621
+ text = str(path or "")
622
+ if not text or not (posixpath.isabs(text) or ntpath.isabs(text)) or os.path.exists(text):
623
+ return ""
624
+ folder, name = ntpath.split(text) # (ntpath takes "/" and "\\" both)
625
+ if not name or ntpath.basename(folder).lower() != "tls":
626
+ return ""
627
+ found = os.path.join(data_dir, "tls", name)
628
+ return found if os.path.isfile(found) and found != text else ""
629
+
630
+
631
+ def normalize_cidr(value: str) -> str:
632
+ """Treat a bare wildcard address as 'any'. Users typing 0.0.0.0 / :: in an
633
+ allow/priority list mean 'all clients', not a single /32 or /128 host."""
634
+ v = value.strip()
635
+ if v == "0.0.0.0":
636
+ return "0.0.0.0/0"
637
+ if v == "::":
638
+ return "::/0"
639
+ return v
640
+
641
+
642
+ # the bind IPs of a listener on every interface: 0.0.0.0 takes IPv4 clients, :: IPv6 ones
643
+ # (IPV6_V6ONLY); NetworkSettings.dual_stack makes either take both
644
+ WILDCARDS = ("0.0.0.0", "::")
645
+
646
+
647
+ def canonical_ip(value: Any) -> Any:
648
+ """An IP address as it is stored: in its canonical form ('0:0:0:0:0:0:0:1' is '::1'),
649
+ an IPv4-mapped IPv6 address ('::ffff:10.0.0.5') as the IPv4 address it stands for
650
+ (that is what a listener on it can take). Anything else is returned as it is, for
651
+ validation to report."""
652
+ if not isinstance(value, str):
653
+ return value
654
+ try:
655
+ ip = ipaddress.ip_address(value)
656
+ except ValueError:
657
+ return value
658
+ if ip.version == 6 and ip.ipv4_mapped is not None:
659
+ return str(ip.ipv4_mapped)
660
+ return str(ip)
661
+
662
+
663
+ @functools.lru_cache(maxsize=None)
664
+ def _ipv6_takes_ipv4() -> bool:
665
+ """Whether an IPv6 socket on this system takes IPv4 too unless told otherwise
666
+ (IPV6_V6ONLY off by default: Linux unless net.ipv6.bindv6only is set, not Windows).
667
+ Before 2.9 a UDP mapping on :: kept that default."""
668
+ try:
669
+ with socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) as s:
670
+ return not s.getsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY)
671
+ except (OSError, AttributeError): # no IPv6 here
672
+ return False
673
+
674
+
675
+ def shown_host(host: Any) -> str:
676
+ """A host as host:port writes it: an IPv6 address in brackets, the admin UI's ""
677
+ (every interface) as *."""
678
+ h = "" if host is None else str(host)
679
+ if not h:
680
+ return "*"
681
+ return f"[{h}]" if ":" in h and not h.startswith("[") else h
682
+
683
+
684
+ def host_port(host: Any, port: Any) -> str:
685
+ """host:port as the log and the UI write it ([::1]:4001; see shown_host)."""
686
+ return f"{shown_host(host)}:{port}"
687
+
688
+
689
+ def bind_ip_problem(value: Any) -> str:
690
+ """Why ``value`` can't be a listener's bind IP ("" when it can): it is not an IP
691
+ address, or it is a multicast address or a reserved one other than the wildcard
692
+ and loopback addresses (0.0.0.0, ::, ::1). An IPv4-mapped IPv6 address is judged by
693
+ its IPv4 address (Python releases differ in how is_reserved reads one). The one rule
694
+ for a mapping's bind IP and for the console's custom admin UI address."""
695
+ try:
696
+ ip = ipaddress.ip_address(value)
697
+ except ValueError:
698
+ return _("Invalid bind IP: {0}.").format(repr(value))
699
+ if ip.version == 6 and ip.ipv4_mapped is not None:
700
+ ip = ip.ipv4_mapped
701
+ if ip.is_multicast or (ip.is_reserved and not (ip.is_loopback or ip.is_unspecified)):
702
+ return _("Bind IP cannot be a multicast/reserved address.")
703
+ return ""
704
+ ClientNetwork = Union[ipaddress.IPv4Network, ipaddress.IPv6Network]
705
+
706
+
707
+ def client_networks(entries: Iterable[Any]) -> tuple[list[ClientNetwork], list[str]]:
708
+ """The networks of an allowed / priority client list (an address is a one-host network;
709
+ 0.0.0.0 and :: are everyone, see normalize_cidr), and the entries that are none: a host
710
+ name, a range, a comma-joined list ... Edits refuse those (NetworkSettings.validate), so
711
+ only a hand-edited config.json has them; where a list is used, they are left out (an
712
+ allow-list none of whose entries is usable admits nobody: engine/access.AccessPolicy)."""
713
+ nets: list[ClientNetwork] = []
714
+ ignored: list[str] = []
715
+ for entry in entries or []:
716
+ text = entry if isinstance(entry, str) else str(entry)
717
+ try:
718
+ nets.append(ipaddress.ip_network(normalize_cidr(text), strict=False))
719
+ except ValueError:
720
+ ignored.append(text)
721
+ return nets, ignored
722
+
723
+
724
+ # ---------------------------------------------------------------------------
725
+ # Serial side
726
+ # ---------------------------------------------------------------------------
727
+ @dataclass
728
+ class SerialAdvanced:
729
+ rs485_enabled: bool = False
730
+ rs485_delay_before_tx_ms: float = 0.0
731
+ rs485_delay_after_tx_ms: float = 0.0
732
+ rs485_rts_level_for_tx: bool = True
733
+ hangup_when_done: bool = False
734
+ nobreak: bool = False
735
+ # keeping other programs off the port (POSIX; Windows opens a port for one program
736
+ # only anyway). `exclusive` (flock) keeps out only programs that flock the port too:
737
+ # tiocexcl: while the port is open, other programs' opens fail (TIOCEXCL; not root's)
738
+ # respect_uucp_locks: don't open a port another program has a UUCP lock file on
739
+ # (minicom, cu: LCK..<tty> in /run/lock or /var/lock; read only, none is made)
740
+ tiocexcl: bool = False
741
+ respect_uucp_locks: bool = False
742
+
743
+
744
+ @dataclass
745
+ class SerialSettings:
746
+ port: str = ""
747
+ # Stable hardware identity; when set, used to (re)resolve the device path on
748
+ # (re)connect so COMx / ttyUSB* renumbering doesn't break a mapping.
749
+ match: dict[str, Any] = field(default_factory=dict)
750
+ baudrate: int = 9600
751
+ bytesize: int = 8
752
+ parity: str = "N"
753
+ stopbits: float = 1
754
+ flowcontrol: str = "none" # none | rtscts | xonxoff | dsrdtr
755
+ rts_on_open: str = "keep" # on | off | keep
756
+ dtr_on_open: str = "keep"
757
+ exclusive: bool = True
758
+ advanced: SerialAdvanced = field(default_factory=SerialAdvanced)
759
+
760
+ @staticmethod
761
+ def from_dict(d: dict[str, Any], notes: Optional[list[FieldNote]] = None,
762
+ path: str = "") -> "SerialSettings":
763
+ d = dict(d or {})
764
+ notes = [] if notes is None else notes
765
+ fields = _read_fields(SerialSettings, d, notes, path, skip=("advanced",))
766
+ adv = _section(d, "advanced", path)
767
+ advanced = SerialAdvanced(**_read_fields(SerialAdvanced, adv, notes, path + "advanced."))
768
+ return SerialSettings(advanced=advanced, **fields)
769
+
770
+ def validate(self) -> None:
771
+ if not self.port:
772
+ raise ConfigError(_("Serial port is required."))
773
+ if not isinstance(self.baudrate, int) or self.baudrate <= 0:
774
+ raise ConfigError(_("Baud rate must be a positive integer."))
775
+ if self.bytesize not in BYTESIZES:
776
+ raise ConfigError(_("Data bits must be one of {0}.").format(_choices(BYTESIZES)))
777
+ if self.parity not in PARITIES:
778
+ raise ConfigError(_("Parity must be one of {0}.").format(_choices(PARITIES)))
779
+ if self.stopbits not in STOPBITS:
780
+ raise ConfigError(_("Stop bits must be one of {0}.").format(_choices(STOPBITS)))
781
+ if self.flowcontrol not in FLOWCONTROLS:
782
+ raise ConfigError(_("Flow control must be one of {0}.").format(_choices(FLOWCONTROLS)))
783
+ if self.rts_on_open not in LINE_STATES or self.dtr_on_open not in LINE_STATES:
784
+ raise ConfigError(_("RTS/DTR on open must be on|off|keep."))
785
+ # rtscts (hardware) flow control conflicts with RS-485 RTS direction toggling.
786
+ if self.advanced.rs485_enabled and self.flowcontrol == "rtscts":
787
+ raise ConfigError(_("RS-485 RTS toggling conflicts with RTS/CTS flow control."))
788
+
789
+ def compact(self) -> str:
790
+ """ser2net-style compact descriptor, e.g. '9600N81'."""
791
+ return f"{self.baudrate}{self.parity}{self.bytesize}{int(self.stopbits)}"
792
+
793
+ @property
794
+ def identity(self) -> dict[str, Any]:
795
+ """The USB identity this port is bound to (usb_identity of ``match``), {} if none."""
796
+ return usb_identity(self.match)
797
+
798
+
799
+ # the keys of a serial port's `match` (the USB identity it is bound to) the engine knows;
800
+ # any other key (a typo, "VID") is not used, so it can't make every port match
801
+ MATCH_KEYS = ("vid", "pid", "serial_number", "location")
802
+
803
+
804
+ def usb_identity(match: Any) -> dict[str, Any]:
805
+ """The known keys of a serial port's ``match`` that are set (MATCH_KEYS); {} when none
806
+ is: the port is opened by its name."""
807
+ if not isinstance(match, dict):
808
+ return {}
809
+ return {k: match[k] for k in MATCH_KEYS if match.get(k)}
810
+
811
+
812
+ def identity_text(match: Any) -> str:
813
+ """A USB identity as the UI shows it: VID:PID [serial number] (@ location)."""
814
+ want = usb_identity(match)
815
+ text = f"{want.get('vid', '*')}:{want.get('pid', '*')}"
816
+ if want.get("serial_number"):
817
+ text += f" [{want['serial_number']}]"
818
+ if want.get("location"):
819
+ text += f" @ {want['location']}"
820
+ return text
821
+
822
+
823
+ _COM_NAME = re.compile(r"(?:\\\\\.\\|//\./)?(COM\d+)", re.I)
824
+
825
+
826
+ def canonical_device(name: Any) -> Any:
827
+ """A serial port name in the form two names of one port have in common, for comparing
828
+ them (never for storing: what the user set is kept as it is). Windows opens 'com3',
829
+ 'COM3', '\\\\.\\COM3' and '//./COM3' alike, so all are 'COM3'; the name's shape decides,
830
+ not this machine's OS (a cluster peer's ports are compared here too). Any other name is
831
+ returned unchanged ('\\\\.\\CNCA0' of com0com keeps its prefix, POSIX paths their case)."""
832
+ m = _COM_NAME.fullmatch(name) if isinstance(name, str) else None
833
+ return m.group(1).upper() if m else name
834
+
835
+
836
+ def _serial_clash(m: "MappingConfig", mappings: list["MappingConfig"]) -> Optional[tuple]:
837
+ """The first serial port enabled mapping ``m`` shares with another enabled one of
838
+ ``mappings``: (other mapping, m's side, the other's side), None when none. Shared means
839
+ the same port in the canonical spelling (canonical_device) with the same USB identity
840
+ binding (none, or the same one: both would open the same device)."""
841
+ if not m.enabled:
842
+ return None
843
+
844
+ def uses(mapping):
845
+ return [(canonical_device(s.port),
846
+ sorted((k, str(v).casefold()) for k, v in s.identity.items()), s)
847
+ for s in mapping.serial_sides() if s.port]
848
+
849
+ mine = uses(m)
850
+ for other in mappings:
851
+ if other.id == m.id or not other.enabled:
852
+ continue
853
+ for key, ident, side in mine:
854
+ for okey, oident, oside in uses(other):
855
+ if key == okey and ident == oident:
856
+ return other, side, oside
857
+ return None
858
+
859
+
860
+ def serial_conflict(m: "MappingConfig", mappings: list["MappingConfig"]) -> str:
861
+ """Why mapping ``m`` can't be enabled as it is next to ``mappings`` ("" when it can): an
862
+ enabled one uses the same serial port, in any spelling Windows opens alike ('com3' is
863
+ 'COM3'), bound to no USB identity or to the same one. Checked where a mapping is created,
864
+ changed or started (the form, the REST API, a cluster peer, an import), never by
865
+ validate(): a stored pair must not stop a start or another save (from_dict notes it)."""
866
+ clash = _serial_clash(m, mappings)
867
+ if clash is None:
868
+ return ""
869
+ other, side, oside = clash
870
+ if side.identity:
871
+ return _("Mappings '{0}' and '{1}' are both bound to the USB identity {2} on serial "
872
+ "port {3}: they would open the same device, and a serial port can only be "
873
+ "opened by one mapping.").format(other.name, m.name, identity_text(side.match),
874
+ side.port)
875
+ if side.port != oside.port:
876
+ return _("Mappings '{0}' and '{1}' both use serial port {2} (written {3} in '{1}') — a "
877
+ "serial port can only be opened by one mapping.").format(
878
+ other.name, m.name, oside.port, side.port)
879
+ return _("Mappings '{0}' and '{1}' both use serial port {2} — a serial "
880
+ "port can only be opened by one mapping.").format(other.name, m.name, side.port)
881
+
882
+
883
+ def url_port(port: Any) -> bool:
884
+ """True for a serial port value that pyserial would open with a URL handler instead
885
+ of as a device: anything with "://" (socket://, rfc2217://, spy://, loop://, alt://,
886
+ hwgrep:// ...), which is what pyserial's serial_for_url dispatches on. A mapping only
887
+ opens serial devices (COM3, \\\\.\\COM10, /dev/ttyUSB0, /dev/serial/by-id/...)."""
888
+ return isinstance(port, str) and "://" in port
889
+
890
+
891
+ _CONTROL_CHARS = re.compile(r"[\x00-\x1f\x7f]")
892
+
893
+
894
+ def port_problem(port: Any) -> str:
895
+ """Why ``port`` can't be set as a mapping's serial port ("" when it can). One permissive
896
+ rule for every edit (a path typed in the form, the REST API, an import, a cluster peer):
897
+ a URL is refused (url_port: ser2netweb opens serial devices only), and so is a name with a
898
+ control character in it (a pasted line break); device paths (/dev/..., a link to one,
899
+ any absolute path), Windows names (COM3, \\\\.\\CNCA0) and a placeholder for a port
900
+ found by its USB identity ("auto") pass. Never checked by validate(): a stored port must
901
+ keep loading (a URL one is refused when it is opened, see engine/serial_io.refuse_url)."""
902
+ if url_port(port):
903
+ return _("Serial port {0} is a URL, not a device: ser2netweb opens serial devices only "
904
+ "(URLs such as socket:// or spy:// are not supported).").format(port)
905
+ if isinstance(port, str) and _CONTROL_CHARS.search(port):
906
+ return _("Serial port {0} has a control character in it (a line break?): type the "
907
+ "device's path again.").format(repr(port))
908
+ return ""
909
+
910
+
911
+ # ---------------------------------------------------------------------------
912
+ # Network side
913
+ # ---------------------------------------------------------------------------
914
+ NET_MODES = ("server", "client", "udp")
915
+
916
+
917
+ @dataclass
918
+ class NetworkSettings:
919
+ mode: str = "server" # server (listen) | client (connect-out) | udp
920
+ protocol: str = "raw" # raw | telnet | rfc2217
921
+ # A listener on 0.0.0.0 takes IPv4 clients only, one on :: IPv6 clients only. With
922
+ # dual_stack a wildcard bind IP (either of the two) takes both: the picker's "All
923
+ # interfaces (IPv4 + IPv6)". It means nothing with another address (edits refuse
924
+ # that); a release without it listens on the bind IP alone.
925
+ bind_ip: str = "0.0.0.0"
926
+ dual_stack: bool = field(default=False, metadata=_ONLY_TRUE)
927
+ port: int = 4001
928
+ # client mode: the connect-out target (UDP does not use these: see udp_peer_host)
929
+ remote_host: str = ""
930
+ remote_port: int = 0
931
+ # UDP: where the device's output goes. Blank (default): to the sender of the last
932
+ # datagram from an allowed IP, so a new sender takes it over. With a reply address
933
+ # (an IP and a port) it always goes there, whoever sends.
934
+ udp_peer_host: str = ""
935
+ udp_peer_port: int = 0
936
+ # UDP without a reply address: forget the last sender after this many seconds
937
+ # without a datagram from it (the output then goes nowhere until someone sends);
938
+ # 0 = never.
939
+ udp_peer_timeout_s: int = 0
940
+ # UDP: more destinations that always get the device's output too (whoever sends, and
941
+ # before anyone has), each "IP:port" ("[IPv6]:port"): e.g. an NMEA 0183 feed to a LAN's
942
+ # broadcast address. Sending to a broadcast address needs udp_broadcast (SO_BROADCAST).
943
+ udp_targets: list[str] = field(default_factory=list)
944
+ udp_broadcast: bool = field(default=False, metadata=_ONLY_TRUE)
945
+ max_connections: int = 1
946
+ kick_old_user: bool = False
947
+ # TCP (raw / telnet / rfc2217, server and connect-out): while the serial port is not
948
+ # open (unplugged, reconnecting) clients are disconnected and new ones refused, after
949
+ # the short wait for the port that an RFC 2217 client gets (options.
950
+ # rfc2217_net_timeout_s); a connect-out mapping connects only while the port is open.
951
+ # Off (default): clients stay connected and are taken, and what they send for the
952
+ # device meanwhile is dropped (counted in bytes_dropped); RFC 2217 sessions follow a
953
+ # reopened port.
954
+ disconnect_without_device: bool = field(default=False, metadata=_ONLY_TRUE)
955
+ allowed_client_ips: list[str] = field(default_factory=list)
956
+ # High-priority clients: when the port is at capacity, a connection from one of
957
+ # these IPs/CIDRs is admitted by kicking an existing client (oldest non-priority
958
+ # first), regardless of kick_old_user.
959
+ priority_client_ips: list[str] = field(default_factory=list)
960
+ read_only: bool = field(default=False, metadata=_PROTECTIVE)
961
+ # Per-client outbound (serial->network) buffer, in chunks. A client slower than
962
+ # the serial source is dropped once this fills. A chunk is one serial read, at most
963
+ # 4 KB (8 KB with telnet / RFC 2217, which double 0xFF bytes), so the worst case per
964
+ # client is ~client_queue_max * 4 KB (8 KB); raise for bursty high-throughput links.
965
+ client_queue_max: int = 2048
966
+ # Dead-peer detection for TCP (server + client mode): after keepalive_idle_s of
967
+ # silence the OS probes the peer every keepalive_interval_s and drops the link
968
+ # after keepalive_count unanswered probes. A client that vanished without a
969
+ # FIN/RST (power cut, cable pull, VPN/NAT drop) is then reaped in about
970
+ # idle + interval * count seconds instead of holding its slot forever.
971
+ tcp_keepalive: bool = True
972
+ keepalive_idle_s: int = 60
973
+ keepalive_interval_s: int = 10
974
+ keepalive_count: int = 3
975
+ # Client (connect-out) mode: abandon a TCP (+TLS) connect attempt after this many
976
+ # seconds and retry. 0 = no limit of our own (the OS gives up after ~21 s on
977
+ # Windows, ~2 min on Linux).
978
+ connect_timeout_s: float = 10.0
979
+ # Per-mapping TLS for the data bridge (server/client TCP), see engine/tls.py. A listener
980
+ # needs tls_cert and tls_key; with the CA file tls_client_ca it asks every client for a
981
+ # certificate that chains to it, and ends the handshake of one without such a
982
+ # certificate (the allow-list still applies). A connect-out link presents them (when
983
+ # set) to a server that asks for a client certificate, and checks the server's
984
+ # certificate: it must chain to a certificate in the CA file tls_ca and name
985
+ # remote_host, and/or be one of the SHA-256 fingerprints in tls_fingerprint (with
986
+ # neither, the link only encrypts). tls_key_password decrypts an encrypted key (never
987
+ # asked for on a terminal); edits load the files with it (MappingConfig.check_tls_files).
988
+ tls: bool = field(default=False, metadata=_PROTECTIVE)
989
+ tls_cert: str = ""
990
+ tls_key: str = ""
991
+ tls_key_password: str = ""
992
+ tls_ca: str = ""
993
+ tls_fingerprint: str = ""
994
+ tls_client_ca: str = ""
995
+
996
+ @property
997
+ def listens(self) -> bool:
998
+ return self.mode in ("server", "udp")
999
+
1000
+ @property
1001
+ def both_families(self) -> bool:
1002
+ """The listener takes IPv4 and IPv6 clients: dual_stack (true itself, not text
1003
+ that reads as true) with a wildcard bind IP."""
1004
+ return self.dual_stack is True and canonical_ip(self.bind_ip) in WILDCARDS
1005
+
1006
+ @property
1007
+ def bind_shown(self) -> str:
1008
+ """The bind IP as the UI shows it before the port: * for every interface of both
1009
+ families, an IPv6 address in brackets."""
1010
+ return "*" if self.both_families else shown_host(self.bind_ip)
1011
+
1012
+ @staticmethod
1013
+ def from_dict(d: dict[str, Any], notes: Optional[list[FieldNote]] = None,
1014
+ path: str = "") -> "NetworkSettings":
1015
+ d = dict(d or {})
1016
+ notes = [] if notes is None else notes
1017
+ net = NetworkSettings(**_read_fields(NetworkSettings, d, notes, path))
1018
+ net.bind_ip = canonical_ip(net.bind_ip) # (AppConfig.from_dict notes a change)
1019
+ # a UDP mapping has no connection to negotiate telnet on, so it always ran raw: say
1020
+ # so (validate() refuses RFC 2217 and Modbus over UDP, but older releases took telnet)
1021
+ if net.mode == "udp" and net.protocol == "telnet":
1022
+ notes.append(FieldNote(
1023
+ path + "protocol", "replaced", "telnet", "raw",
1024
+ why=f"{path}protocol telnet is not supported over UDP: raw is used (as it "
1025
+ "always was: no telnet negotiation or escaping over UDP)",
1026
+ warning=_("Telnet is not supported over UDP, so the protocol is raw (as it "
1027
+ "always was in effect).")))
1028
+ net.protocol = "raw"
1029
+ return net
1030
+
1031
+ def validate(self) -> None:
1032
+ if self.mode not in NET_MODES:
1033
+ raise ConfigError(_("Network mode must be one of {0}.").format(_choices(NET_MODES)))
1034
+ if self.protocol not in PROTOCOLS:
1035
+ raise ConfigError(_("Protocol must be one of {0}.").format(_choices(PROTOCOLS)))
1036
+
1037
+ if self.listens:
1038
+ if problem := bind_ip_problem(self.bind_ip):
1039
+ raise ConfigError(problem)
1040
+ if not isinstance(self.port, int) or not (1 <= self.port <= 65535):
1041
+ raise ConfigError(_("TCP/UDP port must be between 1 and 65535."))
1042
+ else: # client (connect-out)
1043
+ if not self.remote_host.strip():
1044
+ raise ConfigError(_("Connect-out mode requires a remote host."))
1045
+ if not isinstance(self.remote_port, int) or not (1 <= self.remote_port <= 65535):
1046
+ raise ConfigError(_("Remote port must be between 1 and 65535."))
1047
+
1048
+ if not isinstance(self.max_connections, int) or self.max_connections < 1:
1049
+ raise ConfigError(_("Max connections must be >= 1."))
1050
+ if not isinstance(self.client_queue_max, int) or self.client_queue_max < 16:
1051
+ raise ConfigError(_("Client queue size must be an integer >= 16."))
1052
+ # OS limits: Linux caps TCP_KEEPIDLE/TCP_KEEPINTVL at 32767 s and TCP_KEEPCNT at 127.
1053
+ for label, value, high in ((_("Keepalive idle time"), self.keepalive_idle_s, 32767),
1054
+ (_("Keepalive probe interval"), self.keepalive_interval_s, 32767),
1055
+ (_("Keepalive probe count"), self.keepalive_count, 127)):
1056
+ if not _is_int(value) or not (1 <= value <= high):
1057
+ raise ConfigError(_("{0} must be an integer between 1 and {1}.").format(label, high))
1058
+ if not _is_number(self.connect_timeout_s) or not (0 <= self.connect_timeout_s <= 3600):
1059
+ raise ConfigError(_("Connect timeout must be between 0 and 3600 seconds (0 = OS "
1060
+ "default)."))
1061
+ if self.protocol == "rfc2217" and self.mode == "udp":
1062
+ raise ConfigError(_("RFC2217 requires a TCP (server/client) connection, not UDP."))
1063
+ if self.protocol == "modbus" and self.mode != "server":
1064
+ raise ConfigError(_("Modbus gateway requires TCP server mode (masters connect in)."))
1065
+ if self.protocol == "rfc2217" and self.max_connections > 1:
1066
+ raise ConfigError(
1067
+ _("RFC2217 supports a single connection (clients share one serial port's "
1068
+ "settings). Set Max connections = 1.")
1069
+ )
1070
+ if self.tls:
1071
+ if self.mode == "udp":
1072
+ raise ConfigError(_("TLS is not applicable to UDP mappings."))
1073
+ if self.listens and not (self.tls_cert and self.tls_key):
1074
+ raise ConfigError(_("Enable TLS requires both a certificate and a key."))
1075
+ if bool(self.tls_cert) != bool(self.tls_key): # connect-out: both or neither
1076
+ raise ConfigError(_("Connect-out TLS: set both a certificate and a key (for a "
1077
+ "server that asks for a client certificate), or neither."))
1078
+ for label, lst in ((_("allowed"), self.allowed_client_ips),
1079
+ (_("priority"), self.priority_client_ips)):
1080
+ _nets, bad = client_networks(lst)
1081
+ if bad:
1082
+ raise ConfigError(_("Invalid {0} client IP/CIDR: {1}.").format(label, repr(bad[0])))
1083
+
1084
+
1085
+ def tls_files_missing(net: NetworkSettings) -> list[str]:
1086
+ """A TLS listener's certificate / key files that don't exist, one message each. Not a
1087
+ rule of validate(): a file that goes missing (a mount, a renewal) must not block every
1088
+ other change. An edit refuses them (MappingConfig.validate_input), start-up names them
1089
+ (AppConfig.problems) and the mapping then fails to start, saying so."""
1090
+ if not net.tls or net.mode == "udp":
1091
+ return []
1092
+ return [_("TLS file not found: {0}").format(p)
1093
+ for p in dict.fromkeys((net.tls_cert, net.tls_key))
1094
+ if p and not os.path.isfile(data_path(p))]
1095
+
1096
+
1097
+ def udp_peer_address(net: NetworkSettings) -> Optional[tuple[str, int]]:
1098
+ """The reply address of a UDP mapping (udp_peer_host, udp_peer_port): where the
1099
+ device's output always goes, or None when none is set (the last sender gets it).
1100
+ Raises ValueError when it is not a host's IP address and a port; edits refuse that
1101
+ (MappingConfig.validate_input), so only a hand-edited config has it."""
1102
+ host = str(net.udp_peer_host or "").strip()
1103
+ if not host:
1104
+ return None
1105
+ try:
1106
+ ip = ipaddress.ip_address(host)
1107
+ except ValueError:
1108
+ raise ValueError(f"UDP reply address {host!r} is not an IP address") from None
1109
+ if ip.is_unspecified:
1110
+ raise ValueError(f"UDP reply address {host} is not a host's address")
1111
+ port = net.udp_peer_port
1112
+ if not _is_int(port) or not 1 <= port <= 65535:
1113
+ raise ValueError(f"UDP reply port {port!r} is not between 1 and 65535")
1114
+ return host, port
1115
+
1116
+
1117
+ UDP_TARGETS_MAX = 32
1118
+
1119
+
1120
+ def udp_target(entry: Any) -> tuple[str, int]:
1121
+ """One UDP target, "IP:port" or "[IPv6]:port", as (IP, port). Raises ValueError when it
1122
+ is not a host's IP address (or a broadcast / multicast one) and a port."""
1123
+ text = str(entry).strip() if isinstance(entry, str) else ""
1124
+ host, sep, port = text.rpartition(":")
1125
+ if host.startswith("[") and host.endswith("]"):
1126
+ host = host[1:-1]
1127
+ elif ":" in host: # an IPv6 address needs its brackets ([fd00::1]:5000)
1128
+ sep = ""
1129
+ try:
1130
+ if not sep:
1131
+ raise ValueError
1132
+ ip = ipaddress.ip_address(host)
1133
+ number = int(port)
1134
+ except ValueError:
1135
+ raise ValueError(f"UDP target {entry!r} is not an IP address and a port "
1136
+ "(IP:port, [IPv6]:port)") from None
1137
+ if ip.is_unspecified:
1138
+ raise ValueError(f"UDP target {entry!r} is not a host's address")
1139
+ if not 1 <= number <= 65535:
1140
+ raise ValueError(f"UDP target {entry!r}: the port is not between 1 and 65535")
1141
+ return str(ip), number
1142
+
1143
+
1144
+ def udp_targets(net: NetworkSettings) -> list[tuple[str, int]]:
1145
+ """A UDP mapping's targets (udp_targets), each once. Raises ValueError for a stored one
1146
+ that is malformed (edits refuse it: MappingConfig.validate_input), so the mapping fails
1147
+ to start rather than leaving a destination out without a word."""
1148
+ entries = net.udp_targets
1149
+ if not isinstance(entries, list):
1150
+ raise ValueError("udp_targets is not a list of IP:port entries")
1151
+ return list(dict.fromkeys(udp_target(e) for e in entries))
1152
+
1153
+
1154
+ def udp_peer_timeout(net: NetworkSettings) -> float:
1155
+ """A UDP mapping's peer timeout in seconds, 0 = never. A stored value that is not a
1156
+ number of seconds reads as 0, today's behaviour (edits refuse one: validate_input)."""
1157
+ t = net.udp_peer_timeout_s
1158
+ if _is_number(t) and math.isfinite(t) and t > 0:
1159
+ return float(min(t, MAX_TIMER_S))
1160
+ return 0.0
1161
+
1162
+
1163
+ # ---------------------------------------------------------------------------
1164
+ # Per-mapping options
1165
+ # ---------------------------------------------------------------------------
1166
+ RFC2217_SIGNATURE_MAX = 128 # characters (an edit's rule)
1167
+
1168
+ # The text encoding of a mapping's web console and hex view (MappingOptions.
1169
+ # console_encoding): what the device's bytes are shown as, and what is typed is sent as.
1170
+ # The bridge itself passes bytes unchanged. UTF-8, or one of these single-byte code pages.
1171
+ CONSOLE_ENCODINGS = ("utf-8", "iso-8859-9", "cp1254", "cp857", "iso-8859-1", "cp1252", "cp437")
1172
+
1173
+
1174
+ def console_encoding(options: "MappingOptions") -> str:
1175
+ """A mapping's console encoding: a stored value that is not one of CONSOLE_ENCODINGS
1176
+ (hand-edited, or from a newer version) reads as UTF-8, today's behaviour (edits
1177
+ refuse one: MappingConfig.validate_input)."""
1178
+ value = options.console_encoding
1179
+ return value if value in CONSOLE_ENCODINGS else "utf-8"
1180
+
1181
+
1182
+ @functools.lru_cache(maxsize=None)
1183
+ def console_charset(encoding: str) -> str:
1184
+ """What the web console shows for the bytes 0x80-0xFF in `encoding` (app.js decodes
1185
+ the device's bytes with it, and encodes what is typed with it, in the browser: it has
1186
+ no CP857 and encodes nothing but UTF-8): 128 characters, "" for UTF-8 (xterm.js decodes
1187
+ that itself). A C1 control code shows as '.', so no byte becomes a terminal command; a
1188
+ byte the code page leaves undefined as U+FFFD."""
1189
+ if encoding not in CONSOLE_ENCODINGS or encoding == "utf-8":
1190
+ return ""
1191
+ chars = (bytes([b]).decode(encoding, "replace") for b in range(0x80, 0x100))
1192
+ return "".join("." if "\x80" <= c <= "\x9f" else c for c in chars)
1193
+
1194
+
1195
+ @dataclass
1196
+ class MappingOptions:
1197
+ banner: str = ""
1198
+ openstr: str = ""
1199
+ closestr: str = ""
1200
+ closeon: str = ""
1201
+ # Per-connection timers for TCP clients (0 = off): drop a client after this many
1202
+ # seconds without traffic either way / after this long connected, active or not
1203
+ # (a connect-out mapping then reconnects).
1204
+ idle_timeout_s: int = 0
1205
+ max_connection_time_s: int = 0
1206
+ # Traffic trace: a file NAME (empty = off), written as data/traces/<id>-<name>
1207
+ # (see trace_file_name). Rotated past trace_max_mb, keeping trace_keep old files.
1208
+ trace_both: str = ""
1209
+ trace_hexdump: bool = False
1210
+ trace_timestamp: bool = True
1211
+ trace_max_mb: float = 10.0
1212
+ trace_keep: int = 3
1213
+ # the web console's and hex view's text encoding (CONSOLE_ENCODINGS); display and typing
1214
+ # only: what the bridge passes stays the same bytes
1215
+ console_encoding: str = "utf-8"
1216
+ # RFC2217 interop knobs for non-compliant peers:
1217
+ rfc2217_poll_modem_interval_s: float = 1.0
1218
+ rfc2217_net_timeout_s: float = 3.0
1219
+ # RFC2217: once the client has left, put the serial port back to this mapping's baud
1220
+ # rate, data bits, parity, stop bits and flow control (off: what the client set stays
1221
+ # until the port is reopened). RTS/DTR are left alone: toggling them can reset a device.
1222
+ rfc2217_restore_on_disconnect: bool = False
1223
+ # RFC2217: what a client's SIGNATURE request is answered with (blank: "ser2net")
1224
+ rfc2217_signature: str = ""
1225
+ # Modbus gateway: how long an RTU slave gets to begin its reply once the request is out
1226
+ # on the line, before the gateway answers exception 0x0B (gateway target failed to
1227
+ # respond). Read through modbus_response_timeout().
1228
+ modbus_response_timeout_s: float = MODBUS_TIMEOUT_DEFAULT_S
1229
+ # Modbus gateway: unit ids masters send that go to another RTU unit, as {"<id>": unit}
1230
+ # ({"255": 1}: a request for unit 255 goes to unit 1 and is answered as unit 255). Empty
1231
+ # (the default): each request goes to the unit it names, as before. Read through
1232
+ # modbus_unit_map().
1233
+ modbus_unit_map: dict = field(default_factory=dict)
1234
+ # Serial-to-network mappings (not a Modbus gateway): send poll_string to the device
1235
+ # every poll_interval_s seconds while its port is open (0 = off), e.g. a scale's or an
1236
+ # analyzer's query, so its answers reach MQTT, a trace or a UDP target without a
1237
+ # program polling it. \r \n \t \\ and \xHH stand for those bytes (poll_bytes).
1238
+ # Skipped while a TCP client is connected (it does the asking then), unless
1239
+ # poll_with_clients.
1240
+ poll_string: str = ""
1241
+ poll_interval_s: float = 0.0
1242
+ poll_with_clients: bool = field(default=False, metadata=_ONLY_TRUE)
1243
+ # No-data alarm (notify.py: no_data / data_back, the dashboard, /metrics): the serial
1244
+ # port is open but the device has sent nothing for this many seconds (0 = off,
1245
+ # else NO_DATA_MIN_S..NO_DATA_MAX_S); with no_data_reply_only, only while something
1246
+ # sent to it gets no answer (a device that speaks when asked, e.g. Modbus RTU).
1247
+ no_data_alarm_s: int = 0
1248
+ no_data_reply_only: bool = False
1249
+
1250
+ @staticmethod
1251
+ def from_dict(d: dict[str, Any], notes: Optional[list[FieldNote]] = None,
1252
+ path: str = "") -> "MappingOptions":
1253
+ d = dict(d or {})
1254
+ notes = [] if notes is None else notes
1255
+ opts = MappingOptions(**_read_fields(MappingOptions, d, notes, path))
1256
+ opts.idle_timeout_s = _legacy_seconds(opts.idle_timeout_s)
1257
+ opts.trace_both = trace_file_name(opts.trace_both)
1258
+ opts._limit_no_data_alarm(notes, path)
1259
+ if isinstance(opts.modbus_unit_map, dict): # (JSON keys are text)
1260
+ opts.modbus_unit_map = {str(k): v for k, v in opts.modbus_unit_map.items()}
1261
+ return opts
1262
+
1263
+ def _limit_no_data_alarm(self, notes: list[FieldNote], path: str) -> None:
1264
+ """A no-data alarm outside 0 / NO_DATA_MIN_S..NO_DATA_MAX_S takes the nearest value
1265
+ it can have, with a note (loading notes it, an edit warns)."""
1266
+ value = self.no_data_alarm_s
1267
+ if value == 0 or NO_DATA_MIN_S <= value <= NO_DATA_MAX_S:
1268
+ return
1269
+ used = 0 if value < 0 else min(max(value, NO_DATA_MIN_S), NO_DATA_MAX_S)
1270
+ notes.append(FieldNote(
1271
+ path + "no_data_alarm_s", "replaced", value, used,
1272
+ why=f"{path}no_data_alarm_s {value} is not 0 (off) or {NO_DATA_MIN_S}.."
1273
+ f"{NO_DATA_MAX_S} seconds: {used} is used",
1274
+ warning=_("The no-data alarm must be 0 (off) or {0} to {1} seconds: {2} is "
1275
+ "used.").format(NO_DATA_MIN_S, NO_DATA_MAX_S, used)))
1276
+ self.no_data_alarm_s = used
1277
+
1278
+ @property
1279
+ def modbus_unit_map_text(self) -> str:
1280
+ """The unit map as the mapping form shows it: "255=1, 0=1"."""
1281
+ umap = self.modbus_unit_map
1282
+ return ", ".join(f"{k}={v}" for k, v in umap.items()) if isinstance(umap, dict) else ""
1283
+
1284
+ def validate(self) -> None:
1285
+ for label, value in ((_("Idle timeout"), self.idle_timeout_s),
1286
+ (_("Max connection time"), self.max_connection_time_s)):
1287
+ if not _is_number(value) or not (0 <= value <= MAX_TIMER_S):
1288
+ raise ConfigError(_("{0} must be between 0 (off) and {1} seconds.").format(label, MAX_TIMER_S))
1289
+ if not _is_number(self.trace_max_mb) or not (0.1 <= self.trace_max_mb <= 1024):
1290
+ raise ConfigError(_("Trace file size must be between 0.1 and 1024 MB."))
1291
+ if not _is_int(self.trace_keep) or not (0 <= self.trace_keep <= 50):
1292
+ raise ConfigError(_("Rotated trace files kept must be between 0 and 50."))
1293
+
1294
+
1295
+ def modbus_response_timeout(opts: MappingOptions) -> float:
1296
+ """A Modbus gateway's response timeout in seconds. A stored value that is not a
1297
+ positive number of seconds (before 2.9 it was in config.json only, unchecked) reads as
1298
+ the default 1 s, as 0 always did; a numeric string counts as its number. Edits refuse
1299
+ anything outside MODBUS_TIMEOUT_RANGE_S (validate_input)."""
1300
+ t = opts.modbus_response_timeout_s
1301
+ if isinstance(t, str):
1302
+ try:
1303
+ t = float(t.strip())
1304
+ except ValueError:
1305
+ return MODBUS_TIMEOUT_DEFAULT_S
1306
+ if _is_number(t) and math.isfinite(t) and t > 0:
1307
+ return float(min(t, MAX_TIMER_S))
1308
+ return MODBUS_TIMEOUT_DEFAULT_S
1309
+
1310
+
1311
+ def modbus_unit_map(opts: MappingOptions) -> dict[int, int]:
1312
+ """A Modbus gateway's unit map as it is used: {unit id a master sends: the RTU unit
1313
+ the request goes to}. Entries the edits refuse (not ids 0-255 to units 1-255) are left
1314
+ out, with a warning in the gateway's log at start."""
1315
+ out: dict[int, int] = {}
1316
+ umap = opts.modbus_unit_map
1317
+ for key, unit in (umap.items() if isinstance(umap, dict) else ()):
1318
+ sent = _loose_number(key, whole=True)
1319
+ if _is_int(sent) and 0 <= sent <= 255 and _is_int(unit) and 1 <= unit <= 255:
1320
+ out[sent] = unit
1321
+ return out
1322
+
1323
+
1324
+ # ---------------------------------------------------------------------------
1325
+ # Per-mapping MQTT publishing (optional northbound to an IIoT broker)
1326
+ # ---------------------------------------------------------------------------
1327
+ MQTT_VALUE_FORMATS = ("text", "json") # plain-format Modbus values (mqtt.value_format)
1328
+ MQTT_FORMATS = ("plain", "sparkplug_b")
1329
+ # what ends a serial line published to MQTT (MqttSettings.line_end)
1330
+ MQTT_LINE_ENDS = ("lf", "cr", "any")
1331
+ MQTT_TX_MAX_BYTES = 64 * 1024 # an MQTT write (<base_topic>/tx) at most, and all waiting
1332
+
1333
+
1334
+ # What a read-only REST API token gets instead of a stored secret (the MQTT password).
1335
+ # Sent back (a PUT of what such a token read, a form), it keeps the stored password, so it
1336
+ # is never stored as one.
1337
+ REDACTED = "********"
1338
+
1339
+
1340
+ @dataclass
1341
+ class MqttSettings:
1342
+ enabled: bool = False
1343
+ host: str = ""
1344
+ port: int = 1883
1345
+ base_topic: str = "" # e.g. "ser2net/plc1"; serial lines publish to <base_topic>
1346
+ qos: int = 0 # 0 | 1 | 2
1347
+ tls: bool = field(default=False, metadata=_PROTECTIVE)
1348
+ username: str = ""
1349
+ password: str = ""
1350
+ client_id: str = "" # blank => auto
1351
+ # Modbus polling only: "plain" topics per point, or Eclipse Sparkplug B (this mapping
1352
+ # is an edge node spBv1.0/<group>/N*/<node>; blank node id = the mapping's name)
1353
+ payload_format: str = "plain"
1354
+ sparkplug_group_id: str = "ser2net"
1355
+ sparkplug_node_id: str = ""
1356
+ # Sparkplug B: the primary host application (its host id) whose STATE the node follows,
1357
+ # publishing its births only while that host is online; blank = none (birth at once)
1358
+ sparkplug_primary_host_id: str = ""
1359
+ # Plain-format commands: also write a <base_topic>/<point>/set that the broker sends
1360
+ # from its retained store after each (re)connect. Off: a command is written once, when
1361
+ # it is published (Sparkplug NCMD / DCMD are never retained, so a retained one is
1362
+ # always ignored).
1363
+ apply_retained_set: bool = False
1364
+ # Serial lines (not a Modbus gateway): what ends a line. "lf" (default): \n, a \r just
1365
+ # before it is left out (as before); "cr": \r, a \n next to it left out; "any": \r or
1366
+ # \n (\r\n is one line end). A stored value that is none of these reads as "lf".
1367
+ line_end: str = "lf"
1368
+ # Serial lines (not a Modbus gateway): write each message on <base_topic>/tx to the
1369
+ # serial port as it is, and answer on <base_topic>/tx/result ("ok" / "error: ...").
1370
+ # Read-only refuses them; a retained one (the broker's copy, sent again after each
1371
+ # (re)connect) is ignored. Off: MQTT only publishes.
1372
+ tx_enabled: bool = field(default=False, metadata=_ONLY_TRUE)
1373
+ # Plain-format Modbus polling: also publish each polled unit's state, retained, to
1374
+ # <base_topic>/unit<N>/status ("online" / "offline"), on every change and after each
1375
+ # (re)connect. Off: nothing beyond the points' values and <base_topic>/status.
1376
+ unit_status: bool = False
1377
+ # TLS (tls on): the broker's certificate must chain to a CA the system trusts, or with
1378
+ # ca_file to one in that file only (PEM or DER); client_cert (PEM) is presented to a
1379
+ # broker that asks for a certificate (mutual TLS), with its key in client_key or in the
1380
+ # same file, encrypted with client_key_password if at all; tls_check_hostname off: the
1381
+ # certificate need not name the broker host (its chain is still checked). See
1382
+ # engine/mqtt_pub.tls_context; edits check the files (check_mqtt_tls_files).
1383
+ ca_file: str = ""
1384
+ client_cert: str = ""
1385
+ client_key: str = ""
1386
+ client_key_password: str = ""
1387
+ tls_check_hostname: bool = field(default=True, metadata=_PROTECTIVE)
1388
+ # Plain format: publish the data (serial lines, polled Modbus values) retained, so a
1389
+ # subscriber that (re)connects gets the last one at once. Sparkplug B never retains
1390
+ # data (the specification asks so); command results are never retained.
1391
+ retain: bool = False
1392
+ # Plain-format Modbus values: "text" (1 / 0 for a bit), or "json": {"value", "ts" (read
1393
+ # time, ms since the epoch), "quality"}, with "bad" (value null) once for a point whose
1394
+ # read fails or whose unit stops answering. Edits refuse other values; a stored one
1395
+ # (a later version's) is published as text.
1396
+ value_format: str = "text"
1397
+
1398
+ @staticmethod
1399
+ def from_dict(d: dict[str, Any], notes: Optional[list[FieldNote]] = None,
1400
+ path: str = "") -> "MqttSettings":
1401
+ d = dict(d or {})
1402
+ notes = [] if notes is None else notes
1403
+ return MqttSettings(**_read_fields(MqttSettings, d, notes, path))
1404
+
1405
+ def validate(self) -> None:
1406
+ if not self.enabled:
1407
+ return
1408
+ if not self.host.strip():
1409
+ raise ConfigError(_("MQTT is enabled but no broker host is set."))
1410
+ if not self.base_topic.strip():
1411
+ raise ConfigError(_("MQTT is enabled but no base topic is set."))
1412
+ if not isinstance(self.port, int) or not (1 <= self.port <= 65535):
1413
+ raise ConfigError(_("MQTT port must be between 1 and 65535."))
1414
+ if self.qos not in (0, 1, 2):
1415
+ raise ConfigError(_("MQTT QoS must be 0, 1 or 2."))
1416
+ if self.payload_format not in MQTT_FORMATS:
1417
+ raise ConfigError(_("MQTT payload format must be one of {0}.").format(_choices(MQTT_FORMATS)))
1418
+ if self.payload_format == "sparkplug_b":
1419
+ for label, value in ((_("group ID"), self.sparkplug_group_id),
1420
+ (_("edge node ID"), self.sparkplug_node_id)):
1421
+ if any(c in value for c in "/+#"):
1422
+ raise ConfigError(_("Sparkplug {0} must not contain '/', '+' or '#'.").format(label))
1423
+ if not self.sparkplug_group_id.strip():
1424
+ raise ConfigError(_("Sparkplug B needs a group ID."))
1425
+
1426
+
1427
+ # ---------------------------------------------------------------------------
1428
+ # Modbus register polling (gateway edge mode: read registers -> MQTT)
1429
+ # ---------------------------------------------------------------------------
1430
+ MODBUS_DTYPES = ("uint16", "int16", "uint32", "int32", "float32", "bool",
1431
+ "int64", "uint64", "float64", "string")
1432
+ # how a device lays out a value's bytes in its registers (engine/modbus.WORD_ORDERS): ABCD
1433
+ # as Modbus sends a register, high register first; CDAB low register first ("word swap"),
1434
+ # BADC each register's bytes swapped, DCBA both
1435
+ MODBUS_WORD_ORDERS = ("ABCD", "CDAB", "BADC", "DCBA")
1436
+ MODBUS_STRING_MAX_REGS = 125 # a "string" point: at most one read's worth of registers
1437
+ MODBUS_BLOCK_GAP_MAX = 124 # block reads: unused registers between two points of a block
1438
+ # a point's whole-number keys (the typed reader takes "1" and 1.0 as 1, and leaves 1.5 for
1439
+ # an edit to refuse: validate_input)
1440
+ _POINT_INTS = ("unit", "fn", "address")
1441
+ # the keys of a point and of the poll that 2.9 added: typed from the start, so read as they
1442
+ # are (not through _read_fields), for an edit to refuse what it can't use (validate_input)
1443
+ _POINT_STRICT = ("min", "max", "allowed", "poll", "word_order", "bit", "length")
1444
+ _POLL_STRICT = ("block_reads", "block_gap")
1445
+
1446
+
1447
+ def _loose_number(value: Any, whole: bool) -> Any:
1448
+ """A number as stored before its type was checked (a unit map's key): a numeric string
1449
+ counts as its number, and where a whole number belongs (``whole``) an integral float as
1450
+ an int. Anything else is returned as it is (edits refuse it)."""
1451
+ if isinstance(value, str):
1452
+ text = value.strip()
1453
+ if re.fullmatch(r"[+-]?\d{1,30}", text):
1454
+ return int(text)
1455
+ try:
1456
+ number = float(text)
1457
+ except ValueError:
1458
+ return value
1459
+ if not math.isfinite(number):
1460
+ return value
1461
+ value = number
1462
+ if whole and isinstance(value, float) and math.isfinite(value) and value.is_integer():
1463
+ return int(value)
1464
+ return value
1465
+
1466
+
1467
+ @dataclass
1468
+ class ModbusPoint:
1469
+ name: str = ""
1470
+ unit: int = 1 # RTU slave id
1471
+ fn: int = 3 # 1 = coils, 2 = discrete inputs, 3 = holding, 4 = input registers
1472
+ address: int = 0 # 0-based coil / register address
1473
+ # registers: uint16 | int16 (one), uint32 | int32 | float32 (two), int64 | uint64 |
1474
+ # float64 (four), string ("length" registers), bool (one of its bits: "bit");
1475
+ # coils / discrete inputs: bool
1476
+ dtype: str = "uint16"
1477
+ scale: float = 1.0 # published value = raw * scale (0 = no scaling, as before)
1478
+ # MQTT may write it: <base_topic>/<name>/set, or a Sparkplug B DCMD to its unit's
1479
+ # device (a coil: FC5, a holding register: FC6 / FC16 for its own registers). Only
1480
+ # coils and holding registers; not a string, nor a bit of a register.
1481
+ writable: bool = False
1482
+ # What MQTT may write to it (optional, writable points; None = no limit, as before):
1483
+ # the lowest and highest value, in the point's own units (as published, before the
1484
+ # scale is undone), and the values it may take (a list; true / false for a coil). A
1485
+ # write outside them is refused. Modbus/TCP masters are not limited by them.
1486
+ min: Optional[float] = None
1487
+ max: Optional[float] = None
1488
+ allowed: Optional[list] = None
1489
+ # False: write-only (a writable point that can't be read, such as a command register):
1490
+ # never polled, and a write is not read back
1491
+ poll: bool = True
1492
+ # How the device lays out the value's bytes in its registers (MODBUS_WORD_ORDERS), for
1493
+ # reads and writes; ABCD (the default) as before. A string: ABCD (first character in the
1494
+ # high byte) or BADC. Read through point_order().
1495
+ word_order: str = "ABCD"
1496
+ # dtype "bool" on a register (fn 3, 4): which of its bits it is, 0 (least significant)
1497
+ # to 15
1498
+ bit: Optional[int] = None
1499
+ # dtype "string": how many registers it takes (two characters each), 1-125
1500
+ length: Optional[int] = None
1501
+
1502
+ @staticmethod
1503
+ def from_dict(d: dict[str, Any], notes: Optional[list[FieldNote]] = None,
1504
+ path: str = "") -> "ModbusPoint":
1505
+ d = dict(d or {})
1506
+ notes = [] if notes is None else notes
1507
+ p = ModbusPoint(**_read_fields(ModbusPoint, d, notes, path, skip=_POINT_STRICT))
1508
+ # the keys 2.9 added are typed from the start: kept as they are, for an edit to
1509
+ # refuse what it can't use (ModbusPoll.validate_input); a stored one the gateway
1510
+ # can't use is ignored, with a warning in its log at start
1511
+ for key in _POINT_STRICT:
1512
+ if key in d:
1513
+ setattr(p, key, d[key])
1514
+ return p
1515
+
1516
+
1517
+ def point_scale(p: ModbusPoint) -> Any:
1518
+ """The scale a point's values are read and written with: its own when it is a number
1519
+ (0 and 1: not scaled), else 1. A stored scale the edits refuse (text, NaN: see
1520
+ ModbusPoll.validate_input) is ignored rather than multiplied in, with a warning in the
1521
+ gateway's log at start."""
1522
+ s = p.scale
1523
+ return s if _is_number(s) and math.isfinite(s) else 1.0
1524
+
1525
+
1526
+ def point_order(p: ModbusPoint) -> str:
1527
+ """The word order a point is read and written with: its own, or ABCD for a stored one
1528
+ the edits refuse (with a warning in the gateway's log at start)."""
1529
+ return p.word_order if p.word_order in MODBUS_WORD_ORDERS else "ABCD"
1530
+
1531
+
1532
+ def poll_block_gap(poll: "ModbusPoll") -> Optional[int]:
1533
+ """Block reads (ModbusPoll.block_reads): how many unused registers (coils) may lie
1534
+ between two points read with one request, or None when block reads are off. A stored
1535
+ gap the edits refuse reads as 0, and anything but true as off."""
1536
+ if poll.block_reads is not True:
1537
+ return None
1538
+ gap = poll.block_gap
1539
+ return gap if _is_int(gap) and 0 <= gap <= MODBUS_BLOCK_GAP_MAX else 0
1540
+
1541
+
1542
+ def _register_bit(value: Any) -> bool:
1543
+ return _is_int(value) and 0 <= value <= 15
1544
+
1545
+
1546
+ def _string_length(value: Any) -> bool:
1547
+ return _is_int(value) and 1 <= value <= MODBUS_STRING_MAX_REGS
1548
+
1549
+
1550
+ MODBUS_POLL_DEFAULT_S = 5.0
1551
+
1552
+
1553
+ @dataclass
1554
+ class ModbusPoll:
1555
+ interval_s: float = MODBUS_POLL_DEFAULT_S
1556
+ points: list[ModbusPoint] = field(default_factory=list)
1557
+ # Block reads (off: one request per point, as before): a unit's points of one function
1558
+ # that lie close together are read with one request, at most 125 registers (2000 coils
1559
+ # or inputs), with up to block_gap unused registers (coils) between two of them. A
1560
+ # block the slave refuses (exception 02 or 03) has its points read one at a time from
1561
+ # then on. Read through poll_block_gap().
1562
+ block_reads: bool = False
1563
+ block_gap: int = 0
1564
+
1565
+ @staticmethod
1566
+ def from_dict(d: dict[str, Any], notes: Optional[list[FieldNote]] = None,
1567
+ path: str = "") -> "ModbusPoll":
1568
+ d = dict(d or {})
1569
+ notes = [] if notes is None else notes
1570
+ poll = ModbusPoll(
1571
+ points=[ModbusPoint.from_dict(p, notes, f"{path}points[{i}].")
1572
+ for i, p in enumerate(_objects(d, "points", path))],
1573
+ **_read_fields(ModbusPoll, d, notes, path, skip=("points", *_POLL_STRICT)),
1574
+ **{key: d[key] for key in _POLL_STRICT if key in d})
1575
+ # 0 or less has always meant the default interval (an <input min> keeps the form
1576
+ # from sending it); said now, where it matters
1577
+ if poll.interval_s <= 0:
1578
+ if poll.points:
1579
+ was, now = _shown(poll.interval_s), f"{MODBUS_POLL_DEFAULT_S:g}"
1580
+ notes.append(FieldNote(
1581
+ path + "interval_s", "replaced", poll.interval_s, MODBUS_POLL_DEFAULT_S,
1582
+ why=f"{path}interval_s {was} is not a poll interval (more than 0 "
1583
+ f"seconds): {now} s is used",
1584
+ warning=_("{0}interval_s {1} is not a poll interval (more than 0 seconds): "
1585
+ "{2} s is used.").format(path, was, now)))
1586
+ poll.interval_s = MODBUS_POLL_DEFAULT_S
1587
+ return poll
1588
+
1589
+ @property
1590
+ def broadcast_points(self) -> list[str]:
1591
+ """The names of the points on unit 0, the broadcast address: never read (no slave
1592
+ answers a broadcast), and a write to one goes to every slave. Allowed, with a
1593
+ warning in the log and the mapping form."""
1594
+ return [p.name for p in self.points if p.unit == 0]
1595
+
1596
+ @property
1597
+ def reserved_unit_points(self) -> list[str]:
1598
+ """The names of the points on units 248-255, which a Modbus serial line reserves
1599
+ (slave addresses are 1-247). Allowed, with a warning like broadcast_points."""
1600
+ return [p.name for p in self.points if _is_int(p.unit) and 248 <= p.unit <= 255]
1601
+
1602
+ def validate(self) -> None:
1603
+ if not self.points:
1604
+ return
1605
+ if not _is_number(self.interval_s) or not (0 < self.interval_s < math.inf):
1606
+ raise ConfigError(_("Modbus poll interval must be > 0 seconds."))
1607
+ seen = set()
1608
+ for p in self.points:
1609
+ if not isinstance(p.name, str) and p.name is not None: # (a model built in code)
1610
+ raise ConfigError(_("Modbus poll point names are text: {0} is not.").format(
1611
+ repr(p.name)[:40]))
1612
+ if not isinstance(p.name, str) or not p.name.strip():
1613
+ raise ConfigError(_("Each Modbus poll point needs a name."))
1614
+ if p.name in seen:
1615
+ raise ConfigError(_("Duplicate Modbus poll point name: {0}.").format(repr(p.name)))
1616
+ seen.add(p.name)
1617
+ # (a number of any type compares as it did; anything else failed before too)
1618
+ if not isinstance(p.unit, (int, float)) or not (0 <= p.unit <= 255):
1619
+ raise ConfigError(_("Point {0}: unit must be 0-255.").format(repr(p.name)))
1620
+ if p.fn not in (1, 2, 3, 4) or not _is_int(p.fn):
1621
+ raise ConfigError(_("Point {0}: function must be 1 (coils), 2 (discrete "
1622
+ "inputs), 3 (holding registers) or 4 (input registers).").format(repr(p.name)))
1623
+ if not isinstance(p.address, (int, float)) or not (0 <= p.address <= 65535):
1624
+ raise ConfigError(_("Point {0}: address must be 0-65535.").format(repr(p.name)))
1625
+ # from_dict reads both as numbers / true or false; a model built in code is
1626
+ # refused here rather than fail at every poll
1627
+ if not _is_number(p.scale) or not (isinstance(p.scale, int) or math.isfinite(p.scale)):
1628
+ raise ConfigError(_("Point {0}: scale must be a number.").format(repr(p.name)))
1629
+ if not isinstance(p.writable, bool):
1630
+ raise ConfigError(_("Point {0}: writable must be true or false.").format(repr(p.name)))
1631
+ if p.dtype not in MODBUS_DTYPES:
1632
+ raise ConfigError(_("Point {0}: dtype must be one of {1}.").format(repr(p.name), _choices(MODBUS_DTYPES)))
1633
+ # (bool on a register and string are new in 2.9: what was refused stays refused
1634
+ # without the bit / length they need)
1635
+ if (p.fn in (1, 2)) != (p.dtype == "bool") and not (
1636
+ p.dtype == "bool" and _register_bit(p.bit)):
1637
+ raise ConfigError(_("Point {0}: coils / discrete inputs (fn 1, 2) use dtype "
1638
+ "'bool'; registers (fn 3, 4) a numeric dtype or 'string', "
1639
+ "or 'bool' with the \"bit\" (0-15) it reads.").format(
1640
+ repr(p.name)))
1641
+ if p.dtype == "string" and not _string_length(p.length):
1642
+ raise ConfigError(_("Point {0}: a string needs its \"length\": how many "
1643
+ "registers it takes (1-{1}, two characters each).").format(
1644
+ repr(p.name), MODBUS_STRING_MAX_REGS))
1645
+ if p.writable and p.fn not in (1, 3):
1646
+ raise ConfigError(_("Point {0}: only coils (fn 1) and holding registers (fn 3) "
1647
+ "can be written.").format(repr(p.name)))
1648
+
1649
+ def validate_input(self) -> None:
1650
+ """The point options an edit may set (min, max, allowed, poll, word_order) and the
1651
+ types of the rest. validate() leaves them alone (a stored config has to keep
1652
+ loading); one stored that can't be used is ignored, with a warning in the gateway's
1653
+ log at start."""
1654
+ if not isinstance(self.block_reads, bool):
1655
+ raise ConfigError(_("Block reads must be on or off (true or false)."))
1656
+ if not (_is_int(self.block_gap) and 0 <= self.block_gap <= MODBUS_BLOCK_GAP_MAX):
1657
+ raise ConfigError(_("The unused registers a block read may span must be a whole "
1658
+ "number from 0 to {0}.").format(MODBUS_BLOCK_GAP_MAX))
1659
+ for p in self.points:
1660
+ name = repr(p.name)
1661
+ # the types validate() leaves alone: a stored point loads as it was
1662
+ for key in _POINT_INTS:
1663
+ if not _is_int(getattr(p, key)):
1664
+ raise ConfigError(_("Point {0}: {1} must be a whole number.").format(
1665
+ name, key))
1666
+ if not (_is_number(p.scale) and math.isfinite(p.scale)):
1667
+ raise ConfigError(_("Point {0}: scale must be a number (0 or 1: not "
1668
+ "scaled).").format(name))
1669
+ if not isinstance(p.writable, bool):
1670
+ raise ConfigError(_("Point {0}: writable must be true or false.").format(name))
1671
+ register = p.fn in (3, 4)
1672
+ if p.word_order not in MODBUS_WORD_ORDERS:
1673
+ raise ConfigError(_("Point {0}: word_order must be one of {1}.").format(
1674
+ name, ", ".join(MODBUS_WORD_ORDERS)))
1675
+ if p.dtype == "bool" and p.word_order != "ABCD":
1676
+ raise ConfigError(_("Point {0}: word_order is for register values, not for a "
1677
+ "bit.").format(name))
1678
+ if p.dtype == "string" and p.word_order not in ("ABCD", "BADC"):
1679
+ raise ConfigError(_("Point {0}: a string's characters are in the high byte "
1680
+ "first (\"ABCD\") or the low byte first (\"BADC\").").format(
1681
+ name))
1682
+ if p.bit is not None and not (p.dtype == "bool" and register):
1683
+ raise ConfigError(_("Point {0}: \"bit\" reads one bit of a register: it needs "
1684
+ "\"dtype\": \"bool\" and fn 3 or 4.").format(name))
1685
+ if p.length is not None and p.dtype != "string":
1686
+ raise ConfigError(_("Point {0}: \"length\" is for a \"string\" point.").format(
1687
+ name))
1688
+ if p.dtype == "string" and p.scale not in (0, 1):
1689
+ raise ConfigError(_("Point {0}: a string can't be scaled.").format(name))
1690
+ if p.writable and (p.dtype == "string" or (p.dtype == "bool" and register)):
1691
+ raise ConfigError(_("Point {0}: MQTT can't write a string or a bit of a "
1692
+ "register; make it read-only.").format(name))
1693
+ if not p.writable and any(v is not None for v in (p.min, p.max, p.allowed)):
1694
+ raise ConfigError(_("Point {0}: min, max and allowed limit what MQTT may "
1695
+ "write, so they need \"writable\": true.").format(name))
1696
+ coil = p.fn in (1, 2)
1697
+ if coil and (p.min is not None or p.max is not None):
1698
+ raise ConfigError(_("Point {0}: a coil is true or false, so min and max don't "
1699
+ "apply to it (use allowed).").format(name))
1700
+ for key, value in (("min", p.min), ("max", p.max)):
1701
+ if value is not None and not (_is_number(value) and math.isfinite(value)):
1702
+ raise ConfigError(_("Point {0}: {1} must be a number.").format(name, key))
1703
+ if p.min is not None and p.max is not None and p.min > p.max:
1704
+ raise ConfigError(_("Point {0}: min is greater than max.").format(name))
1705
+ if p.allowed is not None and not (
1706
+ isinstance(p.allowed, list) and p.allowed
1707
+ and all(isinstance(v, bool) if coil else
1708
+ (_is_number(v) and math.isfinite(v)) for v in p.allowed)):
1709
+ raise ConfigError(_("Point {0}: allowed must be a list of the values MQTT may "
1710
+ "write: true / false for a coil, numbers for a "
1711
+ "register.").format(name))
1712
+ if not isinstance(p.poll, bool):
1713
+ raise ConfigError(_("Point {0}: poll must be true or false.").format(name))
1714
+ if not p.poll and not p.writable:
1715
+ raise ConfigError(_("Point {0}: a point that is not polled (\"poll\": false) "
1716
+ "is write-only, so it needs \"writable\": true.").format(
1717
+ name))
1718
+
1719
+
1720
+ def unstorable_text(value: Any, where: str = "") -> Optional[str]:
1721
+ """Where in ``value`` (a to_dict() tree: dicts, lists, strings) there is text that can't
1722
+ be written as UTF-8 — an unpaired surrogate (U+D800-U+DFFF), which only a JSON escape
1723
+ such as \\ud800 can make — as a path like "modbus_poll.points[0].name"; None when there
1724
+ is none. config.json could not be saved with it."""
1725
+ if isinstance(value, str):
1726
+ try:
1727
+ value.encode("utf-8")
1728
+ except UnicodeEncodeError:
1729
+ return where or "?"
1730
+ return None
1731
+ if isinstance(value, dict):
1732
+ for k, v in value.items():
1733
+ key = str(k).encode("utf-8", "backslashreplace").decode("utf-8")
1734
+ spot = f"{where}.{key}" if where else key
1735
+ if (bad := unstorable_text(k, spot)) or (bad := unstorable_text(v, spot)):
1736
+ return bad
1737
+ elif isinstance(value, (list, tuple)):
1738
+ for i, v in enumerate(value):
1739
+ if bad := unstorable_text(v, f"{where}[{i}]"):
1740
+ return bad
1741
+ return None
1742
+
1743
+
1744
+ # ---------------------------------------------------------------------------
1745
+ # A single serial<->TCP mapping
1746
+ # ---------------------------------------------------------------------------
1747
+ MAPPING_KINDS = ("net", "serialbridge")
1748
+
1749
+
1750
+ @dataclass
1751
+ class MappingConfig:
1752
+ id: str = field(default_factory=_new_id)
1753
+ name: str = ""
1754
+ enabled: bool = True
1755
+ kind: str = "net" # net (serial<->network) | serialbridge (serial<->serial)
1756
+ serial: SerialSettings = field(default_factory=SerialSettings)
1757
+ serial_b: SerialSettings = field(default_factory=SerialSettings) # serialbridge only
1758
+ network: NetworkSettings = field(default_factory=NetworkSettings)
1759
+ options: MappingOptions = field(default_factory=MappingOptions)
1760
+ mqtt: MqttSettings = field(default_factory=MqttSettings)
1761
+ modbus_poll: ModbusPoll = field(default_factory=ModbusPoll)
1762
+ # What from_dict did with the data it was given (FieldNote): loading config.json logs
1763
+ # it, an edit refuses an invalid value (validate_input) and warns about the rest.
1764
+ # Never persisted.
1765
+ input_notes: list[FieldNote] = field(default_factory=list, repr=False, compare=False)
1766
+
1767
+ # what a mapping read back from the REST API has besides its settings (never stored)
1768
+ _READ_ONLY_KEYS = ("status", "warnings")
1769
+
1770
+ @staticmethod
1771
+ def from_dict(d: dict[str, Any]) -> "MappingConfig":
1772
+ # every way into the config (file, import, form, REST, cluster) passes here, so an
1773
+ # id that is not a valid mapping id never gets further (see _mapping_id_from), nor a
1774
+ # value of the wrong type (see _read_fields)
1775
+ d = dict(d or {})
1776
+ notes: list[FieldNote] = []
1777
+ readers: dict[str, Any] = {
1778
+ "serial": SerialSettings.from_dict, "serial_b": SerialSettings.from_dict,
1779
+ "network": NetworkSettings.from_dict, "options": MappingOptions.from_dict,
1780
+ "mqtt": MqttSettings.from_dict, "modbus_poll": ModbusPoll.from_dict}
1781
+ fields = _read_fields(MappingConfig, d, notes, skip=("id", "input_notes", *readers),
1782
+ keep=MappingConfig._READ_ONLY_KEYS)
1783
+ sections = {key: read(_section(d, key), notes, key + ".") for key, read in readers.items()}
1784
+ m = MappingConfig(id=_mapping_id_from(d.get("id")), input_notes=notes,
1785
+ **sections, **fields)
1786
+ net = d.get("network")
1787
+ if (isinstance(net, dict) and "dual_stack" not in net and m.kind == "net"
1788
+ and m.network.mode == "udp" and m.network.bind_ip == "::"
1789
+ and _ipv6_takes_ipv4()):
1790
+ # Written by (or for) a release without dual_stack, whose UDP socket on :: kept
1791
+ # the OS default: on Linux it took IPv4 datagrams too. IPV6_V6ONLY is set now
1792
+ # (NetworkSettings.dual_stack), so such a mapping keeps doing what it did here.
1793
+ m.network.dual_stack = True
1794
+ return m
1795
+
1796
+ def to_dict(self) -> dict[str, Any]:
1797
+ d = asdict(self)
1798
+ d.pop("input_notes", None)
1799
+ return d
1800
+
1801
+ def input_warnings(self) -> list[str]:
1802
+ """What an edit (REST API, import) is told about the data it sent: keys that are no
1803
+ setting, settings replaced by supported ones (in the request's language; the REST
1804
+ API's is English)."""
1805
+ unknown = [n.path for n in self.input_notes if n.kind == "unknown"]
1806
+ out = ([_("Not settings of this version, so they are ignored: {0}.").format(
1807
+ ", ".join(unknown))] if unknown else [])
1808
+ return out + [n.warning for n in self.input_notes if n.kind == "replaced"]
1809
+
1810
+ def validate(self) -> None:
1811
+ if not self.name.strip():
1812
+ raise ConfigError(_("Mapping name is required."))
1813
+ if self.kind not in MAPPING_KINDS:
1814
+ raise ConfigError(_("Mapping kind must be one of {0}.").format(_choices(MAPPING_KINDS)))
1815
+ self.serial.validate()
1816
+ if self.kind == "serialbridge":
1817
+ self.serial_b.validate()
1818
+ if self.serial_b.port == self.serial.port:
1819
+ raise ConfigError(_("Serial-to-serial bridge needs two different ports."))
1820
+ else:
1821
+ self.network.validate()
1822
+ self.options.validate()
1823
+ self.mqtt.validate()
1824
+ self.modbus_poll.validate()
1825
+ if self.modbus_poll.points and not (self.kind == "net" and self.network.protocol == "modbus"):
1826
+ raise ConfigError(_("Modbus register polling requires a Modbus-gateway mapping "
1827
+ "(protocol = modbus)."))
1828
+ if (self.mqtt.enabled and self.mqtt.payload_format == "sparkplug_b"
1829
+ and not self.modbus_poll.points):
1830
+ raise ConfigError(_("Sparkplug B publishes Modbus poll points: add points (and use "
1831
+ "protocol = modbus), or use the plain MQTT format."))
1832
+
1833
+ def ignored_settings(self) -> list[str]:
1834
+ """Settings this mapping has that do nothing, in the current language: the runner
1835
+ logs them when it starts (in English), the edit form and the REST API show them.
1836
+ validate() accepts them, as earlier versions did."""
1837
+ out = []
1838
+ if self.mqtt.enabled and self.kind == "serialbridge":
1839
+ out.append(_("MQTT is on, but a serial-to-serial bridge does not publish to MQTT: "
1840
+ "its MQTT settings are not used."))
1841
+ elif (self.mqtt.enabled and self.kind == "net" and self.network.protocol == "modbus"
1842
+ and not self.modbus_poll.points):
1843
+ out.append(_("MQTT is on, but a Modbus gateway publishes only its poll points, and "
1844
+ "it has none: nothing is published. Add poll points, or turn MQTT off."))
1845
+ if self.kind == "net" and self.network.mode == "udp" and self.options.closeon:
1846
+ out.append(_("A close-on string is set, but a UDP mapping has no connections to "
1847
+ "close: it is not used."))
1848
+ if self.kind == "net": # a list an edit would refuse: only in a hand-edited file
1849
+ nets, bad = client_networks(self.network.allowed_client_ips)
1850
+ if bad:
1851
+ out.append(_("Allowed client IPs: these entries are not IP addresses or "
1852
+ "networks, so they are left out: {0}.").format(", ".join(bad))
1853
+ + ("" if nets else " " + _("No entry is usable, so every client "
1854
+ "is refused.")))
1855
+ _nets, bad = client_networks(self.network.priority_client_ips)
1856
+ if bad:
1857
+ out.append(_("High-priority client IPs: these entries are not IP addresses or "
1858
+ "networks, so they are left out: {0}.").format(", ".join(bad)))
1859
+ return out
1860
+
1861
+ def serial_sides(self) -> list[SerialSettings]:
1862
+ """The serial ports this mapping opens: A, and B for a serial-to-serial bridge."""
1863
+ return [self.serial, self.serial_b] if self.kind == "serialbridge" else [self.serial]
1864
+
1865
+ @property
1866
+ def accepts_mqtt_writes(self) -> bool:
1867
+ """MQTT commands can write to the bus: a Modbus gateway with MQTT on, writable
1868
+ points and Read-only off. Who may send them (<base_topic>/<point>/set, or the
1869
+ Sparkplug NCMD / DCMD topics) is up to the broker's access rules."""
1870
+ return (self.kind == "net" and self.network.protocol == "modbus" and self.mqtt.enabled
1871
+ and not self.network.read_only
1872
+ and any(p.writable for p in self.modbus_poll.points))
1873
+
1874
+ @property
1875
+ def accepts_mqtt_tx(self) -> bool:
1876
+ """MQTT messages on <base_topic>/tx are written to the serial port: a serial-to-network
1877
+ mapping (not a Modbus gateway) with MQTT and tx_enabled on and Read-only off. Who may
1878
+ send them is up to the broker's access rules."""
1879
+ return (self.kind == "net" and self.network.protocol != "modbus" and self.mqtt.enabled
1880
+ and self.mqtt.tx_enabled is True and not self.network.read_only)
1881
+
1882
+ def validate_input(self) -> None:
1883
+ """The rules for a mapping that is being created or changed (web form, REST API,
1884
+ import, cluster), checked there in addition to validate(). validate() must not
1885
+ refuse them: a stored config has to keep loading and saving (Docker re-saves it on
1886
+ every start), so a stored value they forbid is refused where it is used instead
1887
+ (a URL serial port: when the mapping starts, see engine/serial_io.refuse_url), or
1888
+ normalized when it is loaded (a value of the wrong type: from_dict)."""
1889
+ if (spot := unstorable_text(self.to_dict())) is not None:
1890
+ raise ConfigError(_("{0} has a character that can't be stored: an unpaired "
1891
+ "surrogate escape (\\ud800 to \\udfff).").format(spot))
1892
+ for note in self.input_notes:
1893
+ if note.kind in ("invalid", "switch_text"):
1894
+ raise ConfigError(note.error())
1895
+ if self.kind == "net" and (missing := tls_files_missing(self.network)):
1896
+ raise ConfigError(missing[0])
1897
+ for side in self.serial_sides():
1898
+ if problem := port_problem(side.port):
1899
+ raise ConfigError(problem)
1900
+ if (self.kind == "serialbridge" and self.serial.port
1901
+ and canonical_device(self.serial.port) == canonical_device(self.serial_b.port)):
1902
+ raise ConfigError(_("Serial-to-serial bridge needs two different ports."))
1903
+ net = self.network
1904
+ if self.kind == "net" and net.listens and net.dual_stack is True and not net.both_families:
1905
+ raise ConfigError(_("Listening for IPv4 and IPv6 clients (dual_stack) needs the bind "
1906
+ "IP 0.0.0.0 or ::, not {0}.").format(net.bind_ip))
1907
+ if self.kind == "net" and net.mode == "udp":
1908
+ self._validate_udp_peer()
1909
+ sig = self.options.rfc2217_signature
1910
+ if not isinstance(sig, str) or len(sig) > RFC2217_SIGNATURE_MAX or _CONTROL_CHARS.search(sig):
1911
+ raise ConfigError(_("The RFC 2217 signature must be text of at most {0} characters, "
1912
+ "without control characters.").format(RFC2217_SIGNATURE_MAX))
1913
+ if self.kind == "net" and self.network.protocol != "modbus":
1914
+ self._validate_lines_and_poll()
1915
+ if self.kind == "net" and net.tls and net.mode == "client":
1916
+ from .engine.tls import parse_pins
1917
+ try:
1918
+ parse_pins(net.tls_fingerprint)
1919
+ except ValueError as e:
1920
+ raise ConfigError(_("TLS server fingerprint: {0} is not a SHA-256 fingerprint "
1921
+ "(64 hex digits, with or without colons).").format(
1922
+ repr(str(e)))) from None
1923
+ if self.options.console_encoding not in CONSOLE_ENCODINGS:
1924
+ raise ConfigError(_("Unknown console text encoding {0}: use one of {1}.").format(
1925
+ repr(self.options.console_encoding), ", ".join(CONSOLE_ENCODINGS)))
1926
+ if self.kind == "net" and self.network.protocol == "modbus":
1927
+ low, high = MODBUS_TIMEOUT_RANGE_S
1928
+ timeout = self.options.modbus_response_timeout_s
1929
+ if not _is_number(timeout) or not (low <= timeout <= high):
1930
+ raise ConfigError(_("The Modbus response timeout must be between {0} and {1} "
1931
+ "seconds.").format(f"{low:g}", f"{high:g}"))
1932
+ if self.mqtt.value_format not in MQTT_VALUE_FORMATS:
1933
+ raise ConfigError(_("MQTT value format must be one of {0}.").format(
1934
+ MQTT_VALUE_FORMATS))
1935
+ umap = self.options.modbus_unit_map
1936
+ if not isinstance(umap, dict) or len(modbus_unit_map(self.options)) != len(umap):
1937
+ raise ConfigError(_("The unit map sends unit ids 0-255 to RTU units 1-255, "
1938
+ "each id once, such as 255=1."))
1939
+ self.modbus_poll.validate_input()
1940
+ host = self.mqtt.sparkplug_primary_host_id
1941
+ if self.mqtt.payload_format == "sparkplug_b" and host != "":
1942
+ if not isinstance(host, str) or any(c in host for c in "/+#"):
1943
+ raise ConfigError(_("Sparkplug {0} must not contain '/', '+' or '#'.").format(
1944
+ _("primary host ID")))
1945
+ if self.kind == "net" and self.mqtt.enabled and _plain_mqtt(self):
1946
+ # (serial lines and their /tx writes too: a subscription to such a topic would
1947
+ # take every matching one)
1948
+ if mqtt_topic_problem(self.mqtt.base_topic):
1949
+ raise ConfigError(_("The MQTT base topic can't contain '+', '#' or the null "
1950
+ "character: no message can be published to such a topic."))
1951
+ self.check_mqtt_topics()
1952
+
1953
+ def check_tls_files(self) -> None:
1954
+ """An edit's check (mapping form, so also a cluster peer's save on the node it is
1955
+ for; REST API; an import, as a warning), not part of validate()/validate_input(),
1956
+ which can't load files: a TLS mapping's certificate, key and CA file must load as
1957
+ the mapping will load them (engine/tls.py), its key decrypted with tls_key_password,
1958
+ so a key that needs a password, a wrong password, a key that is not the
1959
+ certificate's or a CA file without a certificate is refused here rather than found
1960
+ out when the mapping starts. Raises ConfigError."""
1961
+ net = self.network
1962
+ if not (self.kind == "net" and net.tls and net.mode != "udp"):
1963
+ return
1964
+ from .engine.tls import TlsFileError, mapping_context
1965
+ try:
1966
+ mapping_context(net)
1967
+ except TlsFileError as e:
1968
+ raise ConfigError(tls_file_message(e)) from None
1969
+
1970
+ def _validate_lines_and_poll(self) -> None:
1971
+ """A serial-to-network mapping's MQTT line end and its poll string (an edit's
1972
+ rules: a stored value they refuse reads as off / "lf"). Its base topic, also for
1973
+ MQTT writes, is checked with every plain-format mapping's (validate_input)."""
1974
+ mq = self.mqtt
1975
+ if mq.enabled and mq.line_end not in MQTT_LINE_ENDS:
1976
+ raise ConfigError(_("The MQTT line end must be one of {0}.").format(MQTT_LINE_ENDS))
1977
+ o = self.options
1978
+ interval = o.poll_interval_s
1979
+ if not _is_number(interval) or not (interval == 0 or 0.1 <= interval <= 86400):
1980
+ raise ConfigError(_("The poll interval must be 0 (off) or between 0.1 and 86400 "
1981
+ "seconds."))
1982
+ try:
1983
+ data = poll_bytes(o.poll_string)
1984
+ except ValueError:
1985
+ raise ConfigError(_(r"The poll string can have only \r \n \t \\ or \xHH after a "
1986
+ "backslash.")) from None
1987
+ if len(data) > POLL_STRING_MAX:
1988
+ raise ConfigError(_("The poll string can be at most {0} bytes.").format(
1989
+ POLL_STRING_MAX))
1990
+ if interval and not data:
1991
+ raise ConfigError(_("A poll interval needs a poll string to send."))
1992
+
1993
+ def check_mqtt_topics(self) -> None:
1994
+ """Plain MQTT on a Modbus gateway (an edit's rules, see validate_input): a point's
1995
+ name is part of its topics (<base_topic>/<name>, .../set, .../set/result), so it
1996
+ can't have '+', '#' or the null character, and a point can't be published where the
1997
+ mapping already publishes or takes something else: its status (<base_topic>/status),
1998
+ a unit's status (unit<N>/status, with unit_status on) or another point's command or
1999
+ result topic. A stored mapping that breaks them keeps loading and running; start-up
2000
+ and each start of it log a warning (mqtt_warnings). Raises ConfigError."""
2001
+ if not (self.kind == "net" and self.network.protocol == "modbus"):
2002
+ return
2003
+ for p in self.modbus_poll.points:
2004
+ if mqtt_topic_problem(p.name):
2005
+ raise ConfigError(_("Point {0}: with the plain MQTT format its name is part "
2006
+ "of its topic, which can't contain '+', '#' or the null "
2007
+ "character.").format(repr(p.name)))
2008
+ uses: dict[str, tuple[str, Any]] = {}
2009
+ for topic, use, of in sorted(_mqtt_topics(self), key=lambda t: t[1] == "value"):
2010
+ if topic not in uses:
2011
+ uses[topic] = (use, of)
2012
+ continue
2013
+ # (an overlap is always a point's value topic: the others can't meet)
2014
+ raise ConfigError(_("Point {0}: its MQTT topic {1} is already {2}. Rename the "
2015
+ "point.").format(repr(of), repr(topic), _topic_use(*uses[topic])))
2016
+
2017
+ def check_mqtt_tls_files(self) -> None:
2018
+ """An edit on the node that runs the mapping (its form, the REST API): with MQTT
2019
+ over TLS, the CA file and the client certificate and key must be usable here.
2020
+ Neither validate() nor validate_input() checks them: a stored config whose files
2021
+ are gone has to keep loading (the mapping then runs without MQTT, logged), a fleet
2022
+ edit is checked by the node it is for (its own form save), and an import only warns.
2023
+ Raises ConfigError."""
2024
+ mq = self.mqtt
2025
+ if not (self.kind == "net" and mq.enabled and mq.tls):
2026
+ return
2027
+ from .engine.mqtt_pub import TlsFileError, tls_context
2028
+ try:
2029
+ tls_context(mq)
2030
+ except TlsFileError as e:
2031
+ text = {
2032
+ "ca_unreadable": _("The MQTT CA file {0} can't be read: {1}"),
2033
+ "ca_empty": _("The MQTT CA file {0} holds no certificate (PEM or DER)."),
2034
+ "key_without_cert": _("An MQTT client key needs its client certificate."),
2035
+ "cert_unreadable": _("The MQTT client certificate or key ({0}) can't be read: "
2036
+ "{1}"),
2037
+ "cert_unusable": _("The MQTT client certificate {0} can't be used with its key "
2038
+ "({1}). Is the key password right, and is it the "
2039
+ "certificate's key?"),
2040
+ }[e.kind]
2041
+ raise ConfigError(text.format(e.path, e.detail)) from None
2042
+
2043
+ def _validate_udp_peer(self) -> None:
2044
+ """A UDP mapping's reply address and peer timeout (an edit's rules: a stored
2045
+ malformed reply address makes the mapping fail to start, see udp_peer_address)."""
2046
+ net = self.network
2047
+ host = str(net.udp_peer_host or "").strip()
2048
+ if host:
2049
+ try:
2050
+ ip = ipaddress.ip_address(host)
2051
+ except ValueError:
2052
+ ip = None
2053
+ if ip is None or ip.is_unspecified:
2054
+ raise ConfigError(_("The UDP reply address must be a host's IP address, not "
2055
+ "{0}.").format(repr(host)))
2056
+ if not _is_int(net.udp_peer_port) or not (1 <= net.udp_peer_port <= 65535):
2057
+ raise ConfigError(_("The UDP reply port must be between 1 and 65535."))
2058
+ try:
2059
+ bind_v4 = ipaddress.ip_address(net.bind_ip).version == 4
2060
+ except ValueError:
2061
+ bind_v4 = False # validate() reports a bad bind IP
2062
+ if ip.version == 6 and bind_v4 and not net.both_families:
2063
+ raise ConfigError(_("The UDP reply address {0} is IPv6, but the mapping listens "
2064
+ "on an IPv4 address ({1}).").format(host, net.bind_ip))
2065
+ if ip.version == 4 and not bind_v4 and not net.both_families:
2066
+ raise ConfigError(_("The UDP reply address {0} is IPv4, but the mapping listens "
2067
+ "on an IPv6 address ({1}). To reach IPv4 addresses from "
2068
+ ":: pick “All interfaces (IPv4 + IPv6)”.").format(
2069
+ host, net.bind_ip))
2070
+ timeout = net.udp_peer_timeout_s
2071
+ if not _is_number(timeout) or not (0 <= timeout <= MAX_TIMER_S):
2072
+ raise ConfigError(_("{0} must be between 0 (off) and {1} seconds.").format(
2073
+ _("Peer timeout"), MAX_TIMER_S))
2074
+ self._validate_udp_targets()
2075
+
2076
+ def _validate_udp_targets(self) -> None:
2077
+ """A UDP mapping's targets: IP:port entries of the family the mapping listens on, at
2078
+ most UDP_TARGETS_MAX; 255.255.255.255 needs udp_broadcast (an edit's rules: a stored
2079
+ malformed one makes the mapping fail to start, see udp_targets)."""
2080
+ net = self.network
2081
+ if not isinstance(net.udp_targets, list):
2082
+ raise ConfigError(_("The UDP targets must be a list of IP:port entries."))
2083
+ if len(net.udp_targets) > UDP_TARGETS_MAX:
2084
+ raise ConfigError(_("At most {0} UDP targets can be set.").format(UDP_TARGETS_MAX))
2085
+ try:
2086
+ bind_v4 = ipaddress.ip_address(net.bind_ip).version == 4
2087
+ except ValueError:
2088
+ bind_v4 = False # validate() reports a bad bind IP
2089
+ for entry in net.udp_targets:
2090
+ try:
2091
+ host, _port = udp_target(entry)
2092
+ except ValueError:
2093
+ raise ConfigError(_("UDP target {0} is not an IP address and a port (IP:port, "
2094
+ "[IPv6]:port).").format(repr(entry))) from None
2095
+ ip = ipaddress.ip_address(host)
2096
+ if not net.both_families and (ip.version == 6) == bind_v4:
2097
+ raise ConfigError(_("UDP target {0} is IPv{1}, but the mapping listens on {2}.")
2098
+ .format(entry, ip.version, net.bind_ip))
2099
+ if host == "255.255.255.255" and net.udp_broadcast is not True:
2100
+ raise ConfigError(_("UDP target {0} is the broadcast address: tick “Allow "
2101
+ "broadcast”.").format(entry))
2102
+
2103
+
2104
+ POLL_STRING_MAX = 1024 # bytes (an edit's rule)
2105
+ _POLL_ESCAPE = re.compile(r"\\(x[0-9A-Fa-f]{2}|[rnt\\])|\\")
2106
+
2107
+
2108
+ def poll_bytes(text: Any) -> bytes:
2109
+ r"""options.poll_string as the bytes sent: UTF-8, with \r \n \t \\ and \xHH for those
2110
+ bytes. Raises ValueError for another backslash sequence (an edit refuses it; a stored
2111
+ one: the poll is off)."""
2112
+ if not isinstance(text, str):
2113
+ raise ValueError("not text")
2114
+ out = bytearray()
2115
+ pos = 0
2116
+ for m in _POLL_ESCAPE.finditer(text):
2117
+ out += text[pos:m.start()].encode("utf-8")
2118
+ esc = m.group(1)
2119
+ if esc is None:
2120
+ raise ValueError(f"unknown escape at character {m.start() + 1} (use \\r \\n "
2121
+ "\\t \\\\ or \\xHH)")
2122
+ out += (bytes([int(esc[1:], 16)]) if esc[0] == "x"
2123
+ else {"r": b"\r", "n": b"\n", "t": b"\t", "\\": b"\\"}[esc])
2124
+ pos = m.end()
2125
+ out += text[pos:].encode("utf-8")
2126
+ return bytes(out)
2127
+
2128
+
2129
+ def mqtt_topic_problem(text: Any) -> bool:
2130
+ """'+' / '#' (wildcards: a message can't be published to them, and a subscription would
2131
+ take every matching topic) or U+0000 (which no MQTT string may have) in a topic, or in
2132
+ the part of one a name makes (a base topic, a point's name)."""
2133
+ return any(c in str(text) for c in "+#\x00")
2134
+
2135
+
2136
+ def mqtt_tx_topics(m: "MappingConfig") -> list[tuple[str, str, Any]]:
2137
+ """The topics a serial-to-network mapping's MQTT writes use, as (topic, use, None): use
2138
+ "tx" (the command topic it takes writes from) and "tx_result" (its answers); none when
2139
+ it takes no writes (tx_enabled off, MQTT off, a Modbus gateway). The plain-format topic
2140
+ rules of the Modbus gateway (no wildcard, the status topic reserved) apply to them."""
2141
+ mq = m.mqtt
2142
+ if not (m.kind == "net" and m.network.protocol != "modbus" and mq.enabled
2143
+ and mq.tx_enabled is True):
2144
+ return []
2145
+ base = str(mq.base_topic).rstrip("/")
2146
+ return [(f"{base}/tx", "tx", None), (f"{base}/tx/result", "tx_result", None)]
2147
+
2148
+
2149
+ def tls_file_message(e: Any) -> str:
2150
+ """The UI's text for an engine.tls.TlsFileError."""
2151
+ if e.kind == "unreadable":
2152
+ return _("The TLS file {0} can't be read: {1}").format(e.path, e.detail)
2153
+ if e.kind == "password":
2154
+ return _("The TLS key {0} is password-protected: enter its password (TLS key "
2155
+ "password).").format(e.path)
2156
+ if e.kind == "unusable":
2157
+ return _("The TLS certificate {0} can't be used with its key ({1}). Is the key "
2158
+ "password right, and is it the certificate's key?").format(e.path, e.detail)
2159
+ if e.kind == "no_cert":
2160
+ return _("Enable TLS requires both a certificate and a key.")
2161
+ if e.kind == "ca_unreadable":
2162
+ return _("The CA file {0} can't be read: {1}").format(e.path, e.detail)
2163
+ if e.kind == "ca_empty":
2164
+ return _("The CA file {0} holds no certificate (PEM or DER).").format(e.path)
2165
+ return str(e)
2166
+
2167
+
2168
+ # ---------------------------------------------------------------------------
2169
+ # MQTT: the topics and identity a mapping uses on its broker, and clashes between
2170
+ # mappings. Checked where a mapping is created or changed (check_mqtt_clashes) and
2171
+ # warned about at start-up and at each start (mqtt_warnings); never by validate(): a
2172
+ # stored config with a clash keeps loading and running.
2173
+ # ---------------------------------------------------------------------------
2174
+ def _plain_mqtt(m: MappingConfig) -> bool:
2175
+ """The mapping publishes plain MQTT topics under its base topic (serial lines, or
2176
+ Modbus points), not Sparkplug B (which a Modbus gateway alone can speak)."""
2177
+ return m.network.protocol != "modbus" or m.mqtt.payload_format != "sparkplug_b"
2178
+
2179
+
2180
+ def _mqtt_topics(m: MappingConfig) -> list[tuple[str, str, Any]]:
2181
+ """The topics a plain-format mapping uses while it runs, as (topic, use, of): use
2182
+ "status" (its retained online/offline and its Last Will) and "lines" (serial lines,
2183
+ with "tx" and "tx_result" for its MQTT writes: mqtt_tx_topics), or on a Modbus gateway
2184
+ "value", "set" and "result" (a point's value, its command topic and the answers to its
2185
+ commands; of = the point's name) and "unit" (a unit's status, with unit_status on;
2186
+ of = the unit). As engine/mqtt_pub, engine/bridge and engine/modbus_gateway build
2187
+ them."""
2188
+ base = str(m.mqtt.base_topic).rstrip("/")
2189
+ out: list[tuple[str, str, Any]] = [(f"{base}/status", "status", None)]
2190
+ if m.network.protocol != "modbus":
2191
+ return out + [(base, "lines", None)] + mqtt_tx_topics(m)
2192
+ units: list[Any] = []
2193
+ for p in m.modbus_poll.points:
2194
+ if p.poll is not False: # (a write-only point is never published)
2195
+ out.append((f"{base}/{p.name}", "value", p.name))
2196
+ if p.unit != 0 and p.unit not in units:
2197
+ units.append(p.unit)
2198
+ if p.writable:
2199
+ out.append((f"{base}/{p.name}/set", "set", p.name))
2200
+ out.append((f"{base}/{p.name}/set/result", "result", p.name))
2201
+ if m.mqtt.unit_status:
2202
+ out += [(f"{base}/unit{u}/status", "unit", u) for u in units]
2203
+ return out
2204
+
2205
+
2206
+ def _topic_use(use: str, of: Any) -> str:
2207
+ if use == "status":
2208
+ return _("the mapping's online/offline status topic")
2209
+ if use == "unit":
2210
+ return _("the status topic of unit {0}").format(of)
2211
+ if use == "set":
2212
+ return _("the command topic of point {0}").format(repr(of))
2213
+ if use == "result":
2214
+ return _("the topic of the answers to point {0}'s commands").format(repr(of))
2215
+ return _("the topic of point {0}").format(repr(of))
2216
+
2217
+
2218
+ def sparkplug_identity(m: MappingConfig) -> tuple[str, str]:
2219
+ """The Sparkplug B group and edge node ID a mapping's node uses (as
2220
+ engine/sparkplug_node: a blank node ID is made from the mapping's name)."""
2221
+ from .engine.sparkplug import sanitize_id
2222
+ mq = m.mqtt
2223
+ return (str(mq.sparkplug_group_id or "").strip() or "ser2net",
2224
+ str(mq.sparkplug_node_id or "").strip() or sanitize_id(str(m.name)))
2225
+
2226
+
2227
+ @dataclass
2228
+ class _MqttUse:
2229
+ """What a mapping that runs does on its broker (see _mqtt_use)."""
2230
+ broker: str # host:port (the host in lower case)
2231
+ client_id: str # "" = the client library makes one up
2232
+ node: str # Sparkplug B: "<group>/<edge node>"; plain: ""
2233
+ commands: set[str] # plain: the topics it takes writes from (<point>/set, /tx)
2234
+ published: set[str] # plain: the topics it publishes to
2235
+
2236
+
2237
+ def _mqtt_use(m: MappingConfig) -> Optional[_MqttUse]:
2238
+ """None when the mapping starts no MQTT client (not a network mapping, MQTT off, a
2239
+ Modbus gateway without points, a plain base topic with a wildcard: see
2240
+ engine/mqtt_pub)."""
2241
+ mq = m.mqtt
2242
+ if m.kind != "net" or not mq.enabled:
2243
+ return None
2244
+ if m.network.protocol == "modbus" and not m.modbus_poll.points:
2245
+ return None
2246
+ broker = f"{str(mq.host).strip().lower()}:{mq.port}"
2247
+ client_id = mq.client_id if isinstance(mq.client_id, str) else ""
2248
+ if not _plain_mqtt(m):
2249
+ return _MqttUse(broker, client_id, "/".join(sparkplug_identity(m)), set(), set())
2250
+ if any(c in str(mq.base_topic) for c in "+#"):
2251
+ return None
2252
+ topics = _mqtt_topics(m)
2253
+ commands = {"set", "tx"}
2254
+ return _MqttUse(broker, client_id, "",
2255
+ {t for t, use, _of in topics if use in commands},
2256
+ {t for t, use, _of in topics if use not in commands})
2257
+
2258
+
2259
+ @dataclass(frozen=True)
2260
+ class MqttClash:
2261
+ """Two enabled mappings that get in each other's way on one MQTT broker: kind
2262
+ "client_id" (the same client ID: the broker keeps one connection per ID),
2263
+ "sparkplug" (the same Sparkplug B edge node), "command" (a topic one of them takes
2264
+ writes from that the other uses too) — refused where a mapping is created or changed
2265
+ — or "shared" (both publish to a topic, such as one base topic's status: only
2266
+ warned about, since it can be meant)."""
2267
+ kind: str
2268
+ other: str # the other mapping's name
2269
+ broker: str
2270
+ what: str # the client ID, "<group>/<edge node>" or the topic
2271
+
2272
+ @property
2273
+ def refused(self) -> bool:
2274
+ return self.kind != "shared"
2275
+
2276
+ def text(self) -> str:
2277
+ if self.kind == "client_id":
2278
+ return _("Mapping '{0}' already connects to the MQTT broker {1} with the client ID "
2279
+ "{2}. A broker keeps one connection per client ID, so the two mappings "
2280
+ "would keep disconnecting each other: leave the client ID blank "
2281
+ "(automatic) or use another one.").format(
2282
+ self.other, self.broker, repr(self.what))
2283
+ if self.kind == "sparkplug":
2284
+ return _("Mapping '{0}' is already the Sparkplug B edge node {1} (group/edge node "
2285
+ "ID) on the MQTT broker {2}. Two edge nodes can't share it: a host takes "
2286
+ "them for one and sends its commands to both. Set another edge node ID "
2287
+ "(a blank one is made from the mapping's name).").format(
2288
+ self.other, repr(self.what), self.broker)
2289
+ if self.kind == "command":
2290
+ return _("Mapping '{0}' also uses the MQTT topic {1} on the broker {2}, and one of "
2291
+ "the two takes writes from it (a point's /set, a serial mapping's /tx): "
2292
+ "one MQTT command would be written by both, or the data of one taken as a "
2293
+ "command by the other. Give one of them another base topic.").format(
2294
+ self.other, repr(self.what), self.broker)
2295
+ # only logged (never refused, so never shown in the UI)
2296
+ return (f"mapping '{self.other}' publishes to the MQTT topic {self.what!r} on the "
2297
+ f"broker {self.broker} too, so the messages of the two mix there (a shared "
2298
+ ".../status shows the state of whichever connected last)")
2299
+
2300
+
2301
+ class _MqttIndex:
2302
+ """The MQTT identity and topics of some mappings, by broker, to find what another
2303
+ mapping would share with them."""
2304
+
2305
+ def __init__(self) -> None:
2306
+ # (broker, client ID / "<group>/<node>" / topic) -> the mappings with it
2307
+ self._ids: dict[tuple[str, str], list[MappingConfig]] = {}
2308
+ self._nodes: dict[tuple[str, str], list[MappingConfig]] = {}
2309
+ self._commands: dict[tuple[str, str], list[MappingConfig]] = {}
2310
+ self._published: dict[tuple[str, str], list[MappingConfig]] = {}
2311
+
2312
+ def add(self, m: MappingConfig, use: _MqttUse) -> None:
2313
+ b = use.broker
2314
+ if use.client_id:
2315
+ self._ids.setdefault((b, use.client_id), []).append(m)
2316
+ if use.node:
2317
+ self._nodes.setdefault((b, use.node), []).append(m)
2318
+ for t in use.commands:
2319
+ self._commands.setdefault((b, t), []).append(m)
2320
+ for t in use.published:
2321
+ self._published.setdefault((b, t), []).append(m)
2322
+
2323
+ def clashes(self, use: _MqttUse) -> list[MqttClash]:
2324
+ """What a mapping with ``use`` shares with these: one clash per other mapping, the
2325
+ most serious one; of the clashes only warned about ("shared"), just the first,
2326
+ when there is nothing worse (mappings that share a topic on purpose would
2327
+ otherwise say so to each other over and over)."""
2328
+ b = use.broker
2329
+ found: dict[str, MqttClash] = {}
2330
+
2331
+ def note(kind: str, others: list[MappingConfig], what: str) -> None:
2332
+ for other in others:
2333
+ if other.id not in found:
2334
+ found[other.id] = MqttClash(kind, str(other.name), b, what)
2335
+
2336
+ if use.client_id:
2337
+ note("client_id", self._ids.get((b, use.client_id), []), use.client_id)
2338
+ if use.node:
2339
+ note("sparkplug", self._nodes.get((b, use.node), []), use.node)
2340
+ for t in sorted(use.commands):
2341
+ note("command", self._commands.get((b, t), []) + self._published.get((b, t), []), t)
2342
+ for t in sorted(use.published):
2343
+ note("command", self._commands.get((b, t), []), t)
2344
+ if not found: # a shared base topic shows as its status topic
2345
+ topic = min((t for t in use.published if (b, t) in self._published),
2346
+ key=lambda t: (not t.endswith("/status"), t), default=None)
2347
+ if topic is not None:
2348
+ note("shared", self._published[(b, topic)][:1], topic)
2349
+ return list(found.values())
2350
+
2351
+
2352
+ def mqtt_clashes(mapping: MappingConfig, mappings) -> list[MqttClash]:
2353
+ """What ``mapping`` would share on its MQTT broker with the other enabled ones of
2354
+ ``mappings`` (itself, by id, left out) while both run; nothing when it is disabled."""
2355
+ if not mapping.enabled or (use := _mqtt_use(mapping)) is None:
2356
+ return []
2357
+ index = _MqttIndex()
2358
+ for m in mappings:
2359
+ if m.id == mapping.id or not m.enabled:
2360
+ continue
2361
+ try:
2362
+ other = _mqtt_use(m)
2363
+ except Exception: # a hand-edited value of the wrong type: nothing to compare
2364
+ continue
2365
+ if other is not None and other.broker == use.broker:
2366
+ index.add(m, other)
2367
+ return index.clashes(use)
2368
+
2369
+
2370
+ def check_mqtt_clashes(mapping: MappingConfig, mappings) -> None:
2371
+ """An edit's rule (the mapping form, also a cluster node's save of its own mapping;
2372
+ the REST API; an import): an enabled mapping can't share its MQTT client ID, its
2373
+ Sparkplug B edge node or a command topic with another enabled mapping on the same
2374
+ broker. Raises ConfigError. Not in validate(): a stored config with such a clash keeps
2375
+ loading and running (with a warning, see mqtt_warnings); only an edit has to fix it."""
2376
+ for clash in mqtt_clashes(mapping, mappings):
2377
+ if clash.refused:
2378
+ raise ConfigError(clash.text())
2379
+
2380
+
2381
+ @contextlib.contextmanager
2382
+ def _in_english():
2383
+ """Messages built meanwhile are in English (log lines, whatever the request's
2384
+ language)."""
2385
+ token = i18n.set_language(i18n.DEFAULT)
2386
+ try:
2387
+ yield
2388
+ finally:
2389
+ i18n.reset(token)
2390
+
2391
+
2392
+ def _topic_warnings(m: MappingConfig) -> list[str]:
2393
+ if m.kind == "net" and m.mqtt.enabled and _plain_mqtt(m):
2394
+ try:
2395
+ m.check_mqtt_topics()
2396
+ except ConfigError as e:
2397
+ return [f"MQTT: {e}"]
2398
+ return []
2399
+
2400
+
2401
+ def mqtt_warnings(mapping: MappingConfig, mappings) -> list[str]:
2402
+ """In English, for the log: what the edits refuse in ``mapping``'s MQTT topics (see
2403
+ check_mqtt_topics; a wildcard base topic is logged by the MQTT client itself), and
2404
+ what it shares on its broker with the other enabled ones of ``mappings``. Empty when
2405
+ it is disabled; never raises."""
2406
+ if not mapping.enabled:
2407
+ return []
2408
+ with _in_english():
2409
+ try:
2410
+ return _topic_warnings(mapping) + [f"MQTT: {c.text()}"
2411
+ for c in mqtt_clashes(mapping, mappings)]
2412
+ except Exception: # a hand-edited value of the wrong type: validate() reports it
2413
+ return []
2414
+
2415
+
2416
+ def mqtt_start_warnings(mappings: list[MappingConfig]) -> list[list[str]]:
2417
+ """mqtt_warnings for each of ``mappings``, in order, against the enabled ones before
2418
+ it, in one pass (a clash is said once, by the later mapping of the two): for the
2419
+ start-up's load notes and starting them all. In English; never raises."""
2420
+ out: list[list[str]] = []
2421
+ index = _MqttIndex()
2422
+ with _in_english():
2423
+ for m in mappings:
2424
+ lines: list[str] = []
2425
+ if m.enabled:
2426
+ try:
2427
+ lines = _topic_warnings(m)
2428
+ if (use := _mqtt_use(m)) is not None:
2429
+ lines += [f"MQTT: {c.text()}" for c in index.clashes(use)]
2430
+ index.add(m, use)
2431
+ except Exception: # a hand-edited value of the wrong type: validate() says
2432
+ lines = []
2433
+ out.append(lines)
2434
+ return out
2435
+
2436
+
2437
+ def _mqtt_load_notes(mappings: list[MappingConfig]) -> list[str]:
2438
+ return [f"mapping '{m.name}': {line}" for m, lines in
2439
+ zip(mappings, mqtt_start_warnings(mappings), strict=True) for line in lines]
2440
+
2441
+
2442
+ # ---------------------------------------------------------------------------
2443
+ # Global application config
2444
+ # ---------------------------------------------------------------------------
2445
+ @dataclass
2446
+ class AdminUI:
2447
+ bind_ip: str = "127.0.0.1"
2448
+ port: int = 8080
2449
+ tls_cert: str = ""
2450
+ tls_key: str = ""
2451
+ # decrypts an encrypted tls_key (engine/tls.py: never asked for on a terminal)
2452
+ tls_key_password: str = ""
2453
+ # Web-UI login can be switched off from Settings (trusted networks only): visitors
2454
+ # without a session then act with anonymous_role. Signing in still works, and the
2455
+ # REST API keeps its bearer token.
2456
+ require_login: bool = field(default=True, metadata=_PROTECTIVE)
2457
+ anonymous_role: str = "viewer"
2458
+ # web UI language when a visitor hasn't picked one: auto (the browser's), en, tr
2459
+ language: str = "auto"
2460
+ # Connections to the UI's listener: at most this many open at once in all, and from
2461
+ # one source address (0 = no limit; config.json only). A connection must send a
2462
+ # complete request head within request_timeout_s of connecting, and of the end of the
2463
+ # previous response (0 = no limit), then its body, which gets that time plus a second
2464
+ # per KiB of it that comes in; a client that takes nothing of a response for
2465
+ # request_timeout_s is dropped too. A WebSocket (the console) is not timed.
2466
+ max_connections: int = 256
2467
+ max_connections_per_ip: int = 64
2468
+ request_timeout_s: int = 10
2469
+ # With admin TLS, responses tell the browser to use only https:// for this host name
2470
+ # - on every port - for this many seconds after each visit (Strict-Transport-Security);
2471
+ # 0 sends max-age=0, which turns that off and makes browsers forget it. Settings ->
2472
+ # Admin TLS.
2473
+ hsts_max_age: int = HSTS_MAX_AGE_DEFAULT
2474
+ # Behind a reverse proxy (Settings -> Reverse proxy; ser2netweb/web/proxy.py). public_url: the
2475
+ # address users open, such as https://ser2netweb.example.com, or with a path the UI is then
2476
+ # also served under (https://gw.example.com/ser2netweb); '' = none. trusted_proxies: the
2477
+ # addresses / networks whose X-Forwarded-For and X-Forwarded-Proto are believed; none by
2478
+ # default. proxy_hsts: HSTS (hsts_max_age) also on HTTPS requests through a proxy.
2479
+ public_url: str = ""
2480
+ trusted_proxies: list[str] = field(default_factory=list)
2481
+ proxy_hsts: bool = field(default=False, metadata=_ONLY_TRUE)
2482
+ # A name for this instance (Settings -> Admin interface; SER2NETWEB_NODE_NAME for one run
2483
+ # instead), for instances that share a host name: notifications, the cluster's fleet
2484
+ # view, the browser tab and the top bar use it. '' = the host name, as before.
2485
+ node_name: str = ""
2486
+ # Which web UI requests all.log gets an "HTTP <ip> <method> <path> -> <status>" line
2487
+ # for (Settings -> Audit log): "all" (every page and change but the static files and
2488
+ # the automatic refreshes, as before), "changes" (only requests that change something,
2489
+ # and refused or failed ones: status 400 and up) or "off". Sign-ins, failed sign-ins
2490
+ # and the audit records are logged whatever it says.
2491
+ request_log: str = "all"
2492
+ # A notice the sign-in and first-run setup pages show before anyone signs in (Settings
2493
+ # -> Web UI access): who runs this system, what is recorded, a privacy notice or a
2494
+ # system-use warning. Plain text (line breaks kept, never HTML), at most
2495
+ # LOGIN_NOTICE_MAX characters; login_notice_url: an http(s) link to the full text.
2496
+ # '' = none, as before.
2497
+ login_notice: str = ""
2498
+ login_notice_url: str = ""
2499
+
2500
+ @property
2501
+ def tls_enabled(self) -> bool:
2502
+ return bool(self.tls_cert and self.tls_key)
2503
+
2504
+ def check_tls_files(self) -> None:
2505
+ """The certificate and key, when set, must be files that exist: checked when they
2506
+ are set in Settings, and at start-up (runtime.require_admin_tls), not by every save
2507
+ — a file that goes missing later must not block every other change."""
2508
+ for path in (p for p in (self.tls_cert, self.tls_key) if p):
2509
+ if not os.path.isfile(data_path(path)):
2510
+ raise ConfigError(_("TLS file not found or not readable: {0}").format(path))
2511
+
2512
+
2513
+ NODE_NAME_MAX = 64
2514
+ NODE_NAME_ENV = "SER2NETWEB_NODE_NAME"
2515
+
2516
+
2517
+ def node_name_problem(value: Any) -> str | None:
2518
+ """Why `value` can't be admin_ui.node_name, or None when it can (also when blank): at
2519
+ most NODE_NAME_MAX characters and no control characters or line breaks (it goes into
2520
+ e-mail subjects, where one would stop every message)."""
2521
+ if not isinstance(value, str):
2522
+ return _("The instance name must be text.")
2523
+ text = value.strip()
2524
+ if len(text) > NODE_NAME_MAX:
2525
+ return _("The instance name can have at most {0} characters.").format(NODE_NAME_MAX)
2526
+ if any(unicodedata.category(c) in ("Cc", "Cf", "Cs", "Zl", "Zp") for c in text):
2527
+ return _("The instance name can’t have control characters or line breaks.")
2528
+ return None
2529
+
2530
+
2531
+ def node_name_from_env(env=None) -> str:
2532
+ """SER2NETWEB_NODE_NAME (or SER2NET_NODE_NAME) when it is set and usable, else ''."""
2533
+ raw = envvars.text(NODE_NAME_ENV, env)
2534
+ return raw if raw and node_name_problem(raw) is None else ""
2535
+
2536
+
2537
+ def node_name(admin: "AdminUI", env=None) -> str:
2538
+ """This instance's name ('' = none: the host name is used): SER2NETWEB_NODE_NAME (or
2539
+ SER2NET_NODE_NAME) when it is set and usable, else admin_ui.node_name."""
2540
+ return node_name_from_env(env) or admin.node_name
2541
+
2542
+
2543
+ def node_label(admin: "AdminUI") -> str:
2544
+ """What names this instance where the host name did up to 2.8: node_name(), else the
2545
+ host name (notifications, the cluster's beacon and fleet view)."""
2546
+ return node_name(admin) or socket.gethostname()
2547
+
2548
+
2549
+ REQUEST_LOG_MODES = ("all", "changes", "off")
2550
+ LOGIN_NOTICE_MAX = 2000
2551
+ LOGIN_NOTICE_URL_MAX = 500
2552
+ # characters a sign-in notice keeps out: control characters but tab and line feed
2553
+ _NOTICE_CONTROL = re.compile(r"[\x00-\x08\x0b-\x1f\x7f-\x9f]")
2554
+
2555
+
2556
+ def clean_login_notice(text: str) -> str:
2557
+ """A sign-in notice as it is kept: line breaks as line feeds, no other control
2558
+ characters, no white space at its ends."""
2559
+ return _NOTICE_CONTROL.sub("", str(text).replace("\r\n", "\n").replace("\r", "\n")).strip()
2560
+
2561
+
2562
+ def login_notice_problem(text: Any) -> str | None:
2563
+ """Why `text` can't be admin_ui.login_notice, or None when it can (also when blank)."""
2564
+ if not isinstance(text, str):
2565
+ return _("The sign-in notice must be text.")
2566
+ if len(clean_login_notice(text)) > LOGIN_NOTICE_MAX:
2567
+ return _("The sign-in notice can have at most {0} characters.").format(LOGIN_NOTICE_MAX)
2568
+ return None
2569
+
2570
+
2571
+ def login_notice_url_problem(url: Any) -> str | None:
2572
+ """Why `url` can't be admin_ui.login_notice_url, or None when it can (also when blank):
2573
+ an http:// or https:// address with a host name, without a user name or password."""
2574
+ bad = _("The notice’s link must be an http:// or https:// address, such as "
2575
+ "https://example.com/privacy.")
2576
+ if not isinstance(url, str):
2577
+ return bad
2578
+ text = url.strip()
2579
+ if not text:
2580
+ return None
2581
+ if len(text) > LOGIN_NOTICE_URL_MAX:
2582
+ return _("The notice’s link can have at most {0} characters.").format(
2583
+ LOGIN_NOTICE_URL_MAX)
2584
+ if any(c.isspace() or ord(c) < 32 or ord(c) == 127 for c in text):
2585
+ return bad
2586
+ try:
2587
+ parts = urllib.parse.urlsplit(text)
2588
+ parts.port # noqa: B018 (raises ValueError for a port that is no number)
2589
+ except ValueError:
2590
+ return bad
2591
+ if parts.scheme.lower() not in ("http", "https") or not parts.hostname or "@" in parts.netloc:
2592
+ return bad
2593
+ return None
2594
+
2595
+
2596
+ def _normalize_login_notice(admin: "AdminUI", notes: list[str]) -> None:
2597
+ """A hand-edited sign-in notice or link that can't be used is left out (a notice too
2598
+ long is cut), with a load note."""
2599
+ text = admin.login_notice
2600
+ if not isinstance(text, str):
2601
+ notes.append(f"admin_ui.login_notice is not text ({type(text).__name__}); no sign-in "
2602
+ "notice is shown")
2603
+ admin.login_notice = ""
2604
+ else:
2605
+ admin.login_notice = clean_login_notice(text)
2606
+ if len(admin.login_notice) > LOGIN_NOTICE_MAX:
2607
+ notes.append(f"admin_ui.login_notice has {len(admin.login_notice)} characters; "
2608
+ f"the first {LOGIN_NOTICE_MAX} are shown")
2609
+ admin.login_notice = admin.login_notice[:LOGIN_NOTICE_MAX].rstrip()
2610
+ url = admin.login_notice_url
2611
+ if login_notice_url_problem(url) is not None:
2612
+ notes.append(f"admin_ui.login_notice_url {str(url)[:80]!r} is not an http(s) address; "
2613
+ "the sign-in notice has no link")
2614
+ admin.login_notice_url = ""
2615
+ else:
2616
+ admin.login_notice_url = url.strip()
2617
+
2618
+
2619
+ def _normalize_request_log(admin: "AdminUI", notes: list[str]) -> None:
2620
+ """A hand-edited admin_ui.request_log other than all / changes / off logs every request,
2621
+ as before the setting existed, with a load note."""
2622
+ if admin.request_log not in REQUEST_LOG_MODES:
2623
+ notes.append(f"admin_ui.request_log {str(admin.request_log)[:40]!r} is not one of "
2624
+ f"{', '.join(REQUEST_LOG_MODES)}; using all")
2625
+ admin.request_log = "all"
2626
+
2627
+
2628
+ def _normalize_node_name(admin: "AdminUI", notes: list[str]) -> None:
2629
+ """A hand-edited admin_ui.node_name that can't be used is left out, with a load note."""
2630
+ value = admin.node_name
2631
+ if value == "":
2632
+ return
2633
+ if node_name_problem(value) is not None:
2634
+ notes.append(f"admin_ui.node_name {str(value)[:80]!r} is not usable "
2635
+ f"({node_name_problem(value)}); the host name is used")
2636
+ admin.node_name = ""
2637
+ else:
2638
+ admin.node_name = value.strip()
2639
+
2640
+
2641
+ # The first path segments of the UI's own pages: a public URL's path can't start with one,
2642
+ # or a page asked for without the path would be taken for one under it (ser2netweb/web/proxy.py).
2643
+ RESERVED_PATHS = frozenset({"api", "auth", "favicon.ico", "healthz", "lang", "licenses",
2644
+ "login", "logout", "metrics", "settings", "setup", "static"})
2645
+ _URL_PATH_SEGMENT = re.compile(r"[A-Za-z0-9._~-]+")
2646
+ _HOST_NAME = re.compile(r"(?!-)[a-z0-9_-]{1,63}(?<!-)(\.(?!-)[a-z0-9_-]{1,63}(?<!-))*\.?")
2647
+
2648
+
2649
+ def _public_url_parts(value: str):
2650
+ """(scheme, host as written in a URL, port or None, path without the trailing '/') of
2651
+ a public URL, or the problem (a str) with it."""
2652
+ text = value.strip()
2653
+ bad = _("The public URL must be an http:// or https:// address with a host name, such "
2654
+ "as https://ser2netweb.example.com.")
2655
+ if any(c.isspace() or ord(c) < 32 or c == "\\" for c in text) or not text.isascii():
2656
+ return bad
2657
+ if "?" in text or "#" in text:
2658
+ return _("The public URL can’t have a query (?…) or a fragment (#…).")
2659
+ try:
2660
+ parts = urllib.parse.urlsplit(text)
2661
+ port = parts.port
2662
+ except ValueError:
2663
+ return _("The public URL’s port must be a number from 1 to 65535.")
2664
+ scheme, host = parts.scheme.lower(), parts.hostname or ""
2665
+ if scheme not in ("http", "https") or not host or "@" in parts.netloc:
2666
+ return bad
2667
+ if port == 0:
2668
+ return _("The public URL’s port must be a number from 1 to 65535.")
2669
+ try:
2670
+ ip = ipaddress.ip_address(host)
2671
+ except ValueError:
2672
+ if not _HOST_NAME.fullmatch(host):
2673
+ return bad
2674
+ written = host
2675
+ else:
2676
+ written = f"[{ip.compressed}]" if ip.version == 6 else ip.compressed
2677
+ path = parts.path.rstrip("/")
2678
+ segments = path.split("/")[1:] if path else []
2679
+ if any(not _URL_PATH_SEGMENT.fullmatch(s) or s in (".", "..") for s in segments):
2680
+ return _("The public URL’s path may only have letters, digits and . _ ~ - between "
2681
+ "its slashes, such as /ser2netweb.")
2682
+ if segments and segments[0].lower() in RESERVED_PATHS:
2683
+ return _("The public URL’s path can’t start with /{0}: that is a page of this UI.") \
2684
+ .format(segments[0])
2685
+ return scheme, written, port, path
2686
+
2687
+
2688
+ def public_url_problem(value: Any) -> str | None:
2689
+ """Why `value` can't be admin_ui.public_url, or None when it can (also when blank): an
2690
+ http:// or https:// address with a host name, optionally a port and a path of plain
2691
+ segments that doesn't start like one of the UI's own pages (RESERVED_PATHS); no user
2692
+ name, query or fragment."""
2693
+ if not isinstance(value, str):
2694
+ return _("The public URL must be an http:// or https:// address with a host name, "
2695
+ "such as https://ser2netweb.example.com.")
2696
+ if not value.strip():
2697
+ return None
2698
+ parts = _public_url_parts(value)
2699
+ return parts if isinstance(parts, str) else None
2700
+
2701
+
2702
+ def normalize_public_url(value: str) -> str:
2703
+ """A usable public URL as it is kept: scheme and host in lower case, the scheme's own
2704
+ port and a trailing '/' left out ('' for a blank or unusable one)."""
2705
+ parts = _public_url_parts(value) if isinstance(value, str) and value.strip() else ""
2706
+ if isinstance(parts, str):
2707
+ return ""
2708
+ scheme, host, port, path = parts
2709
+ shown = "" if port is None or port == {"http": 80, "https": 443}[scheme] else f":{port}"
2710
+ return f"{scheme}://{host}{shown}{path}"
2711
+
2712
+
2713
+ def proxy_entry_problem(entry: Any) -> str | None:
2714
+ """Why `entry` can't be one of admin_ui.trusted_proxies (an IP address or a network
2715
+ such as 172.18.0.0/16), or None when it can."""
2716
+ text = entry.strip() if isinstance(entry, str) else ""
2717
+ try:
2718
+ if not text or not text.isascii():
2719
+ raise ValueError
2720
+ ipaddress.ip_network(text, strict=False)
2721
+ except ValueError:
2722
+ return _("“{0}” is not an IP address or network (such as 172.18.0.0/16).").format(
2723
+ str(entry)[:100])
2724
+ return None
2725
+
2726
+
2727
+ def split_proxy_entries(text: str) -> list[str]:
2728
+ """The entries of a list of trusted proxies written on one line (spaces or commas)."""
2729
+ return [e for e in re.split(r"[\s,]+", text or "") if e]
2730
+
2731
+
2732
+ def _normalize_proxy_settings(admin: "AdminUI", notes: list[str]) -> None:
2733
+ """A hand-edited public_url, trusted_proxies or proxy_hsts that can't be used is left
2734
+ out, with a load note, rather than refused: config.json must not keep the UI from
2735
+ starting."""
2736
+ value = admin.trusted_proxies
2737
+ entries = split_proxy_entries(value) if isinstance(value, str) else value
2738
+ if not isinstance(entries, list):
2739
+ notes.append(f"admin_ui.trusted_proxies {value!r} is not a list of IP addresses or "
2740
+ "networks; no proxy is trusted")
2741
+ entries = []
2742
+ kept = [e.strip() for e in entries if proxy_entry_problem(e) is None]
2743
+ if len(kept) != len(entries):
2744
+ dropped = [e for e in entries if proxy_entry_problem(e) is not None]
2745
+ notes.append(f"admin_ui.trusted_proxies: {dropped!r} left out (not IP addresses or "
2746
+ "networks)")
2747
+ admin.trusted_proxies = kept
2748
+ url = admin.public_url
2749
+ if public_url_problem(url) is not None:
2750
+ notes.append(f"admin_ui.public_url {url!r} is not usable ({public_url_problem(url)}); "
2751
+ "none is used")
2752
+ admin.public_url = ""
2753
+ else:
2754
+ admin.public_url = normalize_public_url(url)
2755
+ if not isinstance(admin.proxy_hsts, bool):
2756
+ notes.append(f"admin_ui.proxy_hsts {admin.proxy_hsts!r} is not true or false; using false")
2757
+ admin.proxy_hsts = False
2758
+
2759
+
2760
+ # admin_ui connection limits: (default, lowest value other than 0, highest)
2761
+ _ADMIN_LIMITS = {"max_connections": (256, 16, 65535),
2762
+ "max_connections_per_ip": (64, 8, 65535),
2763
+ "request_timeout_s": (10, 1, 3600)}
2764
+
2765
+
2766
+ def _normalize_admin_limits(admin: "AdminUI", notes: list[str]) -> None:
2767
+ """Put an admin_ui connection limit that is not a usable whole number back to its
2768
+ default (0, no limit, stays 0; a value below the lowest goes up to it), and so an
2769
+ unusable hsts_max_age, with a load note: a hand-edited config.json must not keep the
2770
+ UI from starting, nor make it unusable."""
2771
+ for name, (default, low, high) in _ADMIN_LIMITS.items():
2772
+ value = getattr(admin, name)
2773
+ if isinstance(value, bool) or not isinstance(value, int) or not 0 <= value <= high:
2774
+ fixed = default
2775
+ elif 0 < value < low:
2776
+ fixed = low
2777
+ else:
2778
+ continue
2779
+ setattr(admin, name, fixed)
2780
+ notes.append(f"admin_ui.{name} {value!r} is not usable (0 = no limit, else "
2781
+ f"{low}-{high}); using {fixed}")
2782
+ value = admin.hsts_max_age
2783
+ if isinstance(value, bool) or not isinstance(value, int) or not 0 <= value <= HSTS_MAX_AGE_MAX:
2784
+ admin.hsts_max_age = HSTS_MAX_AGE_DEFAULT
2785
+ notes.append(f"admin_ui.hsts_max_age {value!r} is not a whole number of seconds from 0 "
2786
+ f"to {HSTS_MAX_AGE_MAX}; using {HSTS_MAX_AGE_DEFAULT} (one year)")
2787
+
2788
+
2789
+ # the languages the UI offers are listed once, in ser2netweb/i18n.py ("auto": the browser's)
2790
+ UI_LANGUAGES = ("auto", *LANGUAGES)
2791
+ ROLES = ("admin", "operator", "viewer")
2792
+ # higher rank = more privilege; a route requires >= some rank
2793
+ ROLE_RANK = {"viewer": 1, "operator": 2, "admin": 3}
2794
+
2795
+
2796
+ @dataclass
2797
+ class LdapSettings:
2798
+ """LDAP / Active Directory authentication. Users authenticate by binding to the
2799
+ directory; their group membership maps to a ser2netweb role. Two bind modes:
2800
+ - direct: user_dn_template (e.g. 'uid={username},ou=people,dc=ex,dc=com' or the
2801
+ AD UPN '{username}@corp.local'); the credentials bind as that DN. An AD sign-in
2802
+ name (user@domain, DOMAIN\\user) isn't a DN: the account is then found by
2803
+ userPrincipalName / sAMAccountName in its domain, under user_search_base if set.
2804
+ - search+bind: bind as the service account (bind_dn/bind_password), find the user
2805
+ with user_search_filter under user_search_base, then re-bind as that DN.
2806
+ Group->role: a user in admin_group gets admin, operator_group -> operator, etc.;
2807
+ default_role applies when authenticated but in no mapped group (empty => deny). A
2808
+ group setting is a group's whole name (its CN) or DN, compared without regard to
2809
+ letter case (ldap_auth.group_matches). nested_groups (Active Directory) also
2810
+ counts the groups a user is in through other groups.
2811
+ TLS (ldaps:// or StartTLS): the server's certificate must chain to a CA the system
2812
+ trusts — or, with ca_file set, to one in that file only (PEM or DER) — and name the
2813
+ host in server_uri (ldap_auth.tls_context). tls_verify=False skips the check
2814
+ (insecure)."""
2815
+ enabled: bool = False
2816
+ server_uri: str = "" # ldap://host:389 or ldaps://host:636
2817
+ start_tls: bool = field(default=False, metadata=_PROTECTIVE)
2818
+ # check the server's certificate (LDAPS / StartTLS)
2819
+ tls_verify: bool = field(default=True, metadata=_PROTECTIVE)
2820
+ ca_file: str = "" # CA certificate(s) to trust instead of the system's
2821
+ user_dn_template: str = "" # direct-bind template (mutually exclusive with search)
2822
+ bind_dn: str = "" # service account for search+bind
2823
+ bind_password: str = ""
2824
+ user_search_base: str = ""
2825
+ user_search_filter: str = "(uid={username})" # AD: (sAMAccountName={username})
2826
+ group_attr: str = "memberOf" # attribute on the user entry listing group DNs
2827
+ admin_group: str = ""
2828
+ operator_group: str = ""
2829
+ viewer_group: str = ""
2830
+ default_role: str = "" # role when no group matched; "" => deny login
2831
+ nested_groups: bool = False # AD: LDAP_MATCHING_RULE_IN_CHAIN group search
2832
+
2833
+ @staticmethod
2834
+ def from_dict(d: dict[str, Any], notes: Optional[list[FieldNote]] = None,
2835
+ path: str = "") -> "LdapSettings":
2836
+ d = dict(d or {})
2837
+ notes = [] if notes is None else notes
2838
+ return LdapSettings(**_read_fields(LdapSettings, d, notes, path))
2839
+
2840
+ @property
2841
+ def uses_tls(self) -> bool:
2842
+ """ldaps://, or StartTLS: the bind passwords travel encrypted."""
2843
+ return self.server_uri.strip().lower().startswith("ldaps://") or bool(self.start_tls)
2844
+
2845
+ @property
2846
+ def sends_plaintext(self) -> bool:
2847
+ """Passwords over the network unencrypted: neither ldaps:// nor StartTLS, and no
2848
+ ldapi:// (a local socket)."""
2849
+ return not (self.uses_tls or self.server_uri.strip().lower().startswith("ldapi://"))
2850
+
2851
+ def validate(self) -> None:
2852
+ if not self.enabled:
2853
+ return
2854
+ if not self.server_uri.strip():
2855
+ raise ConfigError(_("LDAP is enabled but no server URI is set."))
2856
+ if not (self.user_dn_template.strip() or
2857
+ (self.bind_dn.strip() and self.user_search_base.strip())):
2858
+ raise ConfigError(_("LDAP needs either a user DN template (direct bind) or a bind "
2859
+ "DN + user search base (search+bind)."))
2860
+ if self.default_role and self.default_role not in ROLES:
2861
+ raise ConfigError(_("LDAP default role must be one of {0} or blank.").format(_choices(ROLES)))
2862
+
2863
+
2864
+ @dataclass
2865
+ class OidcSettings:
2866
+ """OpenID Connect single sign-on (authorization-code flow). Endpoints are
2867
+ discovered from the issuer's /.well-known/openid-configuration. A claim
2868
+ (groups_claim) maps to a ser2netweb role, like the LDAP group mapping: a group setting
2869
+ is the whole value, or its last part (/team/name, DOMAIN\\name)."""
2870
+ enabled: bool = False
2871
+ issuer: str = "" # e.g. https://accounts.google.com or a Keycloak realm URL
2872
+ client_id: str = ""
2873
+ client_secret: str = ""
2874
+ redirect_uri: str = "" # blank => derived from the request
2875
+ scopes: str = "openid email profile"
2876
+ username_claim: str = "preferred_username"
2877
+ groups_claim: str = "groups"
2878
+ admin_group: str = ""
2879
+ operator_group: str = ""
2880
+ viewer_group: str = ""
2881
+ default_role: str = "" # role when no group matched; "" => deny
2882
+
2883
+ @staticmethod
2884
+ def from_dict(d: dict[str, Any], notes: Optional[list[FieldNote]] = None,
2885
+ path: str = "") -> "OidcSettings":
2886
+ d = dict(d or {})
2887
+ notes = [] if notes is None else notes
2888
+ return OidcSettings(**_read_fields(OidcSettings, d, notes, path))
2889
+
2890
+ def validate(self) -> None:
2891
+ if not self.enabled:
2892
+ return
2893
+ if not (self.issuer.strip() and self.client_id.strip() and self.client_secret.strip()):
2894
+ raise ConfigError(_("OIDC needs an issuer, client ID and client secret."))
2895
+ if not self.issuer.strip().lower().startswith("https://"):
2896
+ raise ConfigError(_("OIDC issuer must be an https:// URL."))
2897
+ if self.default_role and self.default_role not in ROLES:
2898
+ raise ConfigError(_("OIDC default role must be one of {0} or blank.").format(_choices(ROLES)))
2899
+
2900
+
2901
+ _PEER_ZONE = re.compile(r"[A-Za-z0-9_.-]+")
2902
+
2903
+
2904
+ def peer_host(host: str) -> str | None:
2905
+ """A cluster peer's host in canonical form: an IP address (IPv6 without brackets,
2906
+ with an optional %zone) or a host name. None when it is neither — a value with
2907
+ anything in it that could change the URL built from it."""
2908
+ h = str(host or "").strip()
2909
+ if h.startswith("[") and h.endswith("]"):
2910
+ h = h[1:-1]
2911
+ addr, sep, zone = h.partition("%")
2912
+ try:
2913
+ ip = ipaddress.ip_address(addr)
2914
+ except ValueError:
2915
+ ip = None
2916
+ if ip is not None:
2917
+ if sep and not (ip.version == 6 and _PEER_ZONE.fullmatch(zone)):
2918
+ return None
2919
+ return f"{ip}%{zone}" if sep else str(ip)
2920
+ labels = (h[:-1] if h.endswith(".") else h).split(".")
2921
+ if len(h) <= 253 and all(
2922
+ 0 < len(lb) <= 63 and lb[0].isalnum() and lb[-1].isalnum()
2923
+ and all(c.isalnum() or c in "-_" for c in lb) for lb in labels):
2924
+ return h
2925
+ return None
2926
+
2927
+
2928
+ def parse_cluster_peer(entry: str, strict: bool = False) -> tuple[str, str, int]:
2929
+ """Parse a manual cluster-peer entry into (scheme, host, port). Accepts
2930
+ 'host:port' (defaults to http) or 'http(s)://host:port'. Raises ConfigError.
2931
+ strict (when an entry is edited) also requires the host to be an IP address or a
2932
+ host name; validate() stays lenient, so an entry an older release accepted still
2933
+ loads (one that is no address is just never contacted)."""
2934
+ s = (entry or "").strip()
2935
+ scheme = "http"
2936
+ if "://" in s:
2937
+ scheme, s = s.split("://", 1)
2938
+ scheme = scheme.lower()
2939
+ if scheme not in ("http", "https"):
2940
+ raise ConfigError(_("Cluster peer scheme must be http or https: {0}").format(entry))
2941
+ host, sep, port_s = s.rpartition(":")
2942
+ if not sep or not host or (strict and peer_host(host) is None):
2943
+ raise ConfigError(_("Cluster peer must be host:port — got: {0}").format(entry))
2944
+ try:
2945
+ port = int(port_s)
2946
+ except ValueError:
2947
+ raise ConfigError(_("Cluster peer port is not a number: {0}").format(entry)) from None
2948
+ if not (1 <= port <= 65535):
2949
+ raise ConfigError(_("Cluster peer port must be 1..65535: {0}").format(entry))
2950
+ return scheme, host, port
2951
+
2952
+
2953
+ @dataclass
2954
+ class ClusterSettings:
2955
+ """LAN cluster: instances discover each other via signed UDP broadcast beacons
2956
+ and one node aggregates every node's mappings into a single read-only view.
2957
+ Opt-in: disabled until `enabled` is set AND a shared `key` is configured. The
2958
+ key both signs beacons (so only same-key nodes trust each other) and guards the
2959
+ peer-facing status endpoint. `advertise_ip` overrides the auto-detected LAN IP
2960
+ that peers use to reach this node's web UI (blank => auto). `peers` is a list of
2961
+ manual 'host:port' entries for routed/L3 networks where UDP broadcast doesn't
2962
+ reach — they're aggregated alongside auto-discovered peers.
2963
+
2964
+ Peer requests are signed with (a key derived from) the key; `legacy_auth` also
2965
+ accepts the raw key from, and sends it to, nodes running 2.8.0 or older, which know
2966
+ nothing else. During a planned key change `previous_key` (the key replaced last) is still
2967
+ accepted until `previous_until`, so the nodes not changed yet keep working."""
2968
+ enabled: bool = False
2969
+ key: str = "" # shared secret (PSK); empty => cluster off
2970
+ discovery_port: int = 41750 # UDP port for broadcast beacons
2971
+ advertise_ip: str = "" # override advertised web-UI IP (blank => auto-detect)
2972
+ peers: list[str] = field(default_factory=list) # manual 'host:port' peer entries
2973
+ previous_key: str = "" # still accepted during a key change (blank => none)
2974
+ previous_until: float = 0.0 # ... until this time (epoch seconds; 0 => no end)
2975
+ legacy_auth: bool = True # raw key with nodes up to 2.8.0 (off once all upgraded)
2976
+ discovery: bool = True # UDP beacons; off => manual peers only
2977
+
2978
+ @staticmethod
2979
+ def from_dict(d: dict[str, Any], notes: Optional[list[FieldNote]] = None,
2980
+ path: str = "") -> "ClusterSettings":
2981
+ d = dict(d or {})
2982
+ notes = [] if notes is None else notes
2983
+ # the fields added after 2.8.0 take their default for a hand-edited value of another
2984
+ # type, as in 2.8.1: no text turns off the link to older nodes or discovery, and the
2985
+ # previous key is never a number's text
2986
+ for name, default in (("legacy_auth", True), ("discovery", True), ("previous_key", "")):
2987
+ value = d.get(name)
2988
+ if name in d and not isinstance(value, type(default)):
2989
+ if value is not None:
2990
+ notes.append(FieldNote(path + name, "invalid", value, default,
2991
+ _kind_of(type(default))))
2992
+ del d[name]
2993
+ # a hand-edited key of null or a number reads as "" (cluster off, which start-up
2994
+ # reports) or the number as text, instead of failing wherever the key is used
2995
+ c = ClusterSettings(**_read_fields(ClusterSettings, d, notes, path))
2996
+ c.peers = [p.strip() for p in c.peers if p.strip()]
2997
+ c.previous_until = _epoch(c.previous_until)
2998
+ return c
2999
+
3000
+ @property
3001
+ def active(self) -> bool:
3002
+ return bool(self.enabled and self.key.strip())
3003
+
3004
+ def validate(self) -> None:
3005
+ if not self.enabled:
3006
+ return
3007
+ if not self.key.strip():
3008
+ raise ConfigError(_("Cluster is enabled but no shared key is set."))
3009
+ try:
3010
+ port = int(self.discovery_port)
3011
+ except (TypeError, ValueError):
3012
+ raise ConfigError(_("Cluster discovery port must be a number.")) from None
3013
+ if not (1 <= port <= 65535):
3014
+ raise ConfigError(_("Cluster discovery port must be 1..65535."))
3015
+ if self.advertise_ip.strip():
3016
+ try:
3017
+ ipaddress.ip_address(self.advertise_ip.strip())
3018
+ except ValueError:
3019
+ raise ConfigError(
3020
+ _("Cluster advertise IP is not a valid address: {0}").format(self.advertise_ip)) from None
3021
+ for entry in self.peers:
3022
+ parse_cluster_peer(entry) # raises ConfigError on a bad entry
3023
+
3024
+
3025
+ NOTIFY_EVENTS = ("device_lost", "device_back", "start_failed", "link_lost", "link_back",
3026
+ "client_connected", "client_disconnected", "login_failed", "disk_low",
3027
+ "no_data", "data_back", "peer_offline", "peer_back", "service_started",
3028
+ "unit_offline", "unit_back")
3029
+ # (a saved events list is kept as it is: these are what a config without one gets)
3030
+ NOTIFY_DEFAULT_EVENTS = ("device_lost", "device_back", "start_failed", "link_lost", "link_back")
3031
+ WEBHOOK_FORMATS = ("json", "slack", "teams", "discord", "telegram")
3032
+ SMTP_SECURITY = ("starttls", "ssl", "none")
3033
+
3034
+
3035
+ @dataclass
3036
+ class NotificationSettings:
3037
+ """Event notifications (ser2netweb/notify.py): a webhook and/or e-mail when a serial
3038
+ device disappears or comes back, a mapping fails to start, clients connect or
3039
+ disconnect, a sign-in fails, the data disk is almost full, a device sends nothing
3040
+ (a mapping's no-data alarm), a cluster node goes offline, a polled Modbus unit stops or
3041
+ starts answering again, or ser2netweb has started. Off by default; the same event for the
3042
+ same mapping is sent at most once per repeat_s."""
3043
+ enabled: bool = False
3044
+ events: list[str] = field(default_factory=lambda: list(NOTIFY_DEFAULT_EVENTS))
3045
+ repeat_s: int = 300
3046
+ webhook_url: str = "" # may carry a token (Slack/Teams/Discord/Telegram)
3047
+ webhook_format: str = "json" # json | slack | teams | discord | telegram
3048
+ telegram_chat_id: str = ""
3049
+ email_to: str = "" # comma-separated recipients
3050
+ email_from: str = ""
3051
+ smtp_host: str = ""
3052
+ smtp_port: int = 587
3053
+ smtp_security: str = "starttls" # starttls | ssl | none
3054
+ smtp_username: str = ""
3055
+ smtp_password: str = ""
3056
+ # Personal data in what is sent (client_connected / client_disconnected: the client's
3057
+ # IP address; login_failed: the account name and the IP address): with
3058
+ # mask_personal_data, an address only as its network (IPv4 /24, IPv6 /48) and no
3059
+ # account name. email_separately: each recipient gets a message of their own, so the
3060
+ # recipients don't see each other's addresses. Both off by default (as up to 2.8).
3061
+ # mask_personal_data protects: text in a hand-edited one keeps it on (_PROTECTIVE)
3062
+ mask_personal_data: bool = field(default=False, metadata=_PROTECTIVE)
3063
+ email_separately: bool = field(default=False, metadata=_ONLY_TRUE)
3064
+
3065
+ @staticmethod
3066
+ def from_dict(d: dict[str, Any], notes: Optional[list[FieldNote]] = None,
3067
+ path: str = "") -> "NotificationSettings":
3068
+ d = dict(d or {})
3069
+ notes = [] if notes is None else notes
3070
+ n = NotificationSettings(**_read_fields(NotificationSettings, d, notes, path))
3071
+ n.events = [e for e in (n.events or []) if e in NOTIFY_EVENTS]
3072
+ # a hand-edited value other than true is off, as before the setting existed
3073
+ n.email_separately = n.email_separately is True
3074
+ return n
3075
+
3076
+ @property
3077
+ def recipients(self) -> list[str]:
3078
+ return [a.strip() for a in self.email_to.replace(";", ",").split(",") if a.strip()]
3079
+
3080
+ def validate(self) -> None:
3081
+ if not self.enabled:
3082
+ return
3083
+ if not (self.webhook_url.strip() or self.recipients):
3084
+ raise ConfigError(_("Notifications need a webhook URL or e-mail recipients."))
3085
+ if self.webhook_url.strip():
3086
+ if not self.webhook_url.strip().lower().startswith(("https://", "http://")):
3087
+ raise ConfigError(_("The webhook URL must start with https:// (or http://)."))
3088
+ if self.webhook_format not in WEBHOOK_FORMATS:
3089
+ raise ConfigError(_("Webhook format must be one of {0}.").format(_choices(WEBHOOK_FORMATS)))
3090
+ if self.webhook_format == "telegram" and not str(self.telegram_chat_id).strip():
3091
+ raise ConfigError(_("Telegram needs a chat ID."))
3092
+ if self.recipients:
3093
+ if not self.smtp_host.strip():
3094
+ raise ConfigError(_("E-mail notifications need an SMTP server."))
3095
+ if not _is_int(self.smtp_port) or not (1 <= self.smtp_port <= 65535):
3096
+ raise ConfigError(_("SMTP port must be between 1 and 65535."))
3097
+ if self.smtp_security not in SMTP_SECURITY:
3098
+ raise ConfigError(_("SMTP security must be one of {0}.").format(_choices(SMTP_SECURITY)))
3099
+ if any("@" not in a for a in self.recipients):
3100
+ raise ConfigError(_("Each e-mail recipient needs an @."))
3101
+ if not _is_int(self.repeat_s) or not (0 <= self.repeat_s <= 86400):
3102
+ raise ConfigError(_("The repeat interval must be between 0 and 86400 seconds."))
3103
+
3104
+
3105
+ class AccountNameError(ValueError):
3106
+ """An LDAP/OIDC sign-in can't have an account under its name (upsert_external_user).
3107
+ The text says why, for the log."""
3108
+
3109
+
3110
+ EXTERNAL_NAME_MAX = 255 # an LDAP/OIDC account name (UPNs and e-mail addresses fit)
3111
+
3112
+
3113
+ def external_name(name) -> str:
3114
+ """The one spelling an LDAP/OIDC sign-in name is kept in: NFC (the same name sent
3115
+ decomposed is not another account), without white space around it."""
3116
+ return unicodedata.normalize("NFC", str(name)).strip()
3117
+
3118
+
3119
+ def external_name_problem(name: str) -> Optional[str]:
3120
+ """Why `name` (as external_name() spells it) can't name an LDAP/OIDC account, for the
3121
+ log; None if it can. Any printable text works — DOMAIN\\user, user@domain, names with
3122
+ / # ? % — because the Users panel sends names in form fields, not in URLs. But every
3123
+ character must be visible: a control, format (such as a right-to-left override or a
3124
+ zero-width space) or line-separator character would hide the real name in the Users
3125
+ panel, the logs and the audit log. A name in parentheses is how the audit log marks
3126
+ who acted without an account ("(no login)", "(command line)")."""
3127
+ if not name:
3128
+ return "it is empty"
3129
+ if len(name) > EXTERNAL_NAME_MAX:
3130
+ return f"it is longer than {EXTERNAL_NAME_MAX} characters"
3131
+ hidden = sorted({c for c in name if unicodedata.category(c)[0] == "C"
3132
+ or unicodedata.category(c) in ("Zl", "Zp")})
3133
+ if hidden:
3134
+ return ("it contains control or invisible characters ("
3135
+ + ", ".join(f"U+{ord(c):04X}" for c in hidden) + ")")
3136
+ if name.startswith("(") and name.endswith(")"):
3137
+ return ("a name in parentheses is how the audit log marks who acted without an "
3138
+ "account, such as '(no login)'")
3139
+ return None
3140
+
3141
+
3142
+ @dataclass
3143
+ class User:
3144
+ """A web-UI account. `role` gates what the user may do; `pwd_version` is bumped
3145
+ on that user's password/role change to revoke only their existing sessions (a new
3146
+ account starts at a random value, see new_pwd_version).
3147
+ `source` is 'local' (password in password_hash) or 'ldap' (a shadow account whose
3148
+ role is derived from LDAP groups on each login; no local password).
3149
+ `totp_secret` (base32, ser2netweb/web/totp.py): a local account that set up two-factor
3150
+ sign-in also gives a code from its authenticator app; "" = off (the default). Written
3151
+ to config.json only when set."""
3152
+ username: str = ""
3153
+ password_hash: str = ""
3154
+ role: str = "admin"
3155
+ pwd_version: int = 0
3156
+ source: str = "local"
3157
+ totp_secret: str = ""
3158
+
3159
+ @property
3160
+ def totp_on(self) -> bool:
3161
+ """Does signing in to this account take a code too? (Local accounts only: LDAP
3162
+ and OIDC sign-ins get a second factor from the directory or identity provider.)"""
3163
+ return self.source == "local" and bool(self.totp_secret)
3164
+
3165
+ def to_dict(self) -> dict[str, Any]:
3166
+ d = asdict(self)
3167
+ if not d["totp_secret"]: # a config.json without two-factor sign-in stays as it was
3168
+ del d["totp_secret"]
3169
+ return d
3170
+
3171
+ @property
3172
+ def kind(self) -> str:
3173
+ """How the account signs in, for messages: 'local', 'LDAP' or 'OIDC'."""
3174
+ return self.source if self.source == "local" else self.source.upper()
3175
+
3176
+ @staticmethod
3177
+ def from_dict(d: dict[str, Any], notes: Optional[list[FieldNote]] = None,
3178
+ path: str = "") -> "User":
3179
+ d = dict(d or {})
3180
+ notes = [] if notes is None else notes
3181
+ # a hand-edited totp_secret that is no text stays on (and matches no code), as text
3182
+ u = User(**_read_fields(User, d, notes, path,
3183
+ fallback={"totp_secret": str(d.get("totp_secret") or "")}))
3184
+ if u.role not in ROLES:
3185
+ u.role = "viewer"
3186
+ if u.source not in ("local", "ldap", "oidc"):
3187
+ u.source = "local"
3188
+ if not isinstance(u.totp_secret, str): # hand-edited: null is off, anything else
3189
+ u.totp_secret = str(u.totp_secret or "") # stays on (and matches no code)
3190
+ # as an app may show it: in groups, lower case, padded
3191
+ u.totp_secret = re.sub(r"[\s=]", "", u.totp_secret).upper()
3192
+ return u
3193
+
3194
+
3195
+ # REST API tokens besides the one of AppConfig.api_token_hash (Settings → REST API token)
3196
+ API_TOKENS_MAX = 50
3197
+ API_TOKEN_DAYS_MAX = 3650
3198
+ API_TOKEN_NAME = re.compile(r"[A-Za-z0-9._-]{1,64}")
3199
+ API_TOKEN_ROLES = ("viewer", "operator") # what Settings offers ("admin" acts as operator)
3200
+ _API_TOKEN_ID = re.compile(r"[0-9a-f]{16}")
3201
+ _SHA256_HEX = re.compile(r"[0-9a-f]{64}")
3202
+
3203
+
3204
+ @dataclass
3205
+ class ApiToken:
3206
+ """A named REST API bearer token, next to the one of AppConfig.api_token_hash: each
3207
+ client can have its own, with its own role and an optional expiry, and is named in the
3208
+ audit log (api-token:<name>). Only the token's SHA-256 is stored. When it was last used
3209
+ is kept in memory only (AppState.api_token_used): every save writes a backup.
3210
+ The token of api_token_hash is handled as an ApiToken without an id (auth.find_api_token)
3211
+ and keeps its audit name 'api-token'."""
3212
+ id: str = ""
3213
+ name: str = ""
3214
+ sha256: str = ""
3215
+ role: str = "viewer"
3216
+ created: float = 0.0
3217
+ expires: float = 0.0 # epoch seconds; 0 = never
3218
+
3219
+ @property
3220
+ def audit_label(self) -> str:
3221
+ return f"api-token:{self.name}" if self.id else "api-token"
3222
+
3223
+ def expired(self, now: Optional[float] = None) -> bool:
3224
+ return bool(self.expires) and (time.time() if now is None else now) >= self.expires
3225
+
3226
+
3227
+ def new_api_token_id() -> str:
3228
+ return secrets.token_hex(8)
3229
+
3230
+
3231
+ def _api_tokens(raw: Any) -> tuple[list[ApiToken], list[str]]:
3232
+ """api_tokens from a config file, and notes on what had to be changed. An entry without
3233
+ a valid SHA-256 can't match any token and is left out; a role that is not one makes the
3234
+ token read-only (viewer), never more; a missing or doubled id or an unusable name gets
3235
+ a new one (the token itself keeps working)."""
3236
+ if raw is None:
3237
+ return [], []
3238
+ if not isinstance(raw, list):
3239
+ return [], ["REST API: api_tokens is not a list, so it was ignored: its tokens no "
3240
+ "longer work"]
3241
+ tokens: list[ApiToken] = []
3242
+ notes: list[str] = []
3243
+ for i, item in enumerate(raw, 1):
3244
+ digest = item.get("sha256") if isinstance(item, dict) else None
3245
+ if not isinstance(digest, str) or not _SHA256_HEX.fullmatch(digest.strip().lower()):
3246
+ notes.append(f"REST API: api_tokens entry #{i} has no valid sha256, so it was "
3247
+ "ignored: that token no longer works")
3248
+ continue
3249
+ tid = item.get("id")
3250
+ if not (isinstance(tid, str) and _API_TOKEN_ID.fullmatch(tid)) or any(
3251
+ t.id == tid for t in tokens):
3252
+ tid = new_api_token_id()
3253
+ name = item.get("name")
3254
+ name = name.strip() if isinstance(name, str) else ""
3255
+ if not API_TOKEN_NAME.fullmatch(name) or any(t.name.casefold() == name.casefold()
3256
+ for t in tokens):
3257
+ new_name = f"token-{tid[:6]}"
3258
+ notes.append(f"REST API: api_tokens entry #{i} has the name {name!r}, which is not "
3259
+ f"1-64 letters, digits, '.', '_' or '-' or is taken: it is now named "
3260
+ f"{new_name!r}")
3261
+ name = new_name
3262
+ role_raw = item.get("role")
3263
+ role = role_raw.strip().lower() if isinstance(role_raw, str) else ""
3264
+ if role not in ROLES:
3265
+ notes.append(f"REST API: the role {role_raw!r} of the API token {name!r} is not one "
3266
+ f"of {', '.join(ROLES)}, so the token is read-only (viewer)")
3267
+ role = "viewer"
3268
+ tokens.append(ApiToken(id=tid, name=name, sha256=digest.strip().lower(), role=role,
3269
+ created=_epoch(item.get("created")),
3270
+ expires=_epoch(item.get("expires"))))
3271
+ return tokens, notes
3272
+
3273
+
3274
+ def new_pwd_version() -> int:
3275
+ """The first pwd_version of a new account. A session cookie is checked against the
3276
+ user name and this number only, so it is random rather than 1: an account deleted
3277
+ and created again under the same name (or an LDAP/OIDC user signing in again after
3278
+ being deleted) must not match the cookies the earlier account was given."""
3279
+ return secrets.randbelow(2 ** 31)
3280
+
3281
+
3282
+ @dataclass
3283
+ class AppConfig:
3284
+ version: int = 1
3285
+ admin_ui: AdminUI = field(default_factory=AdminUI)
3286
+ secret_key: str = field(default_factory=lambda: secrets.token_hex(32))
3287
+ api_token_hash: str = "" # sha256 of the REST API bearer token (empty => API disabled)
3288
+ api_token_role: str = "admin" # noqa: S105 - role granted by the API token (viewer = read-only)
3289
+ # named API tokens besides that one (optional; written only when there are any)
3290
+ api_tokens: list[ApiToken] = field(default_factory=list)
3291
+ # lifetime of a web-UI sign-in (cookie + signed token); Settings -> Web UI access
3292
+ session_timeout_s: int = SESSION_TIMEOUT_DEFAULT_S
3293
+ # LDAP/OIDC accounts not signed in for this many days are removed (0: never, the
3294
+ # default); Settings -> Users
3295
+ external_users_expire_days: int = EXPIRE_DAYS_DEFAULT
3296
+ # trace files (data/traces) not written for this many days are deleted every hour (0:
3297
+ # kept until deleted by hand, the default); Settings -> Retention
3298
+ trace_max_age_days: int = 0
3299
+ # all.log, audit.log and backups/ keep nothing older than this many days, the mapping
3300
+ # logs at most that many of their 15 (0: only their size and count limits, the
3301
+ # default); Settings -> Retention
3302
+ log_max_age_days: int = 0
3303
+ # Web-UI accounts. Empty => first-run setup pending. A legacy single-password
3304
+ # config (top-level password_hash) is migrated to one 'admin' user on load.
3305
+ users: list[User] = field(default_factory=list)
3306
+ ldap: LdapSettings = field(default_factory=LdapSettings)
3307
+ oidc: OidcSettings = field(default_factory=OidcSettings)
3308
+ cluster: ClusterSettings = field(default_factory=ClusterSettings)
3309
+ notifications: NotificationSettings = field(default_factory=NotificationSettings)
3310
+ # Stable per-node identity for the LAN cluster (generated once, persisted).
3311
+ instance_id: str = field(default_factory=lambda: secrets.token_hex(8))
3312
+ defaults: dict[str, Any] = field(default_factory=dict) # serial defaults
3313
+ mappings: list[MappingConfig] = field(default_factory=list)
3314
+ # What loading changed in an older config (logged at startup; never persisted).
3315
+ load_notes: list[str] = field(default_factory=list, repr=False, compare=False)
3316
+ # Loading had to create secret_key / instance_id (config.json lacked them); the
3317
+ # start-up saves them once (runtime.prepare_config). Never persisted.
3318
+ ids_created: bool = field(default=False, repr=False, compare=False)
3319
+ # Parts start-up accepted although validate() rejects them (tolerate_problems): part
3320
+ # key -> its content then. Never persisted.
3321
+ tolerated: dict[str, Any] = field(default_factory=dict, repr=False, compare=False)
3322
+ # What the config.json this was loaded from had that this version doesn't use, for the
3323
+ # Settings page (a load note each, too): the paths of the keys that are no setting, and
3324
+ # the version that wrote it when that is newer than this one. Never persisted (every
3325
+ # save writes this version's "written_by").
3326
+ unknown_keys: list[str] = field(default_factory=list, repr=False, compare=False)
3327
+ newer_writer: str = field(default="", repr=False, compare=False)
3328
+
3329
+ # ----- (de)serialization -----
3330
+ @staticmethod
3331
+ def from_dict(d: dict[str, Any]) -> "AppConfig":
3332
+ """The config in a config.json (a backup, a restore). Values of the wrong type are
3333
+ normalized, with a load note each (_read_fields), never refused: a config that an
3334
+ older release wrote or accepted keeps loading. A part that isn't shaped like one
3335
+ (a section that is no JSON object, a list that is no list) raises ConfigError."""
3336
+ d = dict(d or {})
3337
+ notes: list[FieldNote] = []
3338
+ # read elsewhere: the sections, the values with their own rules, what is never stored
3339
+ own = ("admin_ui", "api_token_role", "api_tokens", "session_timeout_s",
3340
+ "external_users_expire_days", "users", "ldap", "oidc", "cluster",
3341
+ "notifications", "mappings", "load_notes", "ids_created", "tolerated",
3342
+ "unknown_keys", "newer_writer")
3343
+ top = _read_fields(AppConfig, d, notes, skip=own,
3344
+ keep=("password_hash", "pwd_version", "written_by"))
3345
+ admin = dict(_section(d, "admin_ui"))
3346
+ if isinstance(admin.get("trusted_proxies"), str): # one line (spaces or commas)
3347
+ admin["trusted_proxies"] = split_proxy_entries(admin["trusted_proxies"])
3348
+ admin_ui = AdminUI(**_read_fields(AdminUI, admin, notes, "admin_ui."))
3349
+ if not isinstance(d.get("users", []), list): # never read as "no accounts yet"
3350
+ raise ConfigError(_("{0} must be a JSON list, not {1}.").format(
3351
+ "users", _shown(d["users"])))
3352
+ users = [User.from_dict(u, notes, f"users[{i}].")
3353
+ for i, u in enumerate(_objects(d, "users"))]
3354
+ if not users and d.get("password_hash"): # migrate legacy single-password config
3355
+ users = [User(username="admin", password_hash=d["password_hash"],
3356
+ role="admin", pwd_version=int(d.get("pwd_version", 0)))]
3357
+ token_role, token_note = _api_token_role(d)
3358
+ api_tokens, token_notes = _api_tokens(d.get("api_tokens"))
3359
+ cfg = AppConfig(
3360
+ admin_ui=admin_ui,
3361
+ api_token_role=token_role,
3362
+ api_tokens=api_tokens,
3363
+ session_timeout_s=_session_timeout(d.get("session_timeout_s")),
3364
+ external_users_expire_days=_expire_days(d.get("external_users_expire_days")),
3365
+ users=users,
3366
+ ldap=LdapSettings.from_dict(_section(d, "ldap"), notes, "ldap."),
3367
+ oidc=OidcSettings.from_dict(_section(d, "oidc"), notes, "oidc."),
3368
+ cluster=ClusterSettings.from_dict(_section(d, "cluster"), notes, "cluster."),
3369
+ notifications=NotificationSettings.from_dict(_section(d, "notifications"), notes,
3370
+ "notifications."),
3371
+ mappings=[MappingConfig.from_dict(m) for m in _objects(d, "mappings")],
3372
+ **top,
3373
+ )
3374
+ cfg.secret_key = cfg.secret_key or secrets.token_hex(32)
3375
+ cfg.instance_id = cfg.instance_id or secrets.token_hex(8)
3376
+ writer = d.get("written_by")
3377
+ if isinstance(writer, str) and _newer_version(writer, __version__):
3378
+ cfg.newer_writer = writer[:40]
3379
+ cfg.load_notes.append(
3380
+ f"config.json was written by ser2netweb {cfg.newer_writer}, a newer version than "
3381
+ f"this one ({__version__}): what only {cfg.newer_writer} knows is not used here "
3382
+ "and is left out when this version saves config.json. backups/ keeps the file "
3383
+ f"as {cfg.newer_writer} left it: restore it there after upgrading again")
3384
+ cfg._note_fields("config.json", notes)
3385
+ for i, m in enumerate(cfg.mappings):
3386
+ cfg._note_fields(f"mapping '{m.name}'", m.input_notes)
3387
+ cfg.unknown_keys += [f"mappings[{i}].{n.path}" for n in m.input_notes
3388
+ if n.kind == "unknown"]
3389
+ cfg.unknown_keys[:0] = [n.path for n in notes if n.kind == "unknown"]
3390
+ _normalize_admin_limits(cfg.admin_ui, cfg.load_notes)
3391
+ _normalize_proxy_settings(cfg.admin_ui, cfg.load_notes)
3392
+ _normalize_node_name(cfg.admin_ui, cfg.load_notes)
3393
+ _normalize_request_log(cfg.admin_ui, cfg.load_notes)
3394
+ _normalize_login_notice(cfg.admin_ui, cfg.load_notes)
3395
+ cfg.ids_created = not (d.get("secret_key") and d.get("instance_id"))
3396
+ days = cfg.trace_max_age_days
3397
+ if not 0 <= days <= TRACE_DAYS_MAX: # a hand-edited value must not stop a start
3398
+ cfg.trace_max_age_days = 0 if days < 0 else TRACE_DAYS_MAX
3399
+ cfg.load_notes.append(f"trace_max_age_days {days} is not 0 (keep) or 1..{TRACE_DAYS_MAX} "
3400
+ f"days: {cfg.trace_max_age_days} is used")
3401
+ days = cfg.log_max_age_days
3402
+ if not 0 <= days <= LOG_DAYS_MAX: # (likewise)
3403
+ cfg.log_max_age_days = 0 if days < 0 else LOG_DAYS_MAX
3404
+ cfg.load_notes.append(f"log_max_age_days {days} is not 0 (size limits only) or "
3405
+ f"1..{LOG_DAYS_MAX} days: {cfg.log_max_age_days} is used")
3406
+ raw_days = d.get("external_users_expire_days")
3407
+ if raw_days is not None and raw_days != cfg.external_users_expire_days:
3408
+ cfg.load_notes.append(
3409
+ f"external_users_expire_days {raw_days!r} is not 0 (never) or "
3410
+ f"{EXPIRE_DAYS_MIN}..{EXPIRE_DAYS_MAX} days: {cfg.external_users_expire_days} "
3411
+ "is used")
3412
+ spellings: dict[str, list[str]] = {}
3413
+ for u in cfg.users:
3414
+ if u.source == "ldap":
3415
+ spellings.setdefault(fold(u.username), []).append(u.username)
3416
+ from .web import totp # stdlib only; the web package imports this module
3417
+ for u in cfg.users:
3418
+ if u.totp_on and not totp.is_secret(u.totp_secret):
3419
+ cfg.load_notes.append(
3420
+ f"user {u.username!r}: totp_secret is not a base32 secret of at least 16 "
3421
+ "characters, so no two-factor code matches it and the account can't sign "
3422
+ f"in. `ser2netweb --set-password {u.username}` sets a new password and turns "
3423
+ "two-factor sign-in off")
3424
+ for names in (n for n in spellings.values() if len(n) > 1):
3425
+ cfg.load_notes.append(
3426
+ f"LDAP accounts {', '.join(map(repr, names))} are one directory user, in "
3427
+ "different letter case: they become one account at that user's next "
3428
+ "sign-in, which ends the sessions of the other spellings (deleting one in "
3429
+ "Settings → Users deletes them all)")
3430
+ if token_note:
3431
+ cfg.load_notes.append(token_note)
3432
+ cfg.load_notes += token_notes
3433
+ if cfg.ldap.enabled and cfg.ldap.uses_tls and "tls_verify" not in (d.get("ldap") or {}):
3434
+ cfg.load_notes.append(
3435
+ "LDAP: the directory server's certificate is now checked (LDAPS / StartTLS): "
3436
+ "it must come from a CA this system trusts and name the host in the server "
3437
+ "URI. If LDAP sign-in fails with a certificate error, all.log says why; for a "
3438
+ "private CA (Active Directory Certificate Services) or a self-signed "
3439
+ "certificate, set Settings → LDAP / Active Directory → CA certificate file "
3440
+ "(ldap.ca_file). Local accounts are not affected.")
3441
+ for raw, m in zip(_objects(d, "mappings"), cfg.mappings, strict=True):
3442
+ old_id = (raw or {}).get("id")
3443
+ if old_id and old_id != m.id:
3444
+ cfg.load_notes.append(
3445
+ f"mapping '{m.name}': its id {old_id!r} is not a valid mapping id (1-64 "
3446
+ f"letters, digits or '_'), so it is now {m.id}: the REST API and /metrics "
3447
+ "use the new id, and its log and traces start afresh under it")
3448
+ raw_net = (raw or {}).get("network")
3449
+ raw_net = raw_net if isinstance(raw_net, dict) else {}
3450
+ if m.network.dual_stack is True and "dual_stack" not in raw_net: # (from_dict)
3451
+ cfg.load_notes.append(
3452
+ f"mapping '{m.name}': a UDP mapping on :: now takes IPv6 datagrams only, "
3453
+ "unless it listens for IPv4 and IPv6 (dual_stack); this one took IPv4 "
3454
+ "datagrams too on this system, so it is set to do so")
3455
+ raw_bind = raw_net.get("bind_ip")
3456
+ if isinstance(raw_bind, str) and raw_bind != m.network.bind_ip:
3457
+ cfg.load_notes.append(
3458
+ f"mapping '{m.name}': its bind IP {raw_bind!r} is now written "
3459
+ f"{m.network.bind_ip!r}"
3460
+ + (" (an IPv4-mapped IPv6 address: the listener takes that IPv4 address)"
3461
+ if ":" in raw_bind and ":" not in m.network.bind_ip else ""))
3462
+ raw_options = (raw or {}).get("options")
3463
+ raw_options = raw_options if isinstance(raw_options, dict) else {}
3464
+ old = str(raw_options.get("trace_both") or "").strip()
3465
+ # before 2.8 (written without trace_max_mb) a trace was a path: a bare name was a
3466
+ # file in ser2netweb's working directory, so it moved too
3467
+ if old and (old != m.options.trace_both or "trace_max_mb" not in raw_options):
3468
+ cfg.load_notes.append(
3469
+ f"mapping '{m.name}': its trace file is now traces/{m.id}-{m.options.trace_both} "
3470
+ f"in the data directory (configured: {old!r}; traces are no longer written "
3471
+ "outside it)")
3472
+ for key in ("allowed_client_ips", "priority_client_ips"):
3473
+ nets, bad = client_networks(getattr(m.network, key))
3474
+ if bad and m.kind == "net":
3475
+ cfg.load_notes.append(
3476
+ f"mapping '{m.name}': network.{key} entries that are not IP addresses "
3477
+ f"or networks are left out: {', '.join(map(repr, bad))}"
3478
+ + ("; none is usable, so every client is refused until the list is "
3479
+ "fixed" if key == "allowed_client_ips" and not nets else ""))
3480
+ for side in m.serial_sides():
3481
+ if url_port(side.port):
3482
+ cfg.load_notes.append(
3483
+ f"mapping '{m.name}': its serial port {side.port!r} is a URL, not a "
3484
+ "device; URL ports are no longer opened, so the mapping does not start "
3485
+ "until a serial device is set")
3486
+ odd = sorted(str(k) for k in side.match if k not in MATCH_KEYS) \
3487
+ if isinstance(side.match, dict) else []
3488
+ if odd:
3489
+ cfg.load_notes.append(
3490
+ f"mapping '{m.name}': the USB identity of serial port {side.port!r} "
3491
+ f"(match) has keys ser2netweb does not know ({', '.join(odd)}), which are "
3492
+ f"not used; it knows {', '.join(MATCH_KEYS)}"
3493
+ + ("" if side.identity else ", and none of them is set, so the port "
3494
+ "is opened by its name"))
3495
+ enabled: list[MappingConfig] = []
3496
+ for m in cfg.mappings:
3497
+ clash = _serial_clash(m, enabled)
3498
+ if clash is not None:
3499
+ other, side, oside = clash
3500
+ if side.identity:
3501
+ cfg.load_notes.append(
3502
+ f"mappings '{other.name}' and '{m.name}' are both bound to the USB "
3503
+ f"identity {identity_text(side.match)} on serial port {side.port!r}: "
3504
+ "only one of them can open it")
3505
+ elif side.port != oside.port: # (the same name is a problem validate() reports)
3506
+ cfg.load_notes.append(
3507
+ f"mappings '{other.name}' and '{m.name}' both use serial port "
3508
+ f"{oside.port!r} (written {side.port!r} in '{m.name}'): only one of "
3509
+ "them can open it")
3510
+ if m.enabled:
3511
+ enabled.append(m)
3512
+ # MQTT clashes the edits refuse now (or warn about): kept, and said at start-up
3513
+ cfg.load_notes += _mqtt_load_notes(cfg.mappings)
3514
+ return cfg
3515
+
3516
+ def serial_conflict(self, m: MappingConfig) -> str:
3517
+ """serial_conflict of ``m`` next to this config's mappings."""
3518
+ return serial_conflict(m, self.mappings)
3519
+ def _note_fields(self, where: str, notes: list[FieldNote]) -> None:
3520
+ """Add what reading `where` did (FieldNote) to the load notes."""
3521
+ self.load_notes.extend(f"{where}: {n.text()}" for n in notes if n.kind != "unknown")
3522
+ unknown = [n.path for n in notes if n.kind == "unknown"]
3523
+ if unknown:
3524
+ shown = ", ".join(unknown[:20]) + (f" and {len(unknown) - 20} more"
3525
+ if len(unknown) > 20 else "")
3526
+ self.load_notes.append(
3527
+ f"{where}: not settings of this version, so they are not used and are left "
3528
+ f"out when config.json is next saved (backups/ keeps the file as it was): {shown}")
3529
+
3530
+ def to_dict(self) -> dict[str, Any]:
3531
+ return {
3532
+ "version": self.version,
3533
+ # which release wrote the file (older ones drop it): a later load by an older
3534
+ # release notes what it can't keep, and the backups list shows it
3535
+ "written_by": __version__,
3536
+ "admin_ui": asdict(self.admin_ui),
3537
+ "secret_key": self.secret_key,
3538
+ "api_token_hash": self.api_token_hash,
3539
+ "api_token_role": self.api_token_role,
3540
+ # only when there are any: a config.json without named tokens stays as it was
3541
+ **({"api_tokens": [asdict(t) for t in self.api_tokens]} if self.api_tokens else {}),
3542
+ "session_timeout_s": self.session_timeout_s,
3543
+ "external_users_expire_days": self.external_users_expire_days,
3544
+ "trace_max_age_days": self.trace_max_age_days,
3545
+ "log_max_age_days": self.log_max_age_days,
3546
+ "users": [u.to_dict() for u in self.users],
3547
+ "ldap": asdict(self.ldap),
3548
+ "oidc": asdict(self.oidc),
3549
+ "cluster": asdict(self.cluster),
3550
+ "notifications": asdict(self.notifications),
3551
+ "instance_id": self.instance_id,
3552
+ "defaults": self.defaults,
3553
+ "mappings": [m.to_dict() for m in self.mappings],
3554
+ }
3555
+
3556
+ # ----- helpers -----
3557
+ @property
3558
+ def password_set(self) -> bool:
3559
+ return bool(self.users)
3560
+
3561
+ def get_user(self, username: str) -> Optional[User]:
3562
+ return next((u for u in self.users if u.username == username), None)
3563
+
3564
+ def named(self, name: str, source: Optional[str] = None) -> list[User]:
3565
+ """The accounts (of `source`, if given) whose name is `name` apart from letter
3566
+ case (fold: Turkish İ/ı too), the one spelled exactly so first."""
3567
+ key = fold(name)
3568
+ found = [u for u in self.users
3569
+ if (source is None or u.source == source) and fold(u.username) == key]
3570
+ return sorted(found, key=lambda u: u.username != name)
3571
+
3572
+ def admin_count(self) -> int:
3573
+ return sum(1 for u in self.users if u.role == "admin")
3574
+
3575
+ def source_enabled(self, user: User) -> bool:
3576
+ """False for an LDAP/OIDC account while that sign-in method is switched off:
3577
+ its sessions end and it can't sign in until the method is on again."""
3578
+ if user.source == "ldap":
3579
+ return self.ldap.enabled
3580
+ if user.source == "oidc":
3581
+ return self.oidc.enabled
3582
+ return True
3583
+
3584
+ def usable_admin_count(self) -> int:
3585
+ """Admins who can sign in with the current settings (local accounts, and
3586
+ LDAP/OIDC ones while their sign-in method is on)."""
3587
+ return sum(1 for u in self.users if u.role == "admin" and self.source_enabled(u))
3588
+
3589
+ def external_accounts(self, name: str, source: str) -> list[User]:
3590
+ """The accounts of the LDAP/OIDC user who signs in as `name`. An LDAP name is the
3591
+ same name in any letter case (directories compare it that way, and a phone may
3592
+ send "Jdoe" for jdoe): every LDAP account named so, the exact spelling first. An
3593
+ OIDC name is compared exactly (a provider's "sub" is case-sensitive)."""
3594
+ if source == "ldap":
3595
+ return self.named(name, "ldap")
3596
+ return [u for u in self.users if u.source == source and u.username == name]
3597
+
3598
+ def taken_name(self, username: str, source: str) -> Optional[User]:
3599
+ """The account, letter case aside, that keeps an LDAP/OIDC sign-in (`source`)
3600
+ from having one under this name: a local account, else one of the other sign-in
3601
+ method. None when there is none."""
3602
+ other = "oidc" if source == "ldap" else "ldap"
3603
+ found = self.named(username, "local") or self.named(username, other)
3604
+ return found[0] if found else None
3605
+
3606
+ def upsert_external_user(self, username: str, role: str, source: str) -> User:
3607
+ """Create or refresh a shadow account for an externally-authenticated user
3608
+ (LDAP/OIDC). Bumps pwd_version when the role or source changes.
3609
+ A sign-in through a directory or an identity provider never takes over, converts
3610
+ or demotes another kind of account — a local one, or one of the other sign-in
3611
+ method (an LDAP and an OIDC user of one name may be two people): raises
3612
+ AccountNameError when one has this name, letter case aside, or when the name
3613
+ can't be an account's (external_name_problem; `username` is spelled as
3614
+ external_name() spells it).
3615
+ LDAP: the account is the one spelled like `username`, else the first in another
3616
+ letter case (external_accounts); the person's other LDAP accounts — made by
3617
+ earlier versions for each spelling — are removed, which ends their sessions."""
3618
+ if taken := self.taken_name(username, source):
3619
+ raise AccountNameError(f"the {taken.kind} account {taken.username!r} has this name")
3620
+ if (why := external_name_problem(username)) is not None:
3621
+ raise AccountNameError(why)
3622
+ same = self.external_accounts(username, source)
3623
+ if len(same) > 1:
3624
+ self.users = [x for x in self.users if not any(x is y for y in same[1:])]
3625
+ u = same[0] if same else self.get_user(username)
3626
+ if u is None:
3627
+ u = User(username=username, password_hash="", role=role, source=source,
3628
+ pwd_version=new_pwd_version())
3629
+ self.users.append(u)
3630
+ elif u.role != role or u.source != source:
3631
+ u.role, u.source = role, source
3632
+ u.pwd_version += 1
3633
+ return u
3634
+
3635
+ def get_mapping(self, mapping_id: str) -> Optional[MappingConfig]:
3636
+ return next((m for m in self.mappings if m.id == mapping_id), None)
3637
+
3638
+ def find_moved_tls_files(self, data_dir: Optional[str] = None) -> list[str]:
3639
+ """TLS files that are not where the config says, because the data directory was
3640
+ moved or restored elsewhere (a native install moved to Docker's /data, a Windows
3641
+ service given another --data-dir, a backup restored on another machine), taken from
3642
+ the tls folder of ``data_dir`` (else ConfigStore's; moved_tls_file): the admin UI's
3643
+ certificate and key (the pair Settings → Admin TLS generates is always there), and
3644
+ a mapping's. A pair is taken only when both of its files are found. Changes the
3645
+ config (to absolute paths, which older releases read the same) and returns what it
3646
+ did, one log line each."""
3647
+ base = data_dir or _data_dir
3648
+ notes: list[str] = []
3649
+ if not base:
3650
+ return notes
3651
+
3652
+ def take(owner: Any, label: str, attrs: tuple[str, ...]) -> None:
3653
+ paths = [getattr(owner, a) for a in attrs]
3654
+ if not all(paths):
3655
+ return
3656
+ found = [moved_tls_file(p, base) for p in paths]
3657
+ if not all(found):
3658
+ return
3659
+ for attr, old, new in zip(attrs, paths, found, strict=True):
3660
+ setattr(owner, attr, new)
3661
+ notes.append(f"{label}{attr}: {old!r} is not there; {new!r} of this data "
3662
+ "directory is used (the data directory was moved or restored)")
3663
+
3664
+ take(self.admin_ui, "admin_ui.", ("tls_cert", "tls_key"))
3665
+ for m in self.mappings:
3666
+ label = f"mapping '{m.name}': "
3667
+ take(m.network, label, ("tls_cert", "tls_key"))
3668
+ for attr in ("tls_ca", "tls_client_ca"):
3669
+ take(m.network, label, (attr,))
3670
+ return notes
3671
+
3672
+ def admin_key_reuse(self, m: MappingConfig) -> str:
3673
+ """Why mapping ``m`` can't be saved with its TLS key ("" when it can): the key is the
3674
+ admin UI's own key file, so the mapping would present the admin UI's certificate on
3675
+ another port of this host, and browsers send a site's cookies to every port of it.
3676
+ Checked where a mapping is created or changed (the form, so also a cluster peer's
3677
+ save on its own node, and the REST API), never by validate(): a stored one keeps
3678
+ loading and saving."""
3679
+ net, admin = m.network, self.admin_ui
3680
+ if not (m.kind == "net" and net.tls and net.tls_key and admin.tls_key):
3681
+ return ""
3682
+ try:
3683
+ same = os.path.samefile(data_path(net.tls_key), data_path(admin.tls_key))
3684
+ except (OSError, TypeError, ValueError): # one of them is missing: not the same file
3685
+ return ""
3686
+ if not same:
3687
+ return ""
3688
+ return _("The TLS key {0} is the admin UI's own key. Give the mapping a certificate "
3689
+ "and key of its own.").format(net.tls_key)
3690
+
3691
+ def own_port_conflict(self, m: MappingConfig) -> str:
3692
+ """Why mapping ``m`` can't listen where it is set to next to ser2netweb's own
3693
+ listeners ("" when it can): a TCP listener on the admin UI's address and port, or a
3694
+ UDP one on the LAN cluster's discovery port while discovery is on (it would get the
3695
+ other nodes' beacons). Checked where a mapping is created or changed (the form, the
3696
+ REST API, a cluster peer, an import, a copy), never by validate(): a stored clash
3697
+ must not stop a start or another save, and the admin UI wins at start-up anyway
3698
+ (it listens before the mappings start), so such a mapping only fails to start."""
3699
+ if m.kind != "net" or not m.network.listens:
3700
+ return ""
3701
+ net = m.network
3702
+ if net.mode == "server":
3703
+ admin = self.admin_ui
3704
+ # "" binds every interface of both families (as asyncio and uvicorn read it),
3705
+ # "localhost" both loopback addresses
3706
+ ips = {"": ["0.0.0.0", "::"], "localhost": ["127.0.0.1", "::1"]}.get(
3707
+ admin.bind_ip, [admin.bind_ip])
3708
+ if net.port == admin.port and any(
3709
+ _ip_overlaps(ip, net.bind_ip, b_both=net.both_families) for ip in ips):
3710
+ return _("Mapping '{0}': TCP port {1} on {2} is the admin UI's port ({3}:{4}), "
3711
+ "so the mapping could not listen there. Pick another port.").format(
3712
+ m.name, net.port, net.bind_shown, shown_host(admin.bind_ip), admin.port)
3713
+ return ""
3714
+ cluster = self.cluster
3715
+ try:
3716
+ discovery = int(cluster.discovery_port)
3717
+ except (TypeError, ValueError):
3718
+ return ""
3719
+ # discovery listens on IPv4 (0.0.0.0): an IPv6-only listener doesn't get its beacons
3720
+ if (cluster.active and cluster.discovery and net.port == discovery and _ip_overlaps(
3721
+ "0.0.0.0", net.bind_ip, b_both=net.both_families)):
3722
+ return _("Mapping '{0}': UDP port {1} is the LAN cluster's discovery port, so the "
3723
+ "mapping would get the other nodes' beacons. Pick another port (or "
3724
+ "another discovery port under Settings → LAN cluster).").format(
3725
+ m.name, net.port)
3726
+ return ""
3727
+
3728
+ def validate(self) -> None:
3729
+ """Validate every part (the admin UI settings, each section, each mapping) and the
3730
+ cross-mapping invariants (unique IP:port, one mapping per serial port); raises
3731
+ the first problem as ConfigError."""
3732
+ for _key, _label, _content, check in self._parts():
3733
+ check()
3734
+ for _key, error in self._clashes():
3735
+ raise error
3736
+
3737
+ def _validate_admin_ui(self) -> None:
3738
+ # TLS: both files required together (that they exist: AdminUI.check_tls_files)
3739
+ cert, key = self.admin_ui.tls_cert, self.admin_ui.tls_key
3740
+ if bool(cert) != bool(key):
3741
+ raise ConfigError(_("Set both TLS certificate and key, or neither."))
3742
+
3743
+ # Admin UI bind IP (set via the console / --reconfigure, or a hand-edited
3744
+ # config). Catch a bad value here with a clear error instead of letting
3745
+ # uvicorn crash at bind time.
3746
+ if self.admin_ui.bind_ip:
3747
+ try:
3748
+ ipaddress.ip_address(self.admin_ui.bind_ip)
3749
+ except ValueError:
3750
+ raise ConfigError(
3751
+ _("Invalid admin UI bind IP: {0}").format(self.admin_ui.bind_ip)) from None
3752
+ if self.admin_ui.language not in UI_LANGUAGES:
3753
+ raise ConfigError(_("The default language must be one of {0}.").format(_choices(UI_LANGUAGES)))
3754
+ if self.admin_ui.anonymous_role not in ROLES:
3755
+ raise ConfigError(_("Role for visitors without login must be one of {0}.").format(_choices(ROLES)))
3756
+ if not _is_int(self.session_timeout_s) or not (
3757
+ SESSION_TIMEOUT_MIN_S <= self.session_timeout_s <= SESSION_TIMEOUT_MAX_S):
3758
+ raise ConfigError(_("Session lifetime must be between {0} minutes and {1} days.").format(SESSION_TIMEOUT_MIN_S // 60, SESSION_TIMEOUT_MAX_S // 86400))
3759
+
3760
+ def _parts(self):
3761
+ """validate()'s checks, one per part of the config, in order: (key, label,
3762
+ content, check). content() is what tells whether a save changed the part; a
3763
+ mapping's enabled flag is left out of it, since starting or stopping one changes
3764
+ nothing its own check looks at (a serial port it then shares is a clash)."""
3765
+ yield ("admin_ui", "admin UI", lambda: (asdict(self.admin_ui), self.session_timeout_s),
3766
+ self._validate_admin_ui)
3767
+ for key, label in (("ldap", "LDAP"), ("oidc", "OIDC"), ("cluster", "LAN cluster"),
3768
+ ("notifications", "notifications")):
3769
+ section = getattr(self, key)
3770
+ yield key, label, functools.partial(asdict, section), section.validate
3771
+ for m in self.mappings:
3772
+ yield (f"mapping {m.id}", f"mapping '{m.name or m.id}'",
3773
+ functools.partial(_mapping_content, m), m.validate)
3774
+
3775
+ def _clashes(self) -> list[tuple[str, ConfigError]]:
3776
+ """Every pair of mappings that would share a serial port or a listening port, keyed
3777
+ by the resource and both mapping ids."""
3778
+ found = []
3779
+ serial_users: list[tuple[str, MappingConfig]] = [] # (device, mapping)
3780
+ listeners: list[tuple[str, int, MappingConfig]] = [] # (proto, port, mapping)
3781
+ for m in self.mappings:
3782
+ # A serial port can only be opened by one mapping. Flag two ENABLED
3783
+ # mappings that target the same literal device (skip dynamic VID/PID
3784
+ # `match` entries, whose device path is resolved at open time).
3785
+ if m.enabled:
3786
+ devices = []
3787
+ if m.serial.port and not m.serial.match:
3788
+ devices.append(m.serial.port)
3789
+ if m.kind == "serialbridge" and m.serial_b.port and not m.serial_b.match:
3790
+ devices.append(m.serial_b.port)
3791
+ for dev in devices:
3792
+ for used, other in serial_users:
3793
+ if used == dev and other.id != m.id:
3794
+ found.append((f"serial {dev} {other.id} {m.id}", ConfigError(
3795
+ _("Mappings '{0}' and '{1}' both use serial port {2} — a serial "
3796
+ "port can only be opened by one mapping.").format(other.name, m.name, dev))))
3797
+ serial_users.append((dev, m))
3798
+
3799
+ # only listening mappings (server/udp) bind a port; client/serialbridge don't
3800
+ if m.kind != "net" or not m.network.listens:
3801
+ continue
3802
+ # A TCP listener and a UDP listener CAN share the same port number (they
3803
+ # bind different transports), so the clash key includes the transport.
3804
+ proto = "udp" if m.network.mode == "udp" else "tcp"
3805
+ # A wildcard collides with every address of its family on that port (of both,
3806
+ # when it takes both: see _ip_overlaps); treat any overlap as a clash.
3807
+ net = m.network
3808
+ for sproto, bport, other in listeners:
3809
+ onet = other.network
3810
+ if sproto == proto and bport == net.port and _ip_overlaps(
3811
+ onet.bind_ip, net.bind_ip, onet.both_families, net.both_families):
3812
+ found.append((f"{proto} {net.port} {other.id} {m.id}", ConfigError(
3813
+ _("Mappings '{0}' and '{1}' both use {2} port {3} on overlapping "
3814
+ "addresses ({4} / {5}).").format(other.name, m.name, proto.upper(), net.port, onet.bind_ip, net.bind_ip))))
3815
+ listeners.append((proto, net.port, m))
3816
+ return found
3817
+
3818
+ def _broken(self) -> list[tuple[str, str, Any]]:
3819
+ """(key, problem, content) for every part validate() rejects and every clash, and
3820
+ each TLS file a mapping names that doesn't exist (tls_files_missing)."""
3821
+ found = []
3822
+ missing = {f"mapping {m.id}": tls_files_missing(m.network)
3823
+ for m in self.mappings if m.kind == "net"}
3824
+ for key, label, content, check in self._parts():
3825
+ try:
3826
+ check()
3827
+ except Exception as e: # a hand-edited value of the wrong type raises others too
3828
+ found.append((key, f"{label}: {e}", content()))
3829
+ try:
3830
+ found += [(key, f"{label}: {problem}", content())
3831
+ for problem in missing.get(key, ())]
3832
+ except Exception: # its content can't be read: named above already
3833
+ pass
3834
+ try:
3835
+ found += [(key, str(error), None) for key, error in self._clashes()]
3836
+ except Exception as e: # mappings too broken to compare
3837
+ found.append(("clashes", f"mappings: {e}", None))
3838
+ return found
3839
+
3840
+ def problems(self) -> list[str]:
3841
+ """Everything validate() rejects in this config, one line each, naming the part
3842
+ (validate() stops at the first)."""
3843
+ return [problem for _key, problem, _content in self._broken()]
3844
+
3845
+ def tolerate_problems(self) -> list[str]:
3846
+ """Accept what this config breaks now and return it, one line each: later saves
3847
+ may leave those parts as they are (validate_for_save). Start-up does this, since
3848
+ a config.json that loads is never refused: an older release may have accepted
3849
+ it, or a file it names may be missing on this machine (runtime.prepare_config)."""
3850
+ broken = self._broken()
3851
+ self.tolerated = {key: content for key, _problem, content in broken}
3852
+ return [problem for _key, problem, _content in broken]
3853
+
3854
+ def validate_for_save(self) -> dict[str, Any]:
3855
+ """What a save enforces: validate(), except that a part start-up accepted while
3856
+ broken (tolerate_problems) may stay broken as long as it is left as it was, and
3857
+ so may a clash it accepted. What a save changes or adds must validate, so one bad
3858
+ value can't block every other change (signing in with LDAP/OIDC, stopping a
3859
+ mapping, fixing another broken part).
3860
+
3861
+ Returns what is still accepted after this save (a part it fixed is held to the
3862
+ rules again). ConfigStore.save commits that only once the file is written, so a
3863
+ save that fails to write — and is rolled back by its caller — keeps the
3864
+ acceptance it had."""
3865
+ if not self.tolerated:
3866
+ self.validate()
3867
+ return {}
3868
+ still: dict[str, Any] = {}
3869
+ for key, _label, content, check in self._parts():
3870
+ try:
3871
+ check()
3872
+ except Exception as e:
3873
+ if key not in self.tolerated or content() != self.tolerated[key]:
3874
+ if isinstance(e, ConfigError):
3875
+ raise
3876
+ raise ConfigError(str(e)) from e
3877
+ still[key] = self.tolerated[key]
3878
+ try:
3879
+ clashes = self._clashes()
3880
+ except Exception as e:
3881
+ if "clashes" not in self.tolerated:
3882
+ raise ConfigError(str(e)) from e
3883
+ clashes, still["clashes"] = [], None
3884
+ for key, error in clashes:
3885
+ if key not in self.tolerated:
3886
+ raise error
3887
+ still[key] = None
3888
+ return still
3889
+
3890
+
3891
+ def mapping_revision(secret_key: str, m: "MappingConfig") -> str:
3892
+ """An opaque revision of a mapping's settings (never stored): what an edit form or a
3893
+ REST client saw, to tell whether the mapping changed since. An HMAC with this node's
3894
+ secret key, so it tells nothing about the settings (the MQTT password a viewer token
3895
+ doesn't get), and another node's revision is its own."""
3896
+ key = hmac.new(secret_key.encode(), b"ser2net mapping revision", hashlib.sha256).digest()
3897
+ body = json.dumps(m.to_dict(), sort_keys=True, separators=(",", ":"), default=str)
3898
+ return hmac.new(key, body.encode(), hashlib.sha256).hexdigest()[:24]
3899
+
3900
+
3901
+ def _mapping_content(m: "MappingConfig") -> dict[str, Any]:
3902
+ content = m.to_dict()
3903
+ content.pop("enabled", None)
3904
+ return content
3905
+
3906
+
3907
+ def _version_parts(text: str) -> Optional[tuple[int, ...]]:
3908
+ """(2, 9, 0) for "2.9.0" (or "2.9.0rc1"), None for anything else."""
3909
+ m = re.match(r"(\d+)\.(\d+)(?:\.(\d+))?", text.strip())
3910
+ return tuple(int(x or 0) for x in m.groups()) if m else None
3911
+
3912
+
3913
+ def _newer_version(text: str, than: str) -> bool:
3914
+ a, b = _version_parts(text), _version_parts(than)
3915
+ return a is not None and b is not None and a > b
3916
+
3917
+
3918
+ def _api_token_role(d: dict[str, Any]) -> tuple[str, str]:
3919
+ """api_token_role from a config file, and a note when it had to be changed. No key
3920
+ means a token from before token roles (v2.3), which had full access: admin. A role
3921
+ in other letter case or with spaces is that role. Anything else is not a role, and
3922
+ the token gets the least access (viewer, read-only) rather than a guess."""
3923
+ if "api_token_role" not in d:
3924
+ return "admin", ""
3925
+ raw = d["api_token_role"]
3926
+ role = raw.strip().lower() if isinstance(raw, str) else ""
3927
+ if role in ROLES:
3928
+ return role, ""
3929
+ if not d.get("api_token_hash"): # no token: generating one picks its role anyway
3930
+ return "viewer", ""
3931
+ return "viewer", (f"REST API: api_token_role {raw!r} is not one of {', '.join(ROLES)}, so the "
3932
+ "API token is read-only (viewer). For more, generate a new token with the "
3933
+ "role it needs under Settings → REST API token.")
3934
+
3935
+
3936
+ def _session_timeout(value: Any) -> int:
3937
+ """session_timeout_s from a config file. Before v2.8 it was stored but never used
3938
+ (sessions always lasted 8 h), so an odd hand-edited value must not stop a start:
3939
+ unreadable -> the default, anything else clamped into the supported range."""
3940
+ try:
3941
+ seconds = int(float(value))
3942
+ except (TypeError, ValueError, OverflowError):
3943
+ return SESSION_TIMEOUT_DEFAULT_S
3944
+ return min(max(seconds, SESSION_TIMEOUT_MIN_S), SESSION_TIMEOUT_MAX_S)
3945
+
3946
+
3947
+ def _epoch(value: Any) -> float:
3948
+ """A stored point in time (epoch seconds); anything unreadable is 0 (none)."""
3949
+ if isinstance(value, bool):
3950
+ return 0.0
3951
+ try:
3952
+ t = float(value)
3953
+ except (TypeError, ValueError, OverflowError):
3954
+ return 0.0
3955
+ return t if math.isfinite(t) and t > 0 else 0.0
3956
+
3957
+
3958
+ def _expire_days(value: Any) -> int:
3959
+ """external_users_expire_days from a config file: missing or unreadable -> the
3960
+ default (0: never), 0 or less -> 0, anything else clamped into the supported range (a
3961
+ hand-edited value must not stop a start; from_dict notes the change)."""
3962
+ if value is None or isinstance(value, bool):
3963
+ return EXPIRE_DAYS_DEFAULT
3964
+ try:
3965
+ days = int(value)
3966
+ except (TypeError, ValueError, OverflowError):
3967
+ return EXPIRE_DAYS_DEFAULT
3968
+ return 0 if days <= 0 else min(max(days, EXPIRE_DAYS_MIN), EXPIRE_DAYS_MAX)
3969
+
3970
+
3971
+ def _ip_overlaps(a: str, b: str, a_both: bool = False, b_both: bool = False) -> bool:
3972
+ """Whether listeners on bind IPs ``a`` and ``b`` (on one port) need the same address:
3973
+ the same address however it is written ('::' and '0:0:0:0:0:0:0:0'), or a wildcard and
3974
+ an address of its family. A listener on :: takes IPv6 only (IPV6_V6ONLY, TCP and UDP),
3975
+ so it can share a port with one on 0.0.0.0 or another IPv4 address; one that takes both
3976
+ families (``a_both`` / ``b_both``: NetworkSettings.both_families) overlaps either."""
3977
+ xs = list(WILDCARDS) if a_both else [a]
3978
+ ys = list(WILDCARDS) if b_both else [b]
3979
+ return any(_address_overlaps(x, y) for x in xs for y in ys)
3980
+
3981
+
3982
+ def _address_overlaps(a: str, b: str) -> bool:
3983
+ x, y = canonical_ip(a), canonical_ip(b)
3984
+ if x == y:
3985
+ return True
3986
+ try:
3987
+ ax, ay = ipaddress.ip_address(x), ipaddress.ip_address(y)
3988
+ except ValueError:
3989
+ return False # not an address (validate() says so): only the same text overlaps
3990
+ return ax.version == ay.version and (ax.is_unspecified or ay.is_unspecified)
3991
+
3992
+
3993
+ # ---------------------------------------------------------------------------
3994
+ # Atomic store
3995
+ # ---------------------------------------------------------------------------
3996
+ BACKUPS_KEPT = 50
3997
+ # with log_max_age_days set, backups older than that go (ConfigStore.prune_backups), but
3998
+ # never the newest BACKUPS_MIN_KEPT: a clock far ahead must not take every way back
3999
+ BACKUPS_MIN_KEPT = 5
4000
+ _BACKUP_NAME = re.compile(r"^config-\d{8}T\d{6}Z(-\d+)?\.json$")
4001
+
4002
+
4003
+ def now() -> float:
4004
+ """The wall clock that names backups (tests set another)."""
4005
+ return time.time()
4006
+
4007
+
4008
+ class _Seen(typing.NamedTuple):
4009
+ """config.json as a store last read or wrote it."""
4010
+ key: Optional[tuple[int, int]] # (mtime_ns, size); None: unknown
4011
+ digest: str # sha256 of its text
4012
+
4013
+
4014
+ def _digest(text: str) -> str:
4015
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()
4016
+
4017
+
4018
+ def _stat_key(st: os.stat_result) -> tuple[int, int]:
4019
+ return st.st_mtime_ns, st.st_size
4020
+
4021
+
4022
+ class ConfigStore:
4023
+ """config.json with atomic writes, plus automatic backups.
4024
+
4025
+ Every save keeps the file it writes as the newest backup,
4026
+ <data>/backups/config-<UTC time>.json, and first the version it replaces when that is
4027
+ not the newest backup already (an older release wrote it, or it was edited by hand).
4028
+ So backups/ always holds the current config.json too: going back to a release that
4029
+ keeps no backups (2.7) and drops what it doesn't know at its first save loses nothing.
4030
+ Identical content is not copied twice; the newest BACKUPS_KEPT are kept. Starting and
4031
+ stopping mappings takes no backup of its own (save(backup=False)), so it doesn't push
4032
+ the backups of real changes out. A new name always sorts first, also when the clock
4033
+ went back (it then takes the newest name's time, with the next number). Backups are
4034
+ never read back on their own: restoring one is an explicit admin action (deleting
4035
+ config.json alone leaves them, and the next admin sees them in Settings). An admin can
4036
+ delete backups, or remove a deleted account from every backup (remove_account), in
4037
+ Settings -> Configuration backups.
4038
+
4039
+ The store remembers what config.json held when it last read or wrote it. A save that
4040
+ finds something else there replaces an edit made outside ser2netweb (by hand, a tool)
4041
+ that was not reloaded: the save goes ahead (sign-ins, starts and stops must not fail
4042
+ for it), the edited file is kept in backups/ first as any replaced version is, and
4043
+ the replacement is reported (replaced_edit, a WARNING via log, on_replaced).
4044
+ changed_on_disk() tells of such an edit before a save does."""
4045
+
4046
+ def __init__(self, path: str):
4047
+ self.path = os.path.abspath(path)
4048
+ directory = os.path.dirname(self.path)
4049
+ set_data_dir(directory) # (relative TLS paths: data_path)
4050
+ created = not os.path.isdir(directory)
4051
+ os.makedirs(directory, exist_ok=True)
4052
+ # config.json holds the password hash + secret_key; keep the dir private on
4053
+ # BOTH platforms (POSIX chmod 0700 / Windows icacls owner-only inheritance), when
4054
+ # it is ser2netweb's own: just made, or holding only what ser2netweb puts there. Any
4055
+ # other folder (sudo ser2netweb --config /etc/ser2netweb.json) keeps its permissions:
4056
+ # only the config file is made private (on every save), and the start-up log
4057
+ # warns when other accounts can use the folder.
4058
+ self.notes: list[str] = [] # for the start-up log, with the config's load_notes
4059
+ other = [] if created else foreign_entries(directory, os.path.basename(self.path))
4060
+ self.own_dir = other == []
4061
+ if self.own_dir:
4062
+ lock_down_dir(directory)
4063
+ else:
4064
+ if self.exists():
4065
+ lock_down_file(self.path)
4066
+ if others_can_use(directory):
4067
+ what = ("can't be listed" if other is None else "holds files ser2netweb didn't "
4068
+ f"put there ({', '.join(other[:3])}{', ...' if len(other) > 3 else ''})")
4069
+ self.notes.append(
4070
+ f"WARNING: {directory}, the folder of {os.path.basename(self.path)}, "
4071
+ f"{what}, so its permissions are left as they are instead of making it "
4072
+ "private to this account, and other accounts can use it. The "
4073
+ "configuration file, backups/, logs/ and traces/ are kept private; give "
4074
+ "ser2netweb a folder of its own for the rest (--data-dir <folder> without "
4075
+ "--config; README: Configuration & state files)")
4076
+ self.backup_dir = os.path.join(directory, "backups")
4077
+ # what each backup holds, by name: ((size, mtime_ns), _backup_summary()) - see backups()
4078
+ self._backup_summaries: dict[str, tuple[tuple[int, int], dict[str, Any]]] = {}
4079
+ # where a backup that fails, or a clock behind the backups, is reported (AppState
4080
+ # sets its log; a save itself never fails because of a backup)
4081
+ self.log: Any = lambda _msg: None
4082
+ self._clock_noted = False
4083
+ self._seen: Optional[_Seen] = None
4084
+ # the last edit made outside ser2netweb that a save replaced: {"edited": its mtime,
4085
+ # "saved": when, "backup": the backup's name ("" when none could be written)};
4086
+ # on_replaced(that) is called too (AppState: an audit record)
4087
+ self.replaced_edit: Optional[dict] = None
4088
+ self.on_replaced: Any = None
4089
+
4090
+ def exists(self) -> bool:
4091
+ return os.path.isfile(self.path)
4092
+
4093
+ def load(self) -> AppConfig:
4094
+ if not self.exists():
4095
+ config = AppConfig()
4096
+ else:
4097
+ text, seen = self.read_current()
4098
+ config = AppConfig.from_dict(json.loads(text))
4099
+ self.note_read(seen)
4100
+ config.load_notes.extend(self.notes)
4101
+ return config
4102
+
4103
+ def read_current(self) -> tuple[str, _Seen]:
4104
+ """config.json's text, and what note_read() remembers of it once it is used."""
4105
+ with open(self.path, "r", encoding="utf-8") as fh:
4106
+ st = os.fstat(fh.fileno())
4107
+ text = fh.read()
4108
+ return text, _Seen(_stat_key(st), _digest(text))
4109
+
4110
+ def note_read(self, seen: _Seen) -> None:
4111
+ """The configuration in use is the one read_current() gave (a load, a reload)."""
4112
+ self._seen = seen
4113
+
4114
+ def changed_on_disk(self) -> Optional[float]:
4115
+ """When config.json holds something other than what this store last read or wrote
4116
+ (an edit made outside ser2netweb, which is not in use until it is reloaded): when it
4117
+ was changed (its mtime; 0.0 when the file is gone). Else None. Blocking: a stat, and
4118
+ a read only when the stat differs from what was seen."""
4119
+ seen = self._seen
4120
+ if seen is None:
4121
+ return None
4122
+ try:
4123
+ st = os.stat(self.path)
4124
+ except FileNotFoundError:
4125
+ return 0.0
4126
+ except OSError:
4127
+ return None
4128
+ if _stat_key(st) == seen.key:
4129
+ return None
4130
+ try:
4131
+ text, now_seen = self.read_current()
4132
+ except FileNotFoundError:
4133
+ return 0.0
4134
+ except (OSError, ValueError):
4135
+ return st.st_mtime
4136
+ if now_seen.digest == seen.digest: # only touched
4137
+ self._seen = now_seen
4138
+ return None
4139
+ return st.st_mtime
4140
+
4141
+ def matches(self, config: AppConfig) -> bool:
4142
+ """True when config.json already holds exactly this configuration, so saving it
4143
+ would change nothing (start-up then leaves the file alone). Which release wrote it
4144
+ doesn't count: an upgrade alone doesn't rewrite it."""
4145
+ try:
4146
+ with open(self.path, "r", encoding="utf-8") as fh:
4147
+ on_disk = json.load(fh)
4148
+ except (OSError, ValueError):
4149
+ return False
4150
+ mine = json.loads(json.dumps(config.to_dict()))
4151
+ if isinstance(on_disk, dict):
4152
+ on_disk.pop("written_by", None)
4153
+ mine.pop("written_by", None)
4154
+ return on_disk == mine
4155
+
4156
+ def save(self, config: AppConfig, *, validate: bool = True, backup: bool = True) -> bool:
4157
+ """Atomically persist config to a single file (validates first: what start-up
4158
+ accepted while broken may stay as it was, see AppConfig.validate_for_save).
4159
+
4160
+ Uses a temp file + os.replace so config.json is never partially written; the
4161
+ version being replaced is kept in backups/ first, the new one after (see the class
4162
+ docstring). Content config.json holds already is not written again.
4163
+ validate=False is for a change the caller checked itself and that must not
4164
+ fail on an unrelated setting (account recovery, ser2netweb/accounts.py; start-up right
4165
+ after tolerate_problems(), runtime.prepare_config).
4166
+ backup=False: a change only of which mappings run (start / stop). It takes no backup
4167
+ of its own: when the newest backup is the file it replaces, the new file takes its
4168
+ place (as the newest backup, under a new name); otherwise it is kept as usual.
4169
+ Returns whether the configuration config.json held before is kept in backups/
4170
+ (False: that backup could not be written; logged)."""
4171
+ still_accepted = config.validate_for_save() if validate else config.tolerated
4172
+ data = json.dumps(config.to_dict(), indent=2, ensure_ascii=False)
4173
+ directory = os.path.dirname(self.path)
4174
+ try:
4175
+ current, current_seen = self.read_current()
4176
+ except OSError:
4177
+ current, current_seen = None, None # first save: nothing to keep
4178
+ if current == data: # config.json holds this already: nothing to write
4179
+ config.tolerated = still_accepted
4180
+ self._add_backup(data) # (newest already, unless backups/ was emptied)
4181
+ self._seen = current_seen
4182
+ return True
4183
+ # an edit made outside ser2netweb, which this save replaces (kept, and reported)
4184
+ outside = (current_seen is not None and self._seen is not None
4185
+ and current_seen.digest != self._seen.digest)
4186
+ kept, replaces, kept_as = True, None, ""
4187
+ if current is not None:
4188
+ if not backup and (newest := self._newest()) is not None and newest[1] == current:
4189
+ replaces = newest[0] # the newest backup is the file replaced: it goes
4190
+ else:
4191
+ kept_as = self._add_backup(current)
4192
+ kept = bool(kept_as)
4193
+
4194
+ fd, tmp = tempfile.mkstemp(dir=directory, prefix=".cfg.", suffix=".tmp")
4195
+ try:
4196
+ if not self.own_dir: # (Windows: it got the folder's permissions; POSIX: 0600)
4197
+ lock_down_file(tmp)
4198
+ with os.fdopen(fd, "w", encoding="utf-8") as fh:
4199
+ fh.write(data)
4200
+ fh.flush()
4201
+ os.fsync(fh.fileno())
4202
+ written = _stat_key(os.fstat(fh.fileno()))
4203
+ os.replace(tmp, self.path)
4204
+ finally:
4205
+ if os.path.exists(tmp):
4206
+ os.unlink(tmp)
4207
+ self._seen = _Seen(written, _digest(data))
4208
+ # written: only now does a part this save fixed stop being accepted while broken
4209
+ config.tolerated = still_accepted
4210
+ _fsync_dir(directory)
4211
+ if self._add_backup(data, prune=replaces is None) and replaces is not None:
4212
+ with contextlib.suppress(OSError):
4213
+ os.remove(os.path.join(self.backup_dir, replaces))
4214
+ if outside and current_seen is not None:
4215
+ self._replaced(current_seen, kept_as)
4216
+ return kept
4217
+
4218
+ def _replaced(self, edited: _Seen, backup: str) -> None:
4219
+ """Report that a save replaced an edit made outside ser2netweb (see the class doc)."""
4220
+ mtime = edited.key[0] / 1e9 if edited.key else 0.0
4221
+ when = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(mtime)) if mtime else "?"
4222
+ self.replaced_edit = {"edited": mtime, "saved": time.time(), "backup": backup}
4223
+ where = (os.path.join("backups", backup) if backup
4224
+ else "nowhere: its backup could not be written (see above)")
4225
+ self.log(f"WARNING: config.json had been changed outside ser2netweb ({when}) and was not "
4226
+ "reloaded; this save replaced it with the configuration in use. The edited "
4227
+ f"file is kept as {where}: restore it under Settings → Configuration backups, "
4228
+ "or edit config.json again and reload it")
4229
+ if self.on_replaced is not None:
4230
+ self.on_replaced(self.replaced_edit)
4231
+
4232
+ # ----- backups -----
4233
+ def _newest(self) -> Optional[tuple[str, str]]:
4234
+ """(name, text) of the newest backup, or None."""
4235
+ for name in self.backup_names()[:1]:
4236
+ try:
4237
+ with open(os.path.join(self.backup_dir, name), encoding="utf-8") as fh:
4238
+ return name, fh.read()
4239
+ except OSError:
4240
+ return None
4241
+ return None
4242
+
4243
+ def _add_backup(self, current: str, *, prune: bool = True) -> str:
4244
+ """Make `current` (a config.json's text) the newest backup, unless it is that
4245
+ already; its name when it is now, "" when not. Best effort: a failing backup is
4246
+ logged, and never stops the save itself. The name sorts first: when the clock is
4247
+ behind the newest backup, the new one takes that one's time with the next number."""
4248
+ try:
4249
+ os.makedirs(self.backup_dir, exist_ok=True)
4250
+ lock_down_dir(self.backup_dir)
4251
+ newest = self._newest()
4252
+ if newest is not None and newest[1] == current:
4253
+ return newest[0] # this version is already the newest backup
4254
+ clock = time.strftime("%Y%m%dT%H%M%S", time.gmtime(now()))
4255
+ stamp = clock
4256
+ if newest is not None and (top := _backup_order(newest[0])[0]) > clock:
4257
+ stamp = top
4258
+ if not self._clock_noted:
4259
+ self._clock_noted = True
4260
+ self.log(f"WARNING: the system clock ({backup_time(f'config-{clock}')} "
4261
+ f"UTC) is behind the newest configuration backup "
4262
+ f"({backup_time(newest[0])} UTC): new backups are named after "
4263
+ f"that one, so that they stay the newest")
4264
+ # several saves in one second: -2, -3 ... (never reuse a pruned lower number,
4265
+ # or the newest backup would sort as the oldest)
4266
+ same = [_backup_order(n)[1] for n in self.backup_names()
4267
+ if _backup_order(n)[0] == stamp]
4268
+ n = max(same) + 1 if same else 1
4269
+ name = f"config-{stamp}Z.json" if n == 1 else f"config-{stamp}Z-{n}.json"
4270
+ fd, tmp = tempfile.mkstemp(dir=self.backup_dir, prefix=".bak.", suffix=".tmp")
4271
+ with os.fdopen(fd, "w", encoding="utf-8") as fh:
4272
+ fh.write(current)
4273
+ fh.flush()
4274
+ os.fsync(fh.fileno())
4275
+ os.replace(tmp, os.path.join(self.backup_dir, name))
4276
+ for old in self.backup_names()[BACKUPS_KEPT:] if prune else ():
4277
+ if old != name: # (it sorts first; never the one just written)
4278
+ with contextlib.suppress(OSError):
4279
+ os.remove(os.path.join(self.backup_dir, old))
4280
+ return name
4281
+ except OSError as e:
4282
+ self.log(f"WARNING: could not keep a backup of config.json in {self.backup_dir}: "
4283
+ f"{e}")
4284
+ return ""
4285
+
4286
+ def prune_backups(self, max_age_s: float, now: float) -> list[str]:
4287
+ """Delete the backups named for a time more than max_age_s before `now` (UTC, as
4288
+ their names say: a backup taken while the clock was behind is named later, so it
4289
+ stays longer), except the newest BACKUPS_MIN_KEPT; the newest one is what
4290
+ config.json holds. Returns the names deleted. Blocking."""
4291
+ gone = []
4292
+ for name in self.backup_names()[BACKUPS_MIN_KEPT:]:
4293
+ try:
4294
+ named = calendar.timegm(time.strptime(_backup_order(name)[0], "%Y%m%dT%H%M%S"))
4295
+ except ValueError:
4296
+ continue
4297
+ if named < now - max_age_s:
4298
+ with contextlib.suppress(OSError):
4299
+ os.remove(os.path.join(self.backup_dir, name))
4300
+ gone.append(name)
4301
+ return gone
4302
+
4303
+ def backup_names(self) -> list[str]:
4304
+ """Backup file names, newest first."""
4305
+ try:
4306
+ names = [n for n in os.listdir(self.backup_dir) if _BACKUP_NAME.match(n)]
4307
+ except OSError:
4308
+ return []
4309
+ return sorted(names, key=_backup_order, reverse=True)
4310
+
4311
+ def backups(self) -> list[dict]:
4312
+ """[{name, time (UTC, 'YYYY-MM-DD HH:MM:SS'), size, mappings, users, accounts:
4313
+ [{username, role, source}], written_by (the release that wrote it; None: 2.8.1 or
4314
+ older), current (it is what config.json holds now), file_time (when the file was
4315
+ written, UTC, if that is more than a minute before the time in its name: the clock
4316
+ was behind the newest backup then, so it is named after that one; else "")}],
4317
+ newest first.
4318
+
4319
+ The Settings page lists them at every visit: a file's contents are read once and
4320
+ kept by its name, size and modification time (a backup is written once; one that
4321
+ remove_account rewrites gets a new modification time), so only a new or changed one
4322
+ is parsed again. In memory only: the files and their names stay as they are."""
4323
+ try:
4324
+ with open(self.path, "rb") as fh:
4325
+ current: Optional[str] = hashlib.sha256(fh.read()).hexdigest()
4326
+ except OSError:
4327
+ current = None
4328
+ known, kept, out = self._backup_summaries, {}, []
4329
+ for name in self.backup_names():
4330
+ path = os.path.join(self.backup_dir, name)
4331
+ info: dict[str, Any] = {"name": name, "size": 0, "mappings": None, "users": None,
4332
+ "accounts": [], "time": backup_time(name),
4333
+ "written_by": None, "current": False, "file_time": ""}
4334
+ try:
4335
+ st = os.stat(path)
4336
+ except OSError:
4337
+ out.append(info)
4338
+ continue
4339
+ stamp = (st.st_size, st.st_mtime_ns)
4340
+ hit = known.get(name)
4341
+ summary = hit[1] if hit is not None and hit[0] == stamp else _backup_summary(path)
4342
+ if summary is not None:
4343
+ kept[name] = (stamp, summary)
4344
+ info.update({k: v for k, v in summary.items() if k != "digest"})
4345
+ info["current"] = current is not None and summary["digest"] == current
4346
+ info["size"] = st.st_size
4347
+ try:
4348
+ named = calendar.timegm(time.strptime(_backup_order(name)[0], "%Y%m%dT%H%M%S"))
4349
+ except ValueError:
4350
+ named = 0
4351
+ if named - st.st_mtime > 60: # (a copy of data/ makes it later: not shown)
4352
+ info["file_time"] = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(st.st_mtime))
4353
+ out.append(info)
4354
+ self._backup_summaries = kept # pruned to the files there are
4355
+ return out
4356
+
4357
+ def _known_backup(self, name: str) -> str:
4358
+ """The path of the backup `name`; ConfigError for an unknown / invalid name."""
4359
+ if not _BACKUP_NAME.match(name or "") or name not in self.backup_names():
4360
+ raise ConfigError(_("No such backup."))
4361
+ return os.path.join(self.backup_dir, name)
4362
+
4363
+ def delete_backup(self, name: str) -> None:
4364
+ """Delete one backup (ConfigError for an unknown name)."""
4365
+ os.remove(self._known_backup(name))
4366
+
4367
+ def delete_backups(self) -> int:
4368
+ """Delete every backup; how many were deleted."""
4369
+ gone = 0
4370
+ for name in self.backup_names():
4371
+ with contextlib.suppress(FileNotFoundError):
4372
+ os.remove(os.path.join(self.backup_dir, name))
4373
+ gone += 1
4374
+ return gone
4375
+
4376
+ def remove_account(self, username: str) -> int:
4377
+ """Take the account named exactly `username` out of every backup that has it (in
4378
+ a backup from before accounts existed, the top-level password_hash is the 'admin'
4379
+ account), rewriting each one atomically under its name with everything else as
4380
+ it was: restoring those backups can't bring the account back. Returns how many
4381
+ backups were changed."""
4382
+ changed = 0
4383
+ for name in self.backup_names():
4384
+ path = os.path.join(self.backup_dir, name)
4385
+ try:
4386
+ with open(path, encoding="utf-8") as fh:
4387
+ doc = json.load(fh)
4388
+ except (OSError, ValueError):
4389
+ continue
4390
+ if isinstance(doc, dict) and _drop_backup_account(doc, username):
4391
+ _replace_file(self.backup_dir, path, json.dumps(doc, indent=2, ensure_ascii=False))
4392
+ changed += 1
4393
+ return changed
4394
+
4395
+ def read_backup(self, name: str) -> str:
4396
+ """The text of one backup. Raises ConfigError for an unknown / invalid name
4397
+ (only names this store generated are accepted: no path tricks)."""
4398
+ with open(self._known_backup(name), encoding="utf-8") as fh:
4399
+ return fh.read()
4400
+
4401
+
4402
+ def _backup_summary(path: str) -> dict[str, Any] | None:
4403
+ """What the backups list shows of one backup's contents: mappings, users, accounts and
4404
+ the release that wrote it (no counts and no accounts for a file that isn't a
4405
+ configuration; None itself when it can't be read now), and the digest of its text (is
4406
+ it what config.json holds: ConfigStore.backups)."""
4407
+ try:
4408
+ with open(path, "rb") as fh:
4409
+ data = fh.read()
4410
+ except OSError:
4411
+ return None
4412
+ empty: dict[str, Any] = {"mappings": None, "users": None, "accounts": [], "written_by": None,
4413
+ "digest": hashlib.sha256(data).hexdigest()}
4414
+ try:
4415
+ doc = json.loads(data.decode("utf-8"))
4416
+ except ValueError:
4417
+ return empty
4418
+ try:
4419
+ accounts = _backup_accounts(doc)
4420
+ writer = doc.get("written_by")
4421
+ return {**empty, "mappings": len(doc.get("mappings") or []), "users": len(accounts),
4422
+ "accounts": accounts,
4423
+ "written_by": writer[:40] if isinstance(writer, str) else None}
4424
+ except (AttributeError, TypeError):
4425
+ return empty
4426
+
4427
+
4428
+ def _backup_accounts(doc: dict) -> list[dict]:
4429
+ """The accounts in a backup's JSON: [{username, role, source}] (a config from before
4430
+ accounts existed has one, 'admin', as its top-level password_hash)."""
4431
+ users = doc.get("users")
4432
+ if isinstance(users, list) and users:
4433
+ return [{"username": str(u.get("username", "")), "role": str(u.get("role", "")),
4434
+ "source": str(u.get("source") or "local")} for u in users if isinstance(u, dict)]
4435
+ if doc.get("password_hash"):
4436
+ return [{"username": "admin", "role": "admin", "source": "local"}]
4437
+ return []
4438
+
4439
+
4440
+ def _drop_backup_account(doc: dict, username: str) -> bool:
4441
+ """Take the account named `username` out of a backup's JSON; True if it was there."""
4442
+ users = doc.get("users")
4443
+ if isinstance(users, list) and users:
4444
+ kept = [u for u in users if not (isinstance(u, dict) and u.get("username") == username)]
4445
+ if len(kept) == len(users):
4446
+ return False
4447
+ doc["users"] = kept
4448
+ return True
4449
+ if username == "admin" and doc.get("password_hash"):
4450
+ doc.pop("password_hash", None)
4451
+ doc.pop("pwd_version", None)
4452
+ return True
4453
+ return False
4454
+
4455
+
4456
+ def _replace_file(directory: str, path: str, text: str) -> None:
4457
+ """Write `text` to `path` atomically (a temp file in `directory`, fsync, rename)."""
4458
+ fd, tmp = tempfile.mkstemp(dir=directory, prefix=".bak.", suffix=".tmp")
4459
+ try:
4460
+ with os.fdopen(fd, "w", encoding="utf-8") as fh:
4461
+ fh.write(text)
4462
+ fh.flush()
4463
+ os.fsync(fh.fileno())
4464
+ os.replace(tmp, path)
4465
+ finally:
4466
+ if os.path.exists(tmp):
4467
+ os.unlink(tmp)
4468
+
4469
+
4470
+ def backup_time(name: str) -> str:
4471
+ """'config-20260101T120000Z.json' -> '2026-01-01 12:00:00' (UTC)."""
4472
+ stamp = name[len("config-"):len("config-") + 15]
4473
+ return (f"{stamp[0:4]}-{stamp[4:6]}-{stamp[6:8]} "
4474
+ f"{stamp[9:11]}:{stamp[11:13]}:{stamp[13:15]}")
4475
+
4476
+
4477
+ def _backup_order(name: str) -> tuple:
4478
+ m = re.match(r"^config-(\d{8}T\d{6})Z(?:-(\d+))?\.json$", name)
4479
+ return (m.group(1), int(m.group(2) or 1)) if m else ("", 0)
4480
+
4481
+
4482
+ def _fsync_dir(directory: str) -> None:
4483
+ """fsync a directory so a rename into it is durable (not on Windows)."""
4484
+ try:
4485
+ dfd = os.open(directory, os.O_RDONLY)
4486
+ try:
4487
+ os.fsync(dfd)
4488
+ finally:
4489
+ os.close(dfd)
4490
+ except (OSError, AttributeError):
4491
+ pass # not supported on some platforms (e.g. Windows dirs)