ser2netweb 3.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. ser2netweb/__init__.py +20 -0
  2. ser2netweb/__main__.py +6 -0
  3. ser2netweb/accounts.py +503 -0
  4. ser2netweb/cli.py +489 -0
  5. ser2netweb/clock.py +128 -0
  6. ser2netweb/config.py +4491 -0
  7. ser2netweb/console.py +181 -0
  8. ser2netweb/datalock.py +181 -0
  9. ser2netweb/engine/__init__.py +0 -0
  10. ser2netweb/engine/access.py +173 -0
  11. ser2netweb/engine/bindhint.py +115 -0
  12. ser2netweb/engine/bridge.py +1928 -0
  13. ser2netweb/engine/cluster.py +1140 -0
  14. ser2netweb/engine/modbus.py +530 -0
  15. ser2netweb/engine/modbus_gateway.py +1465 -0
  16. ser2netweb/engine/mqtt_pub.py +632 -0
  17. ser2netweb/engine/netguard.py +805 -0
  18. ser2netweb/engine/netinfo.py +199 -0
  19. ser2netweb/engine/portdiag.py +414 -0
  20. ser2netweb/engine/portlist.py +471 -0
  21. ser2netweb/engine/protocols/__init__.py +28 -0
  22. ser2netweb/engine/protocols/base.py +117 -0
  23. ser2netweb/engine/protocols/raw.py +12 -0
  24. ser2netweb/engine/protocols/rfc2217.py +586 -0
  25. ser2netweb/engine/protocols/telnet.py +172 -0
  26. ser2netweb/engine/serial_compat.py +598 -0
  27. ser2netweb/engine/serial_io.py +210 -0
  28. ser2netweb/engine/serial_link.py +414 -0
  29. ser2netweb/engine/sockopts.py +103 -0
  30. ser2netweb/engine/sparkplug.py +538 -0
  31. ser2netweb/engine/sparkplug_node.py +550 -0
  32. ser2netweb/engine/supervisor.py +533 -0
  33. ser2netweb/engine/tls.py +232 -0
  34. ser2netweb/envvars.py +57 -0
  35. ser2netweb/explain.py +117 -0
  36. ser2netweb/healthcheck.py +78 -0
  37. ser2netweb/i18n.py +110 -0
  38. ser2netweb/importer.py +550 -0
  39. ser2netweb/layout.py +118 -0
  40. ser2netweb/licences.py +86 -0
  41. ser2netweb/locale/tr.json +1569 -0
  42. ser2netweb/logsetup.py +253 -0
  43. ser2netweb/logtext.py +23 -0
  44. ser2netweb/notify.py +761 -0
  45. ser2netweb/optional.py +76 -0
  46. ser2netweb/reload.py +295 -0
  47. ser2netweb/runtime.py +762 -0
  48. ser2netweb/sdnotify.py +85 -0
  49. ser2netweb/ser2net_import.py +2001 -0
  50. ser2netweb/signins.py +98 -0
  51. ser2netweb/standalone.py +154 -0
  52. ser2netweb/state.py +1457 -0
  53. ser2netweb/storage.py +130 -0
  54. ser2netweb/textfold.py +51 -0
  55. ser2netweb/tlsreload.py +37 -0
  56. ser2netweb/watchdog.py +201 -0
  57. ser2netweb/web/__init__.py +0 -0
  58. ser2netweb/web/api.py +793 -0
  59. ser2netweb/web/auth.py +699 -0
  60. ser2netweb/web/certinfo.py +144 -0
  61. ser2netweb/web/httpguard.py +413 -0
  62. ser2netweb/web/ldap_auth.py +556 -0
  63. ser2netweb/web/oidc_auth.py +330 -0
  64. ser2netweb/web/proxy.py +398 -0
  65. ser2netweb/web/routes/__init__.py +78 -0
  66. ser2netweb/web/routes/cluster.py +669 -0
  67. ser2netweb/web/routes/common.py +359 -0
  68. ser2netweb/web/routes/console.py +451 -0
  69. ser2netweb/web/routes/forms.py +853 -0
  70. ser2netweb/web/routes/login.py +573 -0
  71. ser2netweb/web/routes/mappings.py +470 -0
  72. ser2netweb/web/routes/ops.py +374 -0
  73. ser2netweb/web/routes/settings.py +2135 -0
  74. ser2netweb/web/routes/system.py +308 -0
  75. ser2netweb/web/routes/totp.py +282 -0
  76. ser2netweb/web/selfsigned.py +129 -0
  77. ser2netweb/web/server.py +483 -0
  78. ser2netweb/web/static/app.css +327 -0
  79. ser2netweb/web/static/app.js +1102 -0
  80. ser2netweb/web/static/htmx.min.js +1 -0
  81. ser2netweb/web/static/points.js +218 -0
  82. ser2netweb/web/static/theme.js +7 -0
  83. ser2netweb/web/static/xterm-addon-fit.js +2 -0
  84. ser2netweb/web/static/xterm.css +218 -0
  85. ser2netweb/web/static/xterm.js +2 -0
  86. ser2netweb/web/templates/_cluster_body.html +109 -0
  87. ser2netweb/web/templates/_console.html +21 -0
  88. ser2netweb/web/templates/_footer.html +13 -0
  89. ser2netweb/web/templates/_login_notice.html +8 -0
  90. ser2netweb/web/templates/_macros.html +21 -0
  91. ser2netweb/web/templates/_mapping_form.html +477 -0
  92. ser2netweb/web/templates/_mapping_log.html +28 -0
  93. ser2netweb/web/templates/_mapping_traces.html +44 -0
  94. ser2netweb/web/templates/_mappings_body.html +149 -0
  95. ser2netweb/web/templates/_modbus_read.html +18 -0
  96. ser2netweb/web/templates/_ports_table.html +37 -0
  97. ser2netweb/web/templates/_serial_port_select.html +48 -0
  98. ser2netweb/web/templates/_shell.html +41 -0
  99. ser2netweb/web/templates/audit.html +55 -0
  100. ser2netweb/web/templates/base.html +77 -0
  101. ser2netweb/web/templates/dashboard.html +72 -0
  102. ser2netweb/web/templates/licenses.html +18 -0
  103. ser2netweb/web/templates/login.html +32 -0
  104. ser2netweb/web/templates/login_totp.html +23 -0
  105. ser2netweb/web/templates/settings.html +830 -0
  106. ser2netweb/web/templates/setup.html +43 -0
  107. ser2netweb/web/templates/totp.html +41 -0
  108. ser2netweb/web/totp.py +94 -0
  109. ser2netweb/winservice.py +1793 -0
  110. ser2netweb-3.0.0.dist-info/METADATA +120 -0
  111. ser2netweb-3.0.0.dist-info/RECORD +121 -0
  112. ser2netweb-3.0.0.dist-info/WHEEL +5 -0
  113. ser2netweb-3.0.0.dist-info/entry_points.txt +2 -0
  114. ser2netweb-3.0.0.dist-info/licenses/COMMERCIAL_LICENSE.md +94 -0
  115. ser2netweb-3.0.0.dist-info/licenses/LICENSE +662 -0
  116. ser2netweb-3.0.0.dist-info/licenses/NOTICE +20 -0
  117. ser2netweb-3.0.0.dist-info/licenses/THIRD-PARTY-NOTICES.md +105 -0
  118. ser2netweb-3.0.0.dist-info/licenses/tools/licenses/npm/htmx.org.txt +13 -0
  119. ser2netweb-3.0.0.dist-info/licenses/tools/licenses/npm/xterm-addon-fit.txt +19 -0
  120. ser2netweb-3.0.0.dist-info/licenses/tools/licenses/npm/xterm.txt +21 -0
  121. ser2netweb-3.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,805 @@
1
+ """The process's open files: their limit, how many are in use, and what the mappings need.
2
+
3
+ Every serial port, listener, client connection, log and trace file and MQTT connection
4
+ holds a file descriptor, and a process may have at most its soft RLIMIT_NOFILE open at
5
+ once: 1024 under systemd unless the unit raises it. Past it, whatever opens the next
6
+ file fails, wherever that is (a serial reopen, a client, a config save). So:
7
+ - ``raise_fd_limit`` lifts the soft limit toward the hard one at start-up (at most
8
+ FD_LIMIT_TARGET), with prlimit() on Linux: the shipped unit's system call filter
9
+ denies setrlimit() but allows prlimit64. Nothing in the process waits with select()
10
+ any more (serial_compat, mqtt_pub.PollSelect), so descriptors above 1024 work;
11
+ - ``fd_budget_warning`` says when the configured mappings need about as many as the
12
+ limit allows;
13
+ - ``open_fds`` / ``fd_limit`` are what /metrics reports.
14
+ Windows has no such limit for sockets and serial ports (they are handles): each of these
15
+ answers None or "" there.
16
+
17
+ Incoming connections pass the process's budget (``BUDGET``) right after accept, whatever
18
+ listener they reach, before they cost anything more:
19
+ - ``Budget.headroom``: a connection is refused (closed, with a log line at most once a
20
+ minute) while the process is within ``fd_reserve`` of its open-file limit, so clients
21
+ can't take the files that serial ports, logs and the configuration need;
22
+ - TLS handshakes in progress hold a socket and a 256 KiB buffer each: at most
23
+ TLS_HANDSHAKES_MAX at a time, TLS_HANDSHAKES_PER_IP from one address (an IPv6 one
24
+ counted by its /64, see address_key, as the admin UI's per-address cap), each given
25
+ TLS_HANDSHAKE_TIMEOUT_S (asyncio's default is 60 s). A connection past those waits
26
+ for a handshake to end (first come, first served; reading paused, so it holds only
27
+ its socket) for up to TLS_HANDSHAKE_TIMEOUT_S, at most TLS_WAITING_PER_IP from one
28
+ address and TLS_WAITING_MAX in all: a client's burst of reconnects is served, not
29
+ refused. Past those too, a connection is refused.
30
+ - Windows' SelectorEventLoop (``CappedSelectSelector``): select() takes at most 512
31
+ sockets there, so connections are refused while it watches nearly that many.
32
+ ``start_server`` is asyncio.start_server for a mapping's listener with that admission. For
33
+ a TLS listener it also applies the mapping's allow-list before the handshake (a refused
34
+ host never gets one) and does the handshake itself (``start_tls``): the mapping sees the
35
+ connection only once it is done, and decides about capacity, kicking and "Busy" then.
36
+ When the listener can't take its address it raises ``BindError``; ``udp_socket`` makes a
37
+ UDP mapping's socket, with the same error. Nor can it take one that overlaps a socket
38
+ ``reserve`` holds (the admin UI's): Windows lets a listener on a wildcard address share a
39
+ port with one on a specific address, where Linux refuses the second bind.
40
+
41
+ Address families: a listener on :: takes IPv6 clients only, on every OS (IPV6_V6ONLY: asyncio
42
+ sets it for TCP, ``udp_socket`` for UDP); one that takes both (network.dual_stack on a
43
+ wildcard) has two TCP sockets, 0.0.0.0 and :: (``listen_hosts``), so its IPv4 clients have
44
+ plain IPv4 addresses, or one UDP socket on :: that takes IPv4 senders too.
45
+ """
46
+ from __future__ import annotations
47
+
48
+ import asyncio
49
+ import errno
50
+ import ipaddress
51
+ import os
52
+ import selectors
53
+ import socket
54
+ import sys
55
+ import time
56
+ import weakref
57
+ from typing import TYPE_CHECKING, Any, Callable, Optional, Sequence, Union
58
+
59
+ from ..config import WILDCARDS, canonical_ip, host_port
60
+
61
+ if TYPE_CHECKING:
62
+ from ..config import NetworkSettings
63
+
64
+ # the soft open-file limit raise_fd_limit aims for: far above what a few hundred
65
+ # mappings need, without handing child processes a huge limit
66
+ FD_LIMIT_TARGET = 65536
67
+ # what a process holds open before any mapping: stdio, all.log, the event loop, the admin
68
+ # listener and its connections, the executor's threads' files
69
+ FD_BASE = 40
70
+
71
+ # TLS on the listeners: the handshake must be over in this time; closing a TLS connection
72
+ # waits this long for the peer's close_notify (asyncio's ssl_shutdown_timeout from Python
73
+ # 3.11; on 3.10, _bound_close)
74
+ TLS_HANDSHAKE_TIMEOUT_S = 15.0
75
+ TLS_SHUTDOWN_TIMEOUT_S = 5.0
76
+ # TLS handshakes in progress at once: in the process, and from one source address
77
+ TLS_HANDSHAKES_MAX = 128
78
+ TLS_HANDSHAKES_PER_IP = 32
79
+ # connections waiting for one of those to end (each for at most TLS_HANDSHAKE_TIMEOUT_S):
80
+ # in the process, and from one source address
81
+ TLS_WAITING_MAX = 256
82
+ TLS_WAITING_PER_IP = 64
83
+ # Windows' select() takes at most this many sockets in each of its sets (FD_SETSIZE);
84
+ # incoming connections are refused once the event loop watches all but SELECT_HEADROOM
85
+ # of them, which stay for listeners, connections out and UDP sockets
86
+ SELECT_LIMIT = 512
87
+ SELECT_HEADROOM = 64
88
+
89
+
90
+ class BindError(OSError):
91
+ """A listener could not take its address and port: another socket holds it, or the
92
+ address is not on this machine (yet: DHCP, a NIC or a tentative IPv6 address that comes
93
+ up later). Worth another try later (the supervisor makes them), whatever form the error
94
+ came in: asyncio on Python 3.11+ reports a missing address as "could not bind on any
95
+ address", with no errno."""
96
+
97
+ @classmethod
98
+ def of(cls, e: OSError) -> BindError:
99
+ err = cls(str(e))
100
+ err.errno = e.errno
101
+ return err
102
+
103
+
104
+ class LogEvery:
105
+ """A log line that can repeat many times a second (dropped data, a UDP peer that keeps
106
+ changing, refused connections): written at most once per ``interval`` seconds, saying
107
+ how many were held back since the last one."""
108
+
109
+ def __init__(self, log: Callable[[str], None], interval: float = 60.0) -> None:
110
+ self._log = log
111
+ self._interval = interval
112
+ self._next = 0.0
113
+ self._held = 0
114
+
115
+ def __call__(self, msg: str) -> None:
116
+ now = time.monotonic()
117
+ if now < self._next:
118
+ self._held += 1
119
+ return
120
+ if self._held:
121
+ msg += f" ({self._held} more since the last such line)"
122
+ self._held = 0
123
+ self._next = now + self._interval
124
+ self._log(msg)
125
+
126
+
127
+ class RefusalLog:
128
+ """The log lines of refused connections ("rejected <peer>: <why>"), which a client
129
+ that retries in a loop, or a scan, can cause many times a second: each reason at most
130
+ once per ``interval`` seconds (LogEvery), the next one saying how many were held back.
131
+ Other lines (a kick) are logged as they are."""
132
+
133
+ MAX_REASONS = 32 # (the reasons are fixed texts; past that, they share one line)
134
+
135
+ def __init__(self, log: Callable[[str], None], interval: float = 60.0) -> None:
136
+ self._log = log
137
+ self._interval = interval
138
+ self._by_reason: dict[str, LogEvery] = {}
139
+
140
+ def __call__(self, msg: str) -> None:
141
+ if not msg.startswith("rejected "):
142
+ self._log(msg)
143
+ return
144
+ why = msg.partition(": ")[2]
145
+ if why not in self._by_reason and len(self._by_reason) >= self.MAX_REASONS:
146
+ why = ""
147
+ limiter = self._by_reason.get(why)
148
+ if limiter is None:
149
+ limiter = self._by_reason[why] = LogEvery(self._log, self._interval)
150
+ limiter(msg)
151
+
152
+
153
+ def _resource() -> Any:
154
+ try:
155
+ import resource
156
+ except ImportError: # Windows
157
+ return None
158
+ return resource
159
+
160
+
161
+ def fd_limit() -> Optional[int]:
162
+ """The soft open-file limit (None: unlimited, or Windows)."""
163
+ resource = _resource()
164
+ if resource is None:
165
+ return None
166
+ soft, _hard = resource.getrlimit(resource.RLIMIT_NOFILE)
167
+ return None if soft == resource.RLIM_INFINITY else int(soft)
168
+
169
+
170
+ def open_fds() -> Optional[int]:
171
+ """How many files this process has open (None where that can't be counted cheaply)."""
172
+ if os.name != "posix":
173
+ return None
174
+ for folder in ("/proc/self/fd", "/dev/fd"):
175
+ try:
176
+ return len(os.listdir(folder)) - 1 # less the one listdir() itself opened
177
+ except OSError:
178
+ continue
179
+ return None
180
+
181
+
182
+ def _shown(value: int, resource: Any) -> str:
183
+ return "unlimited" if value == resource.RLIM_INFINITY else str(value)
184
+
185
+
186
+ def raise_fd_limit(target: int = FD_LIMIT_TARGET) -> str:
187
+ """Raise the soft open-file limit toward the hard limit, at most to ``target`` (never
188
+ lower it). Returns the line for the start-up log ("" on Windows)."""
189
+ resource = _resource()
190
+ if resource is None:
191
+ return ""
192
+ soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
193
+ want = target if hard == resource.RLIM_INFINITY else min(hard, target)
194
+ if soft == resource.RLIM_INFINITY or soft >= want:
195
+ return f"open files: limit {_shown(soft, resource)}"
196
+ try:
197
+ if hasattr(resource, "prlimit"): # Linux: prlimit64, never the setrlimit() call
198
+ resource.prlimit(0, resource.RLIMIT_NOFILE, (want, hard))
199
+ else:
200
+ resource.setrlimit(resource.RLIMIT_NOFILE, (want, hard))
201
+ except (OSError, ValueError) as e:
202
+ return f"open files: limit {soft} (raising it to {want} failed: {e})"
203
+ return f"open files: limit raised from {soft} to {want} (hard limit {_shown(hard, resource)})"
204
+
205
+
206
+ def fd_estimate(config: Any) -> int:
207
+ """About how many files the configured mappings hold open: per mapping its log file
208
+ (disabled ones too), and per enabled one its serial port(s), its socket (a listener, a
209
+ UDP socket or the connection out), each client it admits, its trace file and an MQTT
210
+ connection (its socket and paho's wake-up pair)."""
211
+ need = FD_BASE
212
+ for m in config.mappings:
213
+ need += 1
214
+ if not m.enabled:
215
+ continue
216
+ need += len(m.serial_sides())
217
+ need += 1 if m.options.trace_both else 0
218
+ need += 3 if m.mqtt.enabled else 0
219
+ if m.kind == "net":
220
+ need += 1 + (m.network.max_connections if m.network.mode == "server" else 0)
221
+ return need
222
+
223
+
224
+ def fd_budget_warning(config: Any, limit: Optional[int] = None) -> str:
225
+ """A start-up warning when the mappings need over 80% of the open-file limit ("" when
226
+ they fit, or there is no such limit)."""
227
+ limit = fd_limit() if limit is None else limit
228
+ if limit is None:
229
+ return ""
230
+ need = fd_estimate(config)
231
+ if need <= limit * 0.8:
232
+ return ""
233
+ return (f"WARNING: the {len(config.mappings)} mappings need about {need} open files, "
234
+ f"close to this process's limit of {limit}: beyond it serial ports, clients and "
235
+ "files fail to open. Raise the limit (LimitNOFILE= in the systemd unit, "
236
+ "--ulimit nofile= for Docker) or split the mappings over more instances.")
237
+
238
+
239
+ def fd_reserve(limit: int) -> int:
240
+ """Open files kept free for what is not a client (serial ports, logs, the configuration
241
+ and its backups, traces, MQTT): incoming connections are refused within this many of
242
+ the limit."""
243
+ return min(256, max(32, limit // 8))
244
+
245
+
246
+ # ---------------------------------------------------------------------------
247
+ # incoming connections
248
+ # ---------------------------------------------------------------------------
249
+ def peer_address(transport: Any) -> tuple[str, int]:
250
+ """A connection's (a stream writer's) source address and port: the address in its
251
+ canonical form, an IPv4-mapped IPv6 one (an IPv4 client of a socket that takes both
252
+ families) as its IPv4 address; port 0 when there is none."""
253
+ peer = transport.get_extra_info("peername")
254
+ if isinstance(peer, tuple) and peer:
255
+ port = peer[1] if len(peer) > 1 and isinstance(peer[1], int) else 0
256
+ return canonical_ip(str(peer[0])), port
257
+ return str(peer or "?"), 0
258
+
259
+
260
+ def peer_ip(transport: Any) -> str:
261
+ """A connection's source address (see peer_address)."""
262
+ return peer_address(transport)[0]
263
+
264
+
265
+ def client_label(ip: str, port: int) -> str:
266
+ """ip:port of a client, for the log and the UI ([2001:db8::7]:50123), just the address
267
+ without a port."""
268
+ return host_port(ip, port) if port else ip
269
+
270
+
271
+ def peer_label(transport: Any) -> str:
272
+ """ip:port of a connection, for the log."""
273
+ return client_label(*peer_address(transport))
274
+
275
+
276
+ def address_key(ip: str) -> str:
277
+ """What the per-address limits (the connection caps here, the web UI's sign-in limits
278
+ in web/auth.py) count a source under: an IPv6 address by its /64 network (a host, or a
279
+ LAN, gets a whole /64 and can use any address in it, so one client can't get past a
280
+ limit by moving to the next address), an IPv4 address (also written IPv4-mapped) as it
281
+ is, anything else unchanged."""
282
+ try:
283
+ addr = ipaddress.ip_address(ip)
284
+ except ValueError:
285
+ return ip
286
+ if isinstance(addr, ipaddress.IPv6Address):
287
+ if addr.ipv4_mapped is not None:
288
+ return str(addr.ipv4_mapped)
289
+ return str(ipaddress.IPv6Network((int(addr) >> 64 << 64, 64)))
290
+ return str(addr)
291
+
292
+
293
+ class Pool:
294
+ """Connections (their transports) counted in total and per source address (an IPv6 one
295
+ by its /64: address_key), each up to a cap (0: none). A connection whose transport is
296
+ closing no longer counts."""
297
+
298
+ def __init__(self, total: int = 0, per_ip: int = 0) -> None:
299
+ self.total = total
300
+ self.per_ip = per_ip
301
+ self._members: dict[Any, str] = {} # transport -> address_key of its source
302
+
303
+ def _sweep(self) -> None:
304
+ for t in [t for t in self._members if t.is_closing()]:
305
+ del self._members[t]
306
+
307
+ def __len__(self) -> int:
308
+ self._sweep()
309
+ return len(self._members)
310
+
311
+ def count(self, ip: str) -> int:
312
+ self._sweep()
313
+ key = address_key(ip)
314
+ return sum(1 for member in self._members.values() if member == key)
315
+
316
+ def full(self, ip: str) -> str:
317
+ """"" when one more from ``ip`` fits, else why not."""
318
+ if self.total and len(self) >= self.total:
319
+ return f"{self.total} at once is the limit"
320
+ if self.per_ip and self.count(ip) >= self.per_ip:
321
+ key = address_key(ip)
322
+ source = f"one /64 network ({key})" if key.endswith("/64") else "one address"
323
+ return f"{self.per_ip} at once from {source} is the limit"
324
+ return ""
325
+
326
+ def add(self, transport: Any, ip: str) -> None:
327
+ self._members[transport] = address_key(ip)
328
+
329
+ def discard(self, transport: Any) -> None:
330
+ self._members.pop(transport, None)
331
+
332
+ def members(self) -> list:
333
+ self._sweep()
334
+ return list(self._members)
335
+
336
+
337
+ class Budget:
338
+ """Whether the process takes one more incoming connection, whichever listener it reaches
339
+ (one instance: ``BUDGET``), and the TLS handshakes in progress and waiting."""
340
+
341
+ def __init__(self) -> None:
342
+ self.handshakes = Pool(TLS_HANDSHAKES_MAX, TLS_HANDSHAKES_PER_IP)
343
+ self.waiting = Pool(TLS_WAITING_MAX, TLS_WAITING_PER_IP) # for a handshake to end
344
+ # each waiting connection -> (its address, its turn: True once it has a slot,
345
+ # False when it closed meanwhile); in the order they came
346
+ self._turns: dict[Any, tuple[str, asyncio.Future]] = {}
347
+ self.refused: dict[str, int] = {} # why -> connections refused for it (/metrics)
348
+ # the event loop's selector when it is select() based (Windows; runtime.py sets it)
349
+ self.selector: Optional[CappedSelectSelector] = None
350
+ self._counted = (-1.0, 0) # (when, how many files were open)
351
+ self._admitted = 0 # connections let in since
352
+
353
+ def _files_in_use(self) -> Optional[int]:
354
+ now = time.monotonic()
355
+ if now - self._counted[0] > 0.25: # at most four counts a second, under a flood too
356
+ n = open_fds()
357
+ if n is None:
358
+ return None
359
+ self._counted, self._admitted = (now, n), 0
360
+ return self._counted[1] + self._admitted
361
+
362
+ def headroom(self) -> str:
363
+ """"" when one more connection fits, else why not (for the log)."""
364
+ limit = fd_limit()
365
+ if limit is not None:
366
+ used = self._files_in_use()
367
+ if used is not None and used >= limit - fd_reserve(limit):
368
+ self.note_refused("open_files")
369
+ return (f"{used} of this process's {limit} open files are in use (the rest "
370
+ "is kept for serial ports, logs and the configuration)")
371
+ sel = self.selector
372
+ if sel is not None:
373
+ watched = sel.watched()
374
+ if watched >= sel.limit - SELECT_HEADROOM:
375
+ self.note_refused("event_loop")
376
+ return (f"the event loop watches {watched} sockets, near the {sel.limit} its "
377
+ "select() takes")
378
+ self._admitted += 1
379
+ return ""
380
+
381
+ def handshake_room(self, ip: str) -> str:
382
+ """"" when a TLS handshake from ``ip`` can start, or wait for one to end, else why
383
+ not (for the log)."""
384
+ why = self.handshakes.full(ip)
385
+ if why:
386
+ waiting = self.waiting.full(ip)
387
+ if waiting:
388
+ self.note_refused("tls_handshakes")
389
+ return (f"too many TLS handshakes in progress ({why}) and waiting for one "
390
+ f"to end ({waiting})")
391
+ return ""
392
+
393
+ def enter_handshake(self, transport: Any, ip: str) -> None:
394
+ """Count an admitted TLS connection among the handshakes in progress, or, when
395
+ there is no room for it, among those waiting for one to end."""
396
+ if self.handshakes.full(ip):
397
+ self.waiting.add(transport, ip)
398
+ self._turns[transport] = (ip, asyncio.get_running_loop().create_future())
399
+ else:
400
+ self.handshakes.add(transport, ip)
401
+
402
+ async def handshake_turn(self, transport: Any) -> bool:
403
+ """Wait (up to TLS_HANDSHAKE_TIMEOUT_S) until ``transport`` may start its handshake:
404
+ False when no handshake ended for it in that time, or it closed meanwhile."""
405
+ if transport not in self._turns:
406
+ return True # it had room at once
407
+ turn = self._turns[transport][1]
408
+ try:
409
+ await asyncio.wait((turn,), timeout=TLS_HANDSHAKE_TIMEOUT_S)
410
+ finally:
411
+ self._turns.pop(transport, None)
412
+ self.waiting.discard(transport)
413
+ return turn.done() and turn.result()
414
+
415
+ def handshake_done(self, transport: Any) -> None:
416
+ """A handshake is over (or never started): the connections waiting for one get
417
+ the room there is now, first come, first served."""
418
+ self.handshakes.discard(transport)
419
+ self.waiting.discard(transport)
420
+ for waiter, (ip, turn) in list(self._turns.items()):
421
+ if turn.done():
422
+ continue
423
+ if waiter.is_closing():
424
+ turn.set_result(False)
425
+ elif not self.handshakes.full(ip):
426
+ self.waiting.discard(waiter)
427
+ self.handshakes.add(waiter, ip)
428
+ turn.set_result(True)
429
+ elif self.handshakes.total and len(self.handshakes) >= self.handshakes.total:
430
+ break # no room for anyone
431
+
432
+ def note_refused(self, why: str) -> None:
433
+ self.refused[why] = self.refused.get(why, 0) + 1
434
+
435
+
436
+ BUDGET = Budget()
437
+ # why a connection can be refused right after accept (Budget.refused keys, for /metrics)
438
+ REFUSAL_REASONS = ("open_files", "event_loop", "tls_handshakes", "admin_ui")
439
+
440
+
441
+ class CappedSelectSelector(selectors.SelectSelector):
442
+ """The selector of the SelectorEventLoop on Windows. select() takes at most 512 sockets
443
+ in each set there and raises ValueError past that, inside the event loop, which then
444
+ stopped: every bridge with it. This one refuses the registration that would go past
445
+ the limit instead (an OSError for whatever asked for the socket to be watched), and
446
+ tells Budget.headroom how full it is, so incoming connections are refused before that
447
+ (SELECT_HEADROOM)."""
448
+
449
+ def __init__(self, limit: int = SELECT_LIMIT) -> None:
450
+ super().__init__()
451
+ self.limit = limit
452
+
453
+ def _count(self, event: int) -> int:
454
+ return sum(1 for key in self.get_map().values() if key.events & event)
455
+
456
+ def watched(self) -> int:
457
+ """Sockets in the fuller of the two sets (read, write)."""
458
+ return max(self._count(selectors.EVENT_READ), self._count(selectors.EVENT_WRITE))
459
+
460
+ def _room(self, fileobj: Any, events: int) -> None:
461
+ fd = fileobj if isinstance(fileobj, int) else fileobj.fileno()
462
+ key = self.get_map().get(fd)
463
+ have = key.events if key is not None else 0
464
+ for event in (selectors.EVENT_READ, selectors.EVENT_WRITE):
465
+ if events & event and not have & event and self._count(event) >= self.limit:
466
+ raise OSError(errno.EMFILE, f"the event loop already watches {self.limit} "
467
+ "sockets, the most its select() takes")
468
+
469
+ def register(self, fileobj: Any, events: int, data: Any = None) -> selectors.SelectorKey:
470
+ self._room(fileobj, events)
471
+ return super().register(fileobj, events, data)
472
+
473
+ def modify(self, fileobj: Any, events: int, data: Any = None) -> selectors.SelectorKey:
474
+ self._room(fileobj, events) # before modify() unregisters it
475
+ return super().modify(fileobj, events, data)
476
+
477
+
478
+ class _Early(asyncio.Protocol):
479
+ """What arrives on a TLS connection between the end of its handshake and the hand-over
480
+ to the protocol that serves it (both in one pass of the event loop)."""
481
+
482
+ def __init__(self) -> None:
483
+ self.data = bytearray()
484
+ self.eof = False
485
+ self.lost = False
486
+
487
+ def data_received(self, data: bytes) -> None:
488
+ self.data += data
489
+
490
+ def eof_received(self) -> bool:
491
+ self.eof = True
492
+ return False
493
+
494
+ def connection_lost(self, exc: Optional[BaseException]) -> None:
495
+ self.lost = True
496
+
497
+
498
+ async def start_tls(transport: Any, ctx: Any, protocol: asyncio.BaseProtocol) -> bool:
499
+ """The server side of a TLS handshake on an accepted ``transport`` whose reading the
500
+ caller paused, within TLS_HANDSHAKE_TIMEOUT_S; then ``protocol`` serves the connection
501
+ (its connection_made gets the TLS transport). False when the connection went away
502
+ first; raises what the handshake raised (the transport is closed then)."""
503
+ loop = asyncio.get_running_loop()
504
+ early = _Early()
505
+ if sys.version_info >= (3, 11):
506
+ tls = await loop.start_tls(transport, early, ctx, server_side=True,
507
+ ssl_handshake_timeout=TLS_HANDSHAKE_TIMEOUT_S,
508
+ ssl_shutdown_timeout=TLS_SHUTDOWN_TIMEOUT_S)
509
+ else: # pragma: no cover - exercised only on Python 3.10 (no ssl_shutdown_timeout)
510
+ tls = await loop.start_tls(transport, early, ctx, server_side=True,
511
+ ssl_handshake_timeout=TLS_HANDSHAKE_TIMEOUT_S)
512
+ if tls is None or early.lost or tls.is_closing():
513
+ transport.close()
514
+ return False
515
+ if sys.version_info < (3, 11): # pragma: no cover - Python 3.10 only
516
+ _bound_close(tls, transport)
517
+ tls.set_protocol(protocol)
518
+ protocol.connection_made(tls)
519
+ if early.data and isinstance(protocol, asyncio.Protocol):
520
+ protocol.data_received(bytes(early.data))
521
+ if early.eof and isinstance(protocol, asyncio.Protocol):
522
+ protocol.eof_received()
523
+ return True
524
+
525
+
526
+ def _bound_close(tls: Any, transport: Any) -> None: # pragma: no cover - Python 3.10 only
527
+ """Python 3.10 has no ssl_shutdown_timeout: closing a TLS connection waits for the
528
+ peer's close_notify for as long as that takes, so a peer that never answers kept the
529
+ socket (and its place in the connection limits) open for good. Closing ``tls`` now
530
+ also aborts ``transport``, the socket under it, TLS_SHUTDOWN_TIMEOUT_S later if it is
531
+ still open (aborting a closed one does nothing)."""
532
+ loop = asyncio.get_running_loop()
533
+ close = tls.close
534
+
535
+ def bounded_close() -> None:
536
+ close()
537
+ if not loop.is_closed(): # (a transport collected after the loop ended)
538
+ loop.call_later(TLS_SHUTDOWN_TIMEOUT_S, transport.abort)
539
+
540
+ tls.close = bounded_close
541
+
542
+
543
+ def describe_tls_error(exc: BaseException) -> str:
544
+ text = str(exc) or type(exc).__name__
545
+ return text if isinstance(exc, ConnectionError) else f"{type(exc).__name__}: {text}"
546
+
547
+
548
+ def begin_tls(transport: Any, ctx: Any, protocol: asyncio.BaseProtocol, label: str,
549
+ failed: Callable[[str], None],
550
+ refused: Optional[Callable[[str], None]] = None) -> asyncio.Task:
551
+ """Start the TLS handshake of an admitted connection: counted among the handshakes in
552
+ progress until it is over (or waiting for room among them first), run in a task
553
+ (start_tls) that pauses reading first. A failed handshake is reported to ``failed``, a
554
+ connection that waited in vain to ``refused`` (default: ``failed``; rate-limit both:
555
+ anyone can cause them)."""
556
+ BUDGET.enter_handshake(transport, peer_ip(transport))
557
+
558
+ async def handshake() -> None:
559
+ try:
560
+ # Paused here, not in the caller (a protocol's connection_made): the transport
561
+ # starts reading in a callback of its own after connection_made, and on Python
562
+ # 3.10 that start ignores an earlier pause, so the ClientHello would reach the
563
+ # protocol that admits connections and be lost. The task's first step runs
564
+ # after that callback and before the first read.
565
+ transport.pause_reading() # the ClientHello waits for the TLS protocol
566
+ if not await BUDGET.handshake_turn(transport):
567
+ if not transport.is_closing():
568
+ BUDGET.note_refused("tls_handshakes")
569
+ (refused or failed)(f"refused {label}: it waited "
570
+ f"{TLS_HANDSHAKE_TIMEOUT_S:g} seconds for a TLS "
571
+ "handshake to end")
572
+ transport.abort()
573
+ return
574
+ await start_tls(transport, ctx, protocol)
575
+ except Exception as e: # the timeout, a bad ClientHello, the client went away
576
+ failed(f"TLS handshake with {label} failed: {describe_tls_error(e)}")
577
+ finally:
578
+ BUDGET.handshake_done(transport)
579
+
580
+ return asyncio.get_running_loop().create_task(handshake())
581
+
582
+
583
+ class _Gate(asyncio.Protocol):
584
+ """A new connection's first protocol: its listener admits it (giving it the protocol
585
+ that serves it) or closes it."""
586
+
587
+ def __init__(self, listener: Listener) -> None:
588
+ self._listener = listener
589
+
590
+ def connection_made(self, transport: asyncio.BaseTransport) -> None:
591
+ self._listener.admit(transport)
592
+
593
+
594
+ class Listener:
595
+ """A mapping's TCP listener (``start_server``); close() / wait_closed() / sockets as on
596
+ the asyncio Server it wraps."""
597
+
598
+ def __init__(self, client_connected_cb: Callable, ssl: Any, allowed: Optional[Callable],
599
+ log: Callable[[str], None], limit: int) -> None:
600
+ self._loop = asyncio.get_running_loop()
601
+ self._cb = client_connected_cb
602
+ self._ssl = ssl
603
+ self._allowed = allowed
604
+ self._log = log
605
+ self._limit = limit
606
+ self._refused = LogEvery(log)
607
+ self._failed = LogEvery(log)
608
+ self._rejected = RefusalLog(log)
609
+ self._handshakes: dict[Any, asyncio.Task] = {} # plain transport -> its handshake
610
+ self.server: Any = None
611
+
612
+ @property
613
+ def sockets(self):
614
+ return self.server.sockets
615
+
616
+ @property
617
+ def ssl(self) -> Any:
618
+ """The TLS context new connections get (None: plain TCP)."""
619
+ return self._ssl
620
+
621
+ @ssl.setter
622
+ def ssl(self, context: Any) -> None: # a renewed certificate (ser2netweb/tlsreload.py)
623
+ self._ssl = context
624
+
625
+ def close(self) -> None:
626
+ self.server.close()
627
+ for transport, task in list(self._handshakes.items()):
628
+ transport.abort()
629
+ task.cancel() # (one waiting for a handshake to end, too)
630
+
631
+ async def wait_closed(self) -> None:
632
+ await self.server.wait_closed()
633
+
634
+ def _stream_protocol(self) -> asyncio.StreamReaderProtocol:
635
+ reader = asyncio.StreamReader(limit=self._limit, loop=self._loop)
636
+ return asyncio.StreamReaderProtocol(reader, self._cb, loop=self._loop)
637
+
638
+ def admit(self, transport: Any) -> None:
639
+ label = peer_label(transport)
640
+ why = BUDGET.headroom()
641
+ if not why and self._ssl is not None:
642
+ ip = peer_ip(transport)
643
+ if self._allowed is not None and not self._allowed(ip):
644
+ self._rejected(f"rejected {label}: not in allowed list")
645
+ transport.close()
646
+ return
647
+ why = BUDGET.handshake_room(ip)
648
+ if why:
649
+ self._refused(f"refused {label}: {why}")
650
+ transport.close()
651
+ return
652
+ protocol = self._stream_protocol()
653
+ if self._ssl is None:
654
+ transport.set_protocol(protocol)
655
+ protocol.connection_made(transport)
656
+ return
657
+ task = begin_tls(transport, self._ssl, protocol, label, self._failed, self._refused)
658
+ self._handshakes[transport] = task
659
+ task.add_done_callback(lambda _t: self._handshakes.pop(transport, None))
660
+
661
+
662
+ def listen_hosts(net: NetworkSettings) -> list[str]:
663
+ """The addresses a mapping's TCP listener binds: both wildcards when it takes IPv4 and
664
+ IPv6 clients (NetworkSettings.both_families; 0.0.0.0 alone where Python has no IPv6),
665
+ else its bind IP."""
666
+ if net.both_families:
667
+ return list(WILDCARDS) if socket.has_ipv6 else ["0.0.0.0"]
668
+ return [net.bind_ip]
669
+
670
+
671
+ def listen_where(net: NetworkSettings, sockets: Sequence[socket.socket]) -> str:
672
+ """Where a mapping listens, for its log line: its address and port, and which clients
673
+ that takes when it is a wildcard of one family or both (``sockets``: what it got)."""
674
+ if net.both_families:
675
+ families = {sock.family for sock in sockets}
676
+ if families == {socket.AF_INET6} and _takes_ipv4(sockets[0]):
677
+ families.add(socket.AF_INET) # (UDP: one socket for both)
678
+ if families == {socket.AF_INET, socket.AF_INET6}:
679
+ return f"{host_port('0.0.0.0', net.port)} and {host_port('::', net.port)}, IPv4 and IPv6"
680
+ if families == {socket.AF_INET}:
681
+ return f"{host_port('0.0.0.0', net.port)}, IPv4 only (IPv6 is not available here)"
682
+ where = host_port(net.bind_ip, net.port)
683
+ if canonical_ip(net.bind_ip) == "::" and not net.both_families:
684
+ where += ", IPv6 clients only"
685
+ return where
686
+
687
+
688
+ def _takes_ipv4(sock: socket.socket) -> bool:
689
+ try:
690
+ return not sock.getsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY)
691
+ except (OSError, AttributeError):
692
+ return False
693
+
694
+
695
+ async def udp_socket(net: NetworkSettings) -> socket.socket:
696
+ """A UDP mapping's socket, bound to its address and port. IPV6_V6ONLY is set on an IPv6
697
+ one as the mapping says, not left to the OS (Linux would let a socket on :: take IPv4
698
+ senders too, Windows not): one that takes both families (NetworkSettings.both_families)
699
+ is one socket on :: with it off (where this machine has no IPv6, an IPv4 one on
700
+ 0.0.0.0), any other has it on, so :: takes IPv6 senders only, as a TCP listener does.
701
+ Raises BindError when the address and port can't be taken."""
702
+ both = net.both_families
703
+ address: Any
704
+ try:
705
+ if both:
706
+ family, address = socket.AF_INET6, ("::", net.port)
707
+ try:
708
+ sock = socket.socket(family, socket.SOCK_DGRAM)
709
+ except (OSError, AttributeError): # no IPv6 here
710
+ family, address = socket.AF_INET, ("0.0.0.0", net.port)
711
+ sock = socket.socket(family, socket.SOCK_DGRAM)
712
+ else:
713
+ infos = await asyncio.get_running_loop().getaddrinfo(
714
+ net.bind_ip, net.port, type=socket.SOCK_DGRAM,
715
+ flags=socket.AI_PASSIVE | socket.AI_NUMERICHOST)
716
+ family, kind, proto, _name, address = infos[0]
717
+ sock = socket.socket(family, kind, proto)
718
+ except OSError as e:
719
+ raise BindError.of(e) from e
720
+ try:
721
+ if family == socket.AF_INET6 and hasattr(socket, "IPPROTO_IPV6"):
722
+ sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0 if both else 1)
723
+ if net.udp_broadcast is True: # (udp_targets: a broadcast address)
724
+ sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
725
+ sock.bind(address)
726
+ sock.setblocking(False)
727
+ except OSError as e:
728
+ sock.close()
729
+ raise BindError.of(e) from e
730
+ return sock
731
+
732
+
733
+ async def start_server(client_connected_cb: Callable, host: Union[str, Sequence[str]], port: int,
734
+ *, ssl: Any = None,
735
+ allowed: Optional[Callable[[str], bool]] = None,
736
+ log: Optional[Callable[[str], None]] = None,
737
+ limit: int = 2 ** 16) -> Listener:
738
+ """asyncio.start_server(client_connected_cb, host, port, ssl=ssl) (``host``: one
739
+ address, or several: listen_hosts), admitting each connection right after accept: the
740
+ process's budget, then (TLS) ``allowed`` (the mapping's allow-list) and the handshakes
741
+ in progress, before the handshake.
742
+ ``client_connected_cb(reader, writer)`` gets TLS connections once their handshake is
743
+ done. ``log``: where refusals and failed handshakes are reported (the mapping's log).
744
+ Raises BindError when the address and port can't be taken."""
745
+ listener = Listener(client_connected_cb, ssl, allowed, log or (lambda _m: None), limit)
746
+ try:
747
+ server = await asyncio.get_running_loop().create_server(
748
+ lambda: _Gate(listener), host, port, start_serving=False)
749
+ except OSError as e:
750
+ raise BindError.of(e) from e
751
+ clash = _reserved_clash(server.sockets)
752
+ if clash:
753
+ server.close()
754
+ raise BindError.of(OSError(errno.EADDRINUSE, clash))
755
+ listener.server = server
756
+ await server.start_serving()
757
+ return listener
758
+
759
+
760
+ # Sockets that later listeners may not share an address and port with (reserve), with what
761
+ # holds them. Windows lets a listener on a wildcard address share a port with one on a
762
+ # specific address of its family (that one keeps its address's connections, the other gets
763
+ # the rest) and, without SO_EXCLUSIVEADDRUSE on the first, the reverse too; Linux refuses
764
+ # the second bind while the first listens. start_server refuses it on every OS, so a mapping
765
+ # on the admin UI's address and port fails (and is retried) instead of running next to it.
766
+ _reserved: weakref.WeakKeyDictionary[socket.socket, str] = weakref.WeakKeyDictionary()
767
+
768
+
769
+ def reserve(sockets: Sequence[socket.socket], holder: str) -> None:
770
+ """Keep the addresses and ports of ``sockets`` (bound) from the listeners start_server
771
+ makes afterwards while they are open; ``holder`` names them in the error ("the admin
772
+ UI")."""
773
+ for sock in sockets:
774
+ _reserved[sock] = holder
775
+
776
+
777
+ def _reserved_clash(sockets: Sequence[Any]) -> str:
778
+ """Why one of ``sockets`` (bound, not listening yet) can't listen: it shares the port of
779
+ a reserved socket on an overlapping address (the same one, or a wildcard of its family);
780
+ "" when none does."""
781
+ for sock in sockets:
782
+ try:
783
+ ip, port = sock.getsockname()[:2]
784
+ except OSError:
785
+ continue
786
+ for held, holder in list(_reserved.items()):
787
+ if held.fileno() == -1 or held.family != sock.family:
788
+ continue
789
+ try:
790
+ held_ip, held_port = held.getsockname()[:2]
791
+ except OSError:
792
+ continue
793
+ if held_port == port and _overlaps(held_ip, ip):
794
+ return (f"error while attempting to bind on address {host_port(ip, port)}: "
795
+ f"address already in use ({holder} listens on "
796
+ f"{host_port(held_ip, held_port)})")
797
+ return ""
798
+
799
+
800
+ def _overlaps(a: str, b: str) -> bool:
801
+ try:
802
+ x, y = ipaddress.ip_address(a), ipaddress.ip_address(b)
803
+ except ValueError:
804
+ return a == b
805
+ return x == y or x.is_unspecified or y.is_unspecified