ser2netweb 3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ser2netweb/__init__.py +20 -0
- ser2netweb/__main__.py +6 -0
- ser2netweb/accounts.py +503 -0
- ser2netweb/cli.py +489 -0
- ser2netweb/clock.py +128 -0
- ser2netweb/config.py +4491 -0
- ser2netweb/console.py +181 -0
- ser2netweb/datalock.py +181 -0
- ser2netweb/engine/__init__.py +0 -0
- ser2netweb/engine/access.py +173 -0
- ser2netweb/engine/bindhint.py +115 -0
- ser2netweb/engine/bridge.py +1928 -0
- ser2netweb/engine/cluster.py +1140 -0
- ser2netweb/engine/modbus.py +530 -0
- ser2netweb/engine/modbus_gateway.py +1465 -0
- ser2netweb/engine/mqtt_pub.py +632 -0
- ser2netweb/engine/netguard.py +805 -0
- ser2netweb/engine/netinfo.py +199 -0
- ser2netweb/engine/portdiag.py +414 -0
- ser2netweb/engine/portlist.py +471 -0
- ser2netweb/engine/protocols/__init__.py +28 -0
- ser2netweb/engine/protocols/base.py +117 -0
- ser2netweb/engine/protocols/raw.py +12 -0
- ser2netweb/engine/protocols/rfc2217.py +586 -0
- ser2netweb/engine/protocols/telnet.py +172 -0
- ser2netweb/engine/serial_compat.py +598 -0
- ser2netweb/engine/serial_io.py +210 -0
- ser2netweb/engine/serial_link.py +414 -0
- ser2netweb/engine/sockopts.py +103 -0
- ser2netweb/engine/sparkplug.py +538 -0
- ser2netweb/engine/sparkplug_node.py +550 -0
- ser2netweb/engine/supervisor.py +533 -0
- ser2netweb/engine/tls.py +232 -0
- ser2netweb/envvars.py +57 -0
- ser2netweb/explain.py +117 -0
- ser2netweb/healthcheck.py +78 -0
- ser2netweb/i18n.py +110 -0
- ser2netweb/importer.py +550 -0
- ser2netweb/layout.py +118 -0
- ser2netweb/licences.py +86 -0
- ser2netweb/locale/tr.json +1569 -0
- ser2netweb/logsetup.py +253 -0
- ser2netweb/logtext.py +23 -0
- ser2netweb/notify.py +761 -0
- ser2netweb/optional.py +76 -0
- ser2netweb/reload.py +295 -0
- ser2netweb/runtime.py +762 -0
- ser2netweb/sdnotify.py +85 -0
- ser2netweb/ser2net_import.py +2001 -0
- ser2netweb/signins.py +98 -0
- ser2netweb/standalone.py +154 -0
- ser2netweb/state.py +1457 -0
- ser2netweb/storage.py +130 -0
- ser2netweb/textfold.py +51 -0
- ser2netweb/tlsreload.py +37 -0
- ser2netweb/watchdog.py +201 -0
- ser2netweb/web/__init__.py +0 -0
- ser2netweb/web/api.py +793 -0
- ser2netweb/web/auth.py +699 -0
- ser2netweb/web/certinfo.py +144 -0
- ser2netweb/web/httpguard.py +413 -0
- ser2netweb/web/ldap_auth.py +556 -0
- ser2netweb/web/oidc_auth.py +330 -0
- ser2netweb/web/proxy.py +398 -0
- ser2netweb/web/routes/__init__.py +78 -0
- ser2netweb/web/routes/cluster.py +669 -0
- ser2netweb/web/routes/common.py +359 -0
- ser2netweb/web/routes/console.py +451 -0
- ser2netweb/web/routes/forms.py +853 -0
- ser2netweb/web/routes/login.py +573 -0
- ser2netweb/web/routes/mappings.py +470 -0
- ser2netweb/web/routes/ops.py +374 -0
- ser2netweb/web/routes/settings.py +2135 -0
- ser2netweb/web/routes/system.py +308 -0
- ser2netweb/web/routes/totp.py +282 -0
- ser2netweb/web/selfsigned.py +129 -0
- ser2netweb/web/server.py +483 -0
- ser2netweb/web/static/app.css +327 -0
- ser2netweb/web/static/app.js +1102 -0
- ser2netweb/web/static/htmx.min.js +1 -0
- ser2netweb/web/static/points.js +218 -0
- ser2netweb/web/static/theme.js +7 -0
- ser2netweb/web/static/xterm-addon-fit.js +2 -0
- ser2netweb/web/static/xterm.css +218 -0
- ser2netweb/web/static/xterm.js +2 -0
- ser2netweb/web/templates/_cluster_body.html +109 -0
- ser2netweb/web/templates/_console.html +21 -0
- ser2netweb/web/templates/_footer.html +13 -0
- ser2netweb/web/templates/_login_notice.html +8 -0
- ser2netweb/web/templates/_macros.html +21 -0
- ser2netweb/web/templates/_mapping_form.html +477 -0
- ser2netweb/web/templates/_mapping_log.html +28 -0
- ser2netweb/web/templates/_mapping_traces.html +44 -0
- ser2netweb/web/templates/_mappings_body.html +149 -0
- ser2netweb/web/templates/_modbus_read.html +18 -0
- ser2netweb/web/templates/_ports_table.html +37 -0
- ser2netweb/web/templates/_serial_port_select.html +48 -0
- ser2netweb/web/templates/_shell.html +41 -0
- ser2netweb/web/templates/audit.html +55 -0
- ser2netweb/web/templates/base.html +77 -0
- ser2netweb/web/templates/dashboard.html +72 -0
- ser2netweb/web/templates/licenses.html +18 -0
- ser2netweb/web/templates/login.html +32 -0
- ser2netweb/web/templates/login_totp.html +23 -0
- ser2netweb/web/templates/settings.html +830 -0
- ser2netweb/web/templates/setup.html +43 -0
- ser2netweb/web/templates/totp.html +41 -0
- ser2netweb/web/totp.py +94 -0
- ser2netweb/winservice.py +1793 -0
- ser2netweb-3.0.0.dist-info/METADATA +120 -0
- ser2netweb-3.0.0.dist-info/RECORD +121 -0
- ser2netweb-3.0.0.dist-info/WHEEL +5 -0
- ser2netweb-3.0.0.dist-info/entry_points.txt +2 -0
- ser2netweb-3.0.0.dist-info/licenses/COMMERCIAL_LICENSE.md +94 -0
- ser2netweb-3.0.0.dist-info/licenses/LICENSE +662 -0
- ser2netweb-3.0.0.dist-info/licenses/NOTICE +20 -0
- ser2netweb-3.0.0.dist-info/licenses/THIRD-PARTY-NOTICES.md +105 -0
- ser2netweb-3.0.0.dist-info/licenses/tools/licenses/npm/htmx.org.txt +13 -0
- ser2netweb-3.0.0.dist-info/licenses/tools/licenses/npm/xterm-addon-fit.txt +19 -0
- ser2netweb-3.0.0.dist-info/licenses/tools/licenses/npm/xterm.txt +21 -0
- ser2netweb-3.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,805 @@
|
|
|
1
|
+
"""The process's open files: their limit, how many are in use, and what the mappings need.
|
|
2
|
+
|
|
3
|
+
Every serial port, listener, client connection, log and trace file and MQTT connection
|
|
4
|
+
holds a file descriptor, and a process may have at most its soft RLIMIT_NOFILE open at
|
|
5
|
+
once: 1024 under systemd unless the unit raises it. Past it, whatever opens the next
|
|
6
|
+
file fails, wherever that is (a serial reopen, a client, a config save). So:
|
|
7
|
+
- ``raise_fd_limit`` lifts the soft limit toward the hard one at start-up (at most
|
|
8
|
+
FD_LIMIT_TARGET), with prlimit() on Linux: the shipped unit's system call filter
|
|
9
|
+
denies setrlimit() but allows prlimit64. Nothing in the process waits with select()
|
|
10
|
+
any more (serial_compat, mqtt_pub.PollSelect), so descriptors above 1024 work;
|
|
11
|
+
- ``fd_budget_warning`` says when the configured mappings need about as many as the
|
|
12
|
+
limit allows;
|
|
13
|
+
- ``open_fds`` / ``fd_limit`` are what /metrics reports.
|
|
14
|
+
Windows has no such limit for sockets and serial ports (they are handles): each of these
|
|
15
|
+
answers None or "" there.
|
|
16
|
+
|
|
17
|
+
Incoming connections pass the process's budget (``BUDGET``) right after accept, whatever
|
|
18
|
+
listener they reach, before they cost anything more:
|
|
19
|
+
- ``Budget.headroom``: a connection is refused (closed, with a log line at most once a
|
|
20
|
+
minute) while the process is within ``fd_reserve`` of its open-file limit, so clients
|
|
21
|
+
can't take the files that serial ports, logs and the configuration need;
|
|
22
|
+
- TLS handshakes in progress hold a socket and a 256 KiB buffer each: at most
|
|
23
|
+
TLS_HANDSHAKES_MAX at a time, TLS_HANDSHAKES_PER_IP from one address (an IPv6 one
|
|
24
|
+
counted by its /64, see address_key, as the admin UI's per-address cap), each given
|
|
25
|
+
TLS_HANDSHAKE_TIMEOUT_S (asyncio's default is 60 s). A connection past those waits
|
|
26
|
+
for a handshake to end (first come, first served; reading paused, so it holds only
|
|
27
|
+
its socket) for up to TLS_HANDSHAKE_TIMEOUT_S, at most TLS_WAITING_PER_IP from one
|
|
28
|
+
address and TLS_WAITING_MAX in all: a client's burst of reconnects is served, not
|
|
29
|
+
refused. Past those too, a connection is refused.
|
|
30
|
+
- Windows' SelectorEventLoop (``CappedSelectSelector``): select() takes at most 512
|
|
31
|
+
sockets there, so connections are refused while it watches nearly that many.
|
|
32
|
+
``start_server`` is asyncio.start_server for a mapping's listener with that admission. For
|
|
33
|
+
a TLS listener it also applies the mapping's allow-list before the handshake (a refused
|
|
34
|
+
host never gets one) and does the handshake itself (``start_tls``): the mapping sees the
|
|
35
|
+
connection only once it is done, and decides about capacity, kicking and "Busy" then.
|
|
36
|
+
When the listener can't take its address it raises ``BindError``; ``udp_socket`` makes a
|
|
37
|
+
UDP mapping's socket, with the same error. Nor can it take one that overlaps a socket
|
|
38
|
+
``reserve`` holds (the admin UI's): Windows lets a listener on a wildcard address share a
|
|
39
|
+
port with one on a specific address, where Linux refuses the second bind.
|
|
40
|
+
|
|
41
|
+
Address families: a listener on :: takes IPv6 clients only, on every OS (IPV6_V6ONLY: asyncio
|
|
42
|
+
sets it for TCP, ``udp_socket`` for UDP); one that takes both (network.dual_stack on a
|
|
43
|
+
wildcard) has two TCP sockets, 0.0.0.0 and :: (``listen_hosts``), so its IPv4 clients have
|
|
44
|
+
plain IPv4 addresses, or one UDP socket on :: that takes IPv4 senders too.
|
|
45
|
+
"""
|
|
46
|
+
from __future__ import annotations
|
|
47
|
+
|
|
48
|
+
import asyncio
|
|
49
|
+
import errno
|
|
50
|
+
import ipaddress
|
|
51
|
+
import os
|
|
52
|
+
import selectors
|
|
53
|
+
import socket
|
|
54
|
+
import sys
|
|
55
|
+
import time
|
|
56
|
+
import weakref
|
|
57
|
+
from typing import TYPE_CHECKING, Any, Callable, Optional, Sequence, Union
|
|
58
|
+
|
|
59
|
+
from ..config import WILDCARDS, canonical_ip, host_port
|
|
60
|
+
|
|
61
|
+
if TYPE_CHECKING:
|
|
62
|
+
from ..config import NetworkSettings
|
|
63
|
+
|
|
64
|
+
# the soft open-file limit raise_fd_limit aims for: far above what a few hundred
|
|
65
|
+
# mappings need, without handing child processes a huge limit
|
|
66
|
+
FD_LIMIT_TARGET = 65536
|
|
67
|
+
# what a process holds open before any mapping: stdio, all.log, the event loop, the admin
|
|
68
|
+
# listener and its connections, the executor's threads' files
|
|
69
|
+
FD_BASE = 40
|
|
70
|
+
|
|
71
|
+
# TLS on the listeners: the handshake must be over in this time; closing a TLS connection
|
|
72
|
+
# waits this long for the peer's close_notify (asyncio's ssl_shutdown_timeout from Python
|
|
73
|
+
# 3.11; on 3.10, _bound_close)
|
|
74
|
+
TLS_HANDSHAKE_TIMEOUT_S = 15.0
|
|
75
|
+
TLS_SHUTDOWN_TIMEOUT_S = 5.0
|
|
76
|
+
# TLS handshakes in progress at once: in the process, and from one source address
|
|
77
|
+
TLS_HANDSHAKES_MAX = 128
|
|
78
|
+
TLS_HANDSHAKES_PER_IP = 32
|
|
79
|
+
# connections waiting for one of those to end (each for at most TLS_HANDSHAKE_TIMEOUT_S):
|
|
80
|
+
# in the process, and from one source address
|
|
81
|
+
TLS_WAITING_MAX = 256
|
|
82
|
+
TLS_WAITING_PER_IP = 64
|
|
83
|
+
# Windows' select() takes at most this many sockets in each of its sets (FD_SETSIZE);
|
|
84
|
+
# incoming connections are refused once the event loop watches all but SELECT_HEADROOM
|
|
85
|
+
# of them, which stay for listeners, connections out and UDP sockets
|
|
86
|
+
SELECT_LIMIT = 512
|
|
87
|
+
SELECT_HEADROOM = 64
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class BindError(OSError):
|
|
91
|
+
"""A listener could not take its address and port: another socket holds it, or the
|
|
92
|
+
address is not on this machine (yet: DHCP, a NIC or a tentative IPv6 address that comes
|
|
93
|
+
up later). Worth another try later (the supervisor makes them), whatever form the error
|
|
94
|
+
came in: asyncio on Python 3.11+ reports a missing address as "could not bind on any
|
|
95
|
+
address", with no errno."""
|
|
96
|
+
|
|
97
|
+
@classmethod
|
|
98
|
+
def of(cls, e: OSError) -> BindError:
|
|
99
|
+
err = cls(str(e))
|
|
100
|
+
err.errno = e.errno
|
|
101
|
+
return err
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class LogEvery:
|
|
105
|
+
"""A log line that can repeat many times a second (dropped data, a UDP peer that keeps
|
|
106
|
+
changing, refused connections): written at most once per ``interval`` seconds, saying
|
|
107
|
+
how many were held back since the last one."""
|
|
108
|
+
|
|
109
|
+
def __init__(self, log: Callable[[str], None], interval: float = 60.0) -> None:
|
|
110
|
+
self._log = log
|
|
111
|
+
self._interval = interval
|
|
112
|
+
self._next = 0.0
|
|
113
|
+
self._held = 0
|
|
114
|
+
|
|
115
|
+
def __call__(self, msg: str) -> None:
|
|
116
|
+
now = time.monotonic()
|
|
117
|
+
if now < self._next:
|
|
118
|
+
self._held += 1
|
|
119
|
+
return
|
|
120
|
+
if self._held:
|
|
121
|
+
msg += f" ({self._held} more since the last such line)"
|
|
122
|
+
self._held = 0
|
|
123
|
+
self._next = now + self._interval
|
|
124
|
+
self._log(msg)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
class RefusalLog:
|
|
128
|
+
"""The log lines of refused connections ("rejected <peer>: <why>"), which a client
|
|
129
|
+
that retries in a loop, or a scan, can cause many times a second: each reason at most
|
|
130
|
+
once per ``interval`` seconds (LogEvery), the next one saying how many were held back.
|
|
131
|
+
Other lines (a kick) are logged as they are."""
|
|
132
|
+
|
|
133
|
+
MAX_REASONS = 32 # (the reasons are fixed texts; past that, they share one line)
|
|
134
|
+
|
|
135
|
+
def __init__(self, log: Callable[[str], None], interval: float = 60.0) -> None:
|
|
136
|
+
self._log = log
|
|
137
|
+
self._interval = interval
|
|
138
|
+
self._by_reason: dict[str, LogEvery] = {}
|
|
139
|
+
|
|
140
|
+
def __call__(self, msg: str) -> None:
|
|
141
|
+
if not msg.startswith("rejected "):
|
|
142
|
+
self._log(msg)
|
|
143
|
+
return
|
|
144
|
+
why = msg.partition(": ")[2]
|
|
145
|
+
if why not in self._by_reason and len(self._by_reason) >= self.MAX_REASONS:
|
|
146
|
+
why = ""
|
|
147
|
+
limiter = self._by_reason.get(why)
|
|
148
|
+
if limiter is None:
|
|
149
|
+
limiter = self._by_reason[why] = LogEvery(self._log, self._interval)
|
|
150
|
+
limiter(msg)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _resource() -> Any:
|
|
154
|
+
try:
|
|
155
|
+
import resource
|
|
156
|
+
except ImportError: # Windows
|
|
157
|
+
return None
|
|
158
|
+
return resource
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def fd_limit() -> Optional[int]:
|
|
162
|
+
"""The soft open-file limit (None: unlimited, or Windows)."""
|
|
163
|
+
resource = _resource()
|
|
164
|
+
if resource is None:
|
|
165
|
+
return None
|
|
166
|
+
soft, _hard = resource.getrlimit(resource.RLIMIT_NOFILE)
|
|
167
|
+
return None if soft == resource.RLIM_INFINITY else int(soft)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def open_fds() -> Optional[int]:
|
|
171
|
+
"""How many files this process has open (None where that can't be counted cheaply)."""
|
|
172
|
+
if os.name != "posix":
|
|
173
|
+
return None
|
|
174
|
+
for folder in ("/proc/self/fd", "/dev/fd"):
|
|
175
|
+
try:
|
|
176
|
+
return len(os.listdir(folder)) - 1 # less the one listdir() itself opened
|
|
177
|
+
except OSError:
|
|
178
|
+
continue
|
|
179
|
+
return None
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _shown(value: int, resource: Any) -> str:
|
|
183
|
+
return "unlimited" if value == resource.RLIM_INFINITY else str(value)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def raise_fd_limit(target: int = FD_LIMIT_TARGET) -> str:
|
|
187
|
+
"""Raise the soft open-file limit toward the hard limit, at most to ``target`` (never
|
|
188
|
+
lower it). Returns the line for the start-up log ("" on Windows)."""
|
|
189
|
+
resource = _resource()
|
|
190
|
+
if resource is None:
|
|
191
|
+
return ""
|
|
192
|
+
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
|
|
193
|
+
want = target if hard == resource.RLIM_INFINITY else min(hard, target)
|
|
194
|
+
if soft == resource.RLIM_INFINITY or soft >= want:
|
|
195
|
+
return f"open files: limit {_shown(soft, resource)}"
|
|
196
|
+
try:
|
|
197
|
+
if hasattr(resource, "prlimit"): # Linux: prlimit64, never the setrlimit() call
|
|
198
|
+
resource.prlimit(0, resource.RLIMIT_NOFILE, (want, hard))
|
|
199
|
+
else:
|
|
200
|
+
resource.setrlimit(resource.RLIMIT_NOFILE, (want, hard))
|
|
201
|
+
except (OSError, ValueError) as e:
|
|
202
|
+
return f"open files: limit {soft} (raising it to {want} failed: {e})"
|
|
203
|
+
return f"open files: limit raised from {soft} to {want} (hard limit {_shown(hard, resource)})"
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def fd_estimate(config: Any) -> int:
|
|
207
|
+
"""About how many files the configured mappings hold open: per mapping its log file
|
|
208
|
+
(disabled ones too), and per enabled one its serial port(s), its socket (a listener, a
|
|
209
|
+
UDP socket or the connection out), each client it admits, its trace file and an MQTT
|
|
210
|
+
connection (its socket and paho's wake-up pair)."""
|
|
211
|
+
need = FD_BASE
|
|
212
|
+
for m in config.mappings:
|
|
213
|
+
need += 1
|
|
214
|
+
if not m.enabled:
|
|
215
|
+
continue
|
|
216
|
+
need += len(m.serial_sides())
|
|
217
|
+
need += 1 if m.options.trace_both else 0
|
|
218
|
+
need += 3 if m.mqtt.enabled else 0
|
|
219
|
+
if m.kind == "net":
|
|
220
|
+
need += 1 + (m.network.max_connections if m.network.mode == "server" else 0)
|
|
221
|
+
return need
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def fd_budget_warning(config: Any, limit: Optional[int] = None) -> str:
|
|
225
|
+
"""A start-up warning when the mappings need over 80% of the open-file limit ("" when
|
|
226
|
+
they fit, or there is no such limit)."""
|
|
227
|
+
limit = fd_limit() if limit is None else limit
|
|
228
|
+
if limit is None:
|
|
229
|
+
return ""
|
|
230
|
+
need = fd_estimate(config)
|
|
231
|
+
if need <= limit * 0.8:
|
|
232
|
+
return ""
|
|
233
|
+
return (f"WARNING: the {len(config.mappings)} mappings need about {need} open files, "
|
|
234
|
+
f"close to this process's limit of {limit}: beyond it serial ports, clients and "
|
|
235
|
+
"files fail to open. Raise the limit (LimitNOFILE= in the systemd unit, "
|
|
236
|
+
"--ulimit nofile= for Docker) or split the mappings over more instances.")
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def fd_reserve(limit: int) -> int:
|
|
240
|
+
"""Open files kept free for what is not a client (serial ports, logs, the configuration
|
|
241
|
+
and its backups, traces, MQTT): incoming connections are refused within this many of
|
|
242
|
+
the limit."""
|
|
243
|
+
return min(256, max(32, limit // 8))
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
# ---------------------------------------------------------------------------
|
|
247
|
+
# incoming connections
|
|
248
|
+
# ---------------------------------------------------------------------------
|
|
249
|
+
def peer_address(transport: Any) -> tuple[str, int]:
|
|
250
|
+
"""A connection's (a stream writer's) source address and port: the address in its
|
|
251
|
+
canonical form, an IPv4-mapped IPv6 one (an IPv4 client of a socket that takes both
|
|
252
|
+
families) as its IPv4 address; port 0 when there is none."""
|
|
253
|
+
peer = transport.get_extra_info("peername")
|
|
254
|
+
if isinstance(peer, tuple) and peer:
|
|
255
|
+
port = peer[1] if len(peer) > 1 and isinstance(peer[1], int) else 0
|
|
256
|
+
return canonical_ip(str(peer[0])), port
|
|
257
|
+
return str(peer or "?"), 0
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def peer_ip(transport: Any) -> str:
|
|
261
|
+
"""A connection's source address (see peer_address)."""
|
|
262
|
+
return peer_address(transport)[0]
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def client_label(ip: str, port: int) -> str:
|
|
266
|
+
"""ip:port of a client, for the log and the UI ([2001:db8::7]:50123), just the address
|
|
267
|
+
without a port."""
|
|
268
|
+
return host_port(ip, port) if port else ip
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def peer_label(transport: Any) -> str:
|
|
272
|
+
"""ip:port of a connection, for the log."""
|
|
273
|
+
return client_label(*peer_address(transport))
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def address_key(ip: str) -> str:
|
|
277
|
+
"""What the per-address limits (the connection caps here, the web UI's sign-in limits
|
|
278
|
+
in web/auth.py) count a source under: an IPv6 address by its /64 network (a host, or a
|
|
279
|
+
LAN, gets a whole /64 and can use any address in it, so one client can't get past a
|
|
280
|
+
limit by moving to the next address), an IPv4 address (also written IPv4-mapped) as it
|
|
281
|
+
is, anything else unchanged."""
|
|
282
|
+
try:
|
|
283
|
+
addr = ipaddress.ip_address(ip)
|
|
284
|
+
except ValueError:
|
|
285
|
+
return ip
|
|
286
|
+
if isinstance(addr, ipaddress.IPv6Address):
|
|
287
|
+
if addr.ipv4_mapped is not None:
|
|
288
|
+
return str(addr.ipv4_mapped)
|
|
289
|
+
return str(ipaddress.IPv6Network((int(addr) >> 64 << 64, 64)))
|
|
290
|
+
return str(addr)
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
class Pool:
|
|
294
|
+
"""Connections (their transports) counted in total and per source address (an IPv6 one
|
|
295
|
+
by its /64: address_key), each up to a cap (0: none). A connection whose transport is
|
|
296
|
+
closing no longer counts."""
|
|
297
|
+
|
|
298
|
+
def __init__(self, total: int = 0, per_ip: int = 0) -> None:
|
|
299
|
+
self.total = total
|
|
300
|
+
self.per_ip = per_ip
|
|
301
|
+
self._members: dict[Any, str] = {} # transport -> address_key of its source
|
|
302
|
+
|
|
303
|
+
def _sweep(self) -> None:
|
|
304
|
+
for t in [t for t in self._members if t.is_closing()]:
|
|
305
|
+
del self._members[t]
|
|
306
|
+
|
|
307
|
+
def __len__(self) -> int:
|
|
308
|
+
self._sweep()
|
|
309
|
+
return len(self._members)
|
|
310
|
+
|
|
311
|
+
def count(self, ip: str) -> int:
|
|
312
|
+
self._sweep()
|
|
313
|
+
key = address_key(ip)
|
|
314
|
+
return sum(1 for member in self._members.values() if member == key)
|
|
315
|
+
|
|
316
|
+
def full(self, ip: str) -> str:
|
|
317
|
+
""""" when one more from ``ip`` fits, else why not."""
|
|
318
|
+
if self.total and len(self) >= self.total:
|
|
319
|
+
return f"{self.total} at once is the limit"
|
|
320
|
+
if self.per_ip and self.count(ip) >= self.per_ip:
|
|
321
|
+
key = address_key(ip)
|
|
322
|
+
source = f"one /64 network ({key})" if key.endswith("/64") else "one address"
|
|
323
|
+
return f"{self.per_ip} at once from {source} is the limit"
|
|
324
|
+
return ""
|
|
325
|
+
|
|
326
|
+
def add(self, transport: Any, ip: str) -> None:
|
|
327
|
+
self._members[transport] = address_key(ip)
|
|
328
|
+
|
|
329
|
+
def discard(self, transport: Any) -> None:
|
|
330
|
+
self._members.pop(transport, None)
|
|
331
|
+
|
|
332
|
+
def members(self) -> list:
|
|
333
|
+
self._sweep()
|
|
334
|
+
return list(self._members)
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
class Budget:
|
|
338
|
+
"""Whether the process takes one more incoming connection, whichever listener it reaches
|
|
339
|
+
(one instance: ``BUDGET``), and the TLS handshakes in progress and waiting."""
|
|
340
|
+
|
|
341
|
+
def __init__(self) -> None:
|
|
342
|
+
self.handshakes = Pool(TLS_HANDSHAKES_MAX, TLS_HANDSHAKES_PER_IP)
|
|
343
|
+
self.waiting = Pool(TLS_WAITING_MAX, TLS_WAITING_PER_IP) # for a handshake to end
|
|
344
|
+
# each waiting connection -> (its address, its turn: True once it has a slot,
|
|
345
|
+
# False when it closed meanwhile); in the order they came
|
|
346
|
+
self._turns: dict[Any, tuple[str, asyncio.Future]] = {}
|
|
347
|
+
self.refused: dict[str, int] = {} # why -> connections refused for it (/metrics)
|
|
348
|
+
# the event loop's selector when it is select() based (Windows; runtime.py sets it)
|
|
349
|
+
self.selector: Optional[CappedSelectSelector] = None
|
|
350
|
+
self._counted = (-1.0, 0) # (when, how many files were open)
|
|
351
|
+
self._admitted = 0 # connections let in since
|
|
352
|
+
|
|
353
|
+
def _files_in_use(self) -> Optional[int]:
|
|
354
|
+
now = time.monotonic()
|
|
355
|
+
if now - self._counted[0] > 0.25: # at most four counts a second, under a flood too
|
|
356
|
+
n = open_fds()
|
|
357
|
+
if n is None:
|
|
358
|
+
return None
|
|
359
|
+
self._counted, self._admitted = (now, n), 0
|
|
360
|
+
return self._counted[1] + self._admitted
|
|
361
|
+
|
|
362
|
+
def headroom(self) -> str:
|
|
363
|
+
""""" when one more connection fits, else why not (for the log)."""
|
|
364
|
+
limit = fd_limit()
|
|
365
|
+
if limit is not None:
|
|
366
|
+
used = self._files_in_use()
|
|
367
|
+
if used is not None and used >= limit - fd_reserve(limit):
|
|
368
|
+
self.note_refused("open_files")
|
|
369
|
+
return (f"{used} of this process's {limit} open files are in use (the rest "
|
|
370
|
+
"is kept for serial ports, logs and the configuration)")
|
|
371
|
+
sel = self.selector
|
|
372
|
+
if sel is not None:
|
|
373
|
+
watched = sel.watched()
|
|
374
|
+
if watched >= sel.limit - SELECT_HEADROOM:
|
|
375
|
+
self.note_refused("event_loop")
|
|
376
|
+
return (f"the event loop watches {watched} sockets, near the {sel.limit} its "
|
|
377
|
+
"select() takes")
|
|
378
|
+
self._admitted += 1
|
|
379
|
+
return ""
|
|
380
|
+
|
|
381
|
+
def handshake_room(self, ip: str) -> str:
|
|
382
|
+
""""" when a TLS handshake from ``ip`` can start, or wait for one to end, else why
|
|
383
|
+
not (for the log)."""
|
|
384
|
+
why = self.handshakes.full(ip)
|
|
385
|
+
if why:
|
|
386
|
+
waiting = self.waiting.full(ip)
|
|
387
|
+
if waiting:
|
|
388
|
+
self.note_refused("tls_handshakes")
|
|
389
|
+
return (f"too many TLS handshakes in progress ({why}) and waiting for one "
|
|
390
|
+
f"to end ({waiting})")
|
|
391
|
+
return ""
|
|
392
|
+
|
|
393
|
+
def enter_handshake(self, transport: Any, ip: str) -> None:
|
|
394
|
+
"""Count an admitted TLS connection among the handshakes in progress, or, when
|
|
395
|
+
there is no room for it, among those waiting for one to end."""
|
|
396
|
+
if self.handshakes.full(ip):
|
|
397
|
+
self.waiting.add(transport, ip)
|
|
398
|
+
self._turns[transport] = (ip, asyncio.get_running_loop().create_future())
|
|
399
|
+
else:
|
|
400
|
+
self.handshakes.add(transport, ip)
|
|
401
|
+
|
|
402
|
+
async def handshake_turn(self, transport: Any) -> bool:
|
|
403
|
+
"""Wait (up to TLS_HANDSHAKE_TIMEOUT_S) until ``transport`` may start its handshake:
|
|
404
|
+
False when no handshake ended for it in that time, or it closed meanwhile."""
|
|
405
|
+
if transport not in self._turns:
|
|
406
|
+
return True # it had room at once
|
|
407
|
+
turn = self._turns[transport][1]
|
|
408
|
+
try:
|
|
409
|
+
await asyncio.wait((turn,), timeout=TLS_HANDSHAKE_TIMEOUT_S)
|
|
410
|
+
finally:
|
|
411
|
+
self._turns.pop(transport, None)
|
|
412
|
+
self.waiting.discard(transport)
|
|
413
|
+
return turn.done() and turn.result()
|
|
414
|
+
|
|
415
|
+
def handshake_done(self, transport: Any) -> None:
|
|
416
|
+
"""A handshake is over (or never started): the connections waiting for one get
|
|
417
|
+
the room there is now, first come, first served."""
|
|
418
|
+
self.handshakes.discard(transport)
|
|
419
|
+
self.waiting.discard(transport)
|
|
420
|
+
for waiter, (ip, turn) in list(self._turns.items()):
|
|
421
|
+
if turn.done():
|
|
422
|
+
continue
|
|
423
|
+
if waiter.is_closing():
|
|
424
|
+
turn.set_result(False)
|
|
425
|
+
elif not self.handshakes.full(ip):
|
|
426
|
+
self.waiting.discard(waiter)
|
|
427
|
+
self.handshakes.add(waiter, ip)
|
|
428
|
+
turn.set_result(True)
|
|
429
|
+
elif self.handshakes.total and len(self.handshakes) >= self.handshakes.total:
|
|
430
|
+
break # no room for anyone
|
|
431
|
+
|
|
432
|
+
def note_refused(self, why: str) -> None:
|
|
433
|
+
self.refused[why] = self.refused.get(why, 0) + 1
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
BUDGET = Budget()
|
|
437
|
+
# why a connection can be refused right after accept (Budget.refused keys, for /metrics)
|
|
438
|
+
REFUSAL_REASONS = ("open_files", "event_loop", "tls_handshakes", "admin_ui")
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
class CappedSelectSelector(selectors.SelectSelector):
|
|
442
|
+
"""The selector of the SelectorEventLoop on Windows. select() takes at most 512 sockets
|
|
443
|
+
in each set there and raises ValueError past that, inside the event loop, which then
|
|
444
|
+
stopped: every bridge with it. This one refuses the registration that would go past
|
|
445
|
+
the limit instead (an OSError for whatever asked for the socket to be watched), and
|
|
446
|
+
tells Budget.headroom how full it is, so incoming connections are refused before that
|
|
447
|
+
(SELECT_HEADROOM)."""
|
|
448
|
+
|
|
449
|
+
def __init__(self, limit: int = SELECT_LIMIT) -> None:
|
|
450
|
+
super().__init__()
|
|
451
|
+
self.limit = limit
|
|
452
|
+
|
|
453
|
+
def _count(self, event: int) -> int:
|
|
454
|
+
return sum(1 for key in self.get_map().values() if key.events & event)
|
|
455
|
+
|
|
456
|
+
def watched(self) -> int:
|
|
457
|
+
"""Sockets in the fuller of the two sets (read, write)."""
|
|
458
|
+
return max(self._count(selectors.EVENT_READ), self._count(selectors.EVENT_WRITE))
|
|
459
|
+
|
|
460
|
+
def _room(self, fileobj: Any, events: int) -> None:
|
|
461
|
+
fd = fileobj if isinstance(fileobj, int) else fileobj.fileno()
|
|
462
|
+
key = self.get_map().get(fd)
|
|
463
|
+
have = key.events if key is not None else 0
|
|
464
|
+
for event in (selectors.EVENT_READ, selectors.EVENT_WRITE):
|
|
465
|
+
if events & event and not have & event and self._count(event) >= self.limit:
|
|
466
|
+
raise OSError(errno.EMFILE, f"the event loop already watches {self.limit} "
|
|
467
|
+
"sockets, the most its select() takes")
|
|
468
|
+
|
|
469
|
+
def register(self, fileobj: Any, events: int, data: Any = None) -> selectors.SelectorKey:
|
|
470
|
+
self._room(fileobj, events)
|
|
471
|
+
return super().register(fileobj, events, data)
|
|
472
|
+
|
|
473
|
+
def modify(self, fileobj: Any, events: int, data: Any = None) -> selectors.SelectorKey:
|
|
474
|
+
self._room(fileobj, events) # before modify() unregisters it
|
|
475
|
+
return super().modify(fileobj, events, data)
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
class _Early(asyncio.Protocol):
|
|
479
|
+
"""What arrives on a TLS connection between the end of its handshake and the hand-over
|
|
480
|
+
to the protocol that serves it (both in one pass of the event loop)."""
|
|
481
|
+
|
|
482
|
+
def __init__(self) -> None:
|
|
483
|
+
self.data = bytearray()
|
|
484
|
+
self.eof = False
|
|
485
|
+
self.lost = False
|
|
486
|
+
|
|
487
|
+
def data_received(self, data: bytes) -> None:
|
|
488
|
+
self.data += data
|
|
489
|
+
|
|
490
|
+
def eof_received(self) -> bool:
|
|
491
|
+
self.eof = True
|
|
492
|
+
return False
|
|
493
|
+
|
|
494
|
+
def connection_lost(self, exc: Optional[BaseException]) -> None:
|
|
495
|
+
self.lost = True
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
async def start_tls(transport: Any, ctx: Any, protocol: asyncio.BaseProtocol) -> bool:
|
|
499
|
+
"""The server side of a TLS handshake on an accepted ``transport`` whose reading the
|
|
500
|
+
caller paused, within TLS_HANDSHAKE_TIMEOUT_S; then ``protocol`` serves the connection
|
|
501
|
+
(its connection_made gets the TLS transport). False when the connection went away
|
|
502
|
+
first; raises what the handshake raised (the transport is closed then)."""
|
|
503
|
+
loop = asyncio.get_running_loop()
|
|
504
|
+
early = _Early()
|
|
505
|
+
if sys.version_info >= (3, 11):
|
|
506
|
+
tls = await loop.start_tls(transport, early, ctx, server_side=True,
|
|
507
|
+
ssl_handshake_timeout=TLS_HANDSHAKE_TIMEOUT_S,
|
|
508
|
+
ssl_shutdown_timeout=TLS_SHUTDOWN_TIMEOUT_S)
|
|
509
|
+
else: # pragma: no cover - exercised only on Python 3.10 (no ssl_shutdown_timeout)
|
|
510
|
+
tls = await loop.start_tls(transport, early, ctx, server_side=True,
|
|
511
|
+
ssl_handshake_timeout=TLS_HANDSHAKE_TIMEOUT_S)
|
|
512
|
+
if tls is None or early.lost or tls.is_closing():
|
|
513
|
+
transport.close()
|
|
514
|
+
return False
|
|
515
|
+
if sys.version_info < (3, 11): # pragma: no cover - Python 3.10 only
|
|
516
|
+
_bound_close(tls, transport)
|
|
517
|
+
tls.set_protocol(protocol)
|
|
518
|
+
protocol.connection_made(tls)
|
|
519
|
+
if early.data and isinstance(protocol, asyncio.Protocol):
|
|
520
|
+
protocol.data_received(bytes(early.data))
|
|
521
|
+
if early.eof and isinstance(protocol, asyncio.Protocol):
|
|
522
|
+
protocol.eof_received()
|
|
523
|
+
return True
|
|
524
|
+
|
|
525
|
+
|
|
526
|
+
def _bound_close(tls: Any, transport: Any) -> None: # pragma: no cover - Python 3.10 only
|
|
527
|
+
"""Python 3.10 has no ssl_shutdown_timeout: closing a TLS connection waits for the
|
|
528
|
+
peer's close_notify for as long as that takes, so a peer that never answers kept the
|
|
529
|
+
socket (and its place in the connection limits) open for good. Closing ``tls`` now
|
|
530
|
+
also aborts ``transport``, the socket under it, TLS_SHUTDOWN_TIMEOUT_S later if it is
|
|
531
|
+
still open (aborting a closed one does nothing)."""
|
|
532
|
+
loop = asyncio.get_running_loop()
|
|
533
|
+
close = tls.close
|
|
534
|
+
|
|
535
|
+
def bounded_close() -> None:
|
|
536
|
+
close()
|
|
537
|
+
if not loop.is_closed(): # (a transport collected after the loop ended)
|
|
538
|
+
loop.call_later(TLS_SHUTDOWN_TIMEOUT_S, transport.abort)
|
|
539
|
+
|
|
540
|
+
tls.close = bounded_close
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
def describe_tls_error(exc: BaseException) -> str:
|
|
544
|
+
text = str(exc) or type(exc).__name__
|
|
545
|
+
return text if isinstance(exc, ConnectionError) else f"{type(exc).__name__}: {text}"
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
def begin_tls(transport: Any, ctx: Any, protocol: asyncio.BaseProtocol, label: str,
|
|
549
|
+
failed: Callable[[str], None],
|
|
550
|
+
refused: Optional[Callable[[str], None]] = None) -> asyncio.Task:
|
|
551
|
+
"""Start the TLS handshake of an admitted connection: counted among the handshakes in
|
|
552
|
+
progress until it is over (or waiting for room among them first), run in a task
|
|
553
|
+
(start_tls) that pauses reading first. A failed handshake is reported to ``failed``, a
|
|
554
|
+
connection that waited in vain to ``refused`` (default: ``failed``; rate-limit both:
|
|
555
|
+
anyone can cause them)."""
|
|
556
|
+
BUDGET.enter_handshake(transport, peer_ip(transport))
|
|
557
|
+
|
|
558
|
+
async def handshake() -> None:
|
|
559
|
+
try:
|
|
560
|
+
# Paused here, not in the caller (a protocol's connection_made): the transport
|
|
561
|
+
# starts reading in a callback of its own after connection_made, and on Python
|
|
562
|
+
# 3.10 that start ignores an earlier pause, so the ClientHello would reach the
|
|
563
|
+
# protocol that admits connections and be lost. The task's first step runs
|
|
564
|
+
# after that callback and before the first read.
|
|
565
|
+
transport.pause_reading() # the ClientHello waits for the TLS protocol
|
|
566
|
+
if not await BUDGET.handshake_turn(transport):
|
|
567
|
+
if not transport.is_closing():
|
|
568
|
+
BUDGET.note_refused("tls_handshakes")
|
|
569
|
+
(refused or failed)(f"refused {label}: it waited "
|
|
570
|
+
f"{TLS_HANDSHAKE_TIMEOUT_S:g} seconds for a TLS "
|
|
571
|
+
"handshake to end")
|
|
572
|
+
transport.abort()
|
|
573
|
+
return
|
|
574
|
+
await start_tls(transport, ctx, protocol)
|
|
575
|
+
except Exception as e: # the timeout, a bad ClientHello, the client went away
|
|
576
|
+
failed(f"TLS handshake with {label} failed: {describe_tls_error(e)}")
|
|
577
|
+
finally:
|
|
578
|
+
BUDGET.handshake_done(transport)
|
|
579
|
+
|
|
580
|
+
return asyncio.get_running_loop().create_task(handshake())
|
|
581
|
+
|
|
582
|
+
|
|
583
|
+
class _Gate(asyncio.Protocol):
|
|
584
|
+
"""A new connection's first protocol: its listener admits it (giving it the protocol
|
|
585
|
+
that serves it) or closes it."""
|
|
586
|
+
|
|
587
|
+
def __init__(self, listener: Listener) -> None:
|
|
588
|
+
self._listener = listener
|
|
589
|
+
|
|
590
|
+
def connection_made(self, transport: asyncio.BaseTransport) -> None:
|
|
591
|
+
self._listener.admit(transport)
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
class Listener:
|
|
595
|
+
"""A mapping's TCP listener (``start_server``); close() / wait_closed() / sockets as on
|
|
596
|
+
the asyncio Server it wraps."""
|
|
597
|
+
|
|
598
|
+
def __init__(self, client_connected_cb: Callable, ssl: Any, allowed: Optional[Callable],
|
|
599
|
+
log: Callable[[str], None], limit: int) -> None:
|
|
600
|
+
self._loop = asyncio.get_running_loop()
|
|
601
|
+
self._cb = client_connected_cb
|
|
602
|
+
self._ssl = ssl
|
|
603
|
+
self._allowed = allowed
|
|
604
|
+
self._log = log
|
|
605
|
+
self._limit = limit
|
|
606
|
+
self._refused = LogEvery(log)
|
|
607
|
+
self._failed = LogEvery(log)
|
|
608
|
+
self._rejected = RefusalLog(log)
|
|
609
|
+
self._handshakes: dict[Any, asyncio.Task] = {} # plain transport -> its handshake
|
|
610
|
+
self.server: Any = None
|
|
611
|
+
|
|
612
|
+
@property
|
|
613
|
+
def sockets(self):
|
|
614
|
+
return self.server.sockets
|
|
615
|
+
|
|
616
|
+
@property
|
|
617
|
+
def ssl(self) -> Any:
|
|
618
|
+
"""The TLS context new connections get (None: plain TCP)."""
|
|
619
|
+
return self._ssl
|
|
620
|
+
|
|
621
|
+
@ssl.setter
|
|
622
|
+
def ssl(self, context: Any) -> None: # a renewed certificate (ser2netweb/tlsreload.py)
|
|
623
|
+
self._ssl = context
|
|
624
|
+
|
|
625
|
+
def close(self) -> None:
|
|
626
|
+
self.server.close()
|
|
627
|
+
for transport, task in list(self._handshakes.items()):
|
|
628
|
+
transport.abort()
|
|
629
|
+
task.cancel() # (one waiting for a handshake to end, too)
|
|
630
|
+
|
|
631
|
+
async def wait_closed(self) -> None:
|
|
632
|
+
await self.server.wait_closed()
|
|
633
|
+
|
|
634
|
+
def _stream_protocol(self) -> asyncio.StreamReaderProtocol:
|
|
635
|
+
reader = asyncio.StreamReader(limit=self._limit, loop=self._loop)
|
|
636
|
+
return asyncio.StreamReaderProtocol(reader, self._cb, loop=self._loop)
|
|
637
|
+
|
|
638
|
+
def admit(self, transport: Any) -> None:
|
|
639
|
+
label = peer_label(transport)
|
|
640
|
+
why = BUDGET.headroom()
|
|
641
|
+
if not why and self._ssl is not None:
|
|
642
|
+
ip = peer_ip(transport)
|
|
643
|
+
if self._allowed is not None and not self._allowed(ip):
|
|
644
|
+
self._rejected(f"rejected {label}: not in allowed list")
|
|
645
|
+
transport.close()
|
|
646
|
+
return
|
|
647
|
+
why = BUDGET.handshake_room(ip)
|
|
648
|
+
if why:
|
|
649
|
+
self._refused(f"refused {label}: {why}")
|
|
650
|
+
transport.close()
|
|
651
|
+
return
|
|
652
|
+
protocol = self._stream_protocol()
|
|
653
|
+
if self._ssl is None:
|
|
654
|
+
transport.set_protocol(protocol)
|
|
655
|
+
protocol.connection_made(transport)
|
|
656
|
+
return
|
|
657
|
+
task = begin_tls(transport, self._ssl, protocol, label, self._failed, self._refused)
|
|
658
|
+
self._handshakes[transport] = task
|
|
659
|
+
task.add_done_callback(lambda _t: self._handshakes.pop(transport, None))
|
|
660
|
+
|
|
661
|
+
|
|
662
|
+
def listen_hosts(net: NetworkSettings) -> list[str]:
|
|
663
|
+
"""The addresses a mapping's TCP listener binds: both wildcards when it takes IPv4 and
|
|
664
|
+
IPv6 clients (NetworkSettings.both_families; 0.0.0.0 alone where Python has no IPv6),
|
|
665
|
+
else its bind IP."""
|
|
666
|
+
if net.both_families:
|
|
667
|
+
return list(WILDCARDS) if socket.has_ipv6 else ["0.0.0.0"]
|
|
668
|
+
return [net.bind_ip]
|
|
669
|
+
|
|
670
|
+
|
|
671
|
+
def listen_where(net: NetworkSettings, sockets: Sequence[socket.socket]) -> str:
|
|
672
|
+
"""Where a mapping listens, for its log line: its address and port, and which clients
|
|
673
|
+
that takes when it is a wildcard of one family or both (``sockets``: what it got)."""
|
|
674
|
+
if net.both_families:
|
|
675
|
+
families = {sock.family for sock in sockets}
|
|
676
|
+
if families == {socket.AF_INET6} and _takes_ipv4(sockets[0]):
|
|
677
|
+
families.add(socket.AF_INET) # (UDP: one socket for both)
|
|
678
|
+
if families == {socket.AF_INET, socket.AF_INET6}:
|
|
679
|
+
return f"{host_port('0.0.0.0', net.port)} and {host_port('::', net.port)}, IPv4 and IPv6"
|
|
680
|
+
if families == {socket.AF_INET}:
|
|
681
|
+
return f"{host_port('0.0.0.0', net.port)}, IPv4 only (IPv6 is not available here)"
|
|
682
|
+
where = host_port(net.bind_ip, net.port)
|
|
683
|
+
if canonical_ip(net.bind_ip) == "::" and not net.both_families:
|
|
684
|
+
where += ", IPv6 clients only"
|
|
685
|
+
return where
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
def _takes_ipv4(sock: socket.socket) -> bool:
|
|
689
|
+
try:
|
|
690
|
+
return not sock.getsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY)
|
|
691
|
+
except (OSError, AttributeError):
|
|
692
|
+
return False
|
|
693
|
+
|
|
694
|
+
|
|
695
|
+
async def udp_socket(net: NetworkSettings) -> socket.socket:
|
|
696
|
+
"""A UDP mapping's socket, bound to its address and port. IPV6_V6ONLY is set on an IPv6
|
|
697
|
+
one as the mapping says, not left to the OS (Linux would let a socket on :: take IPv4
|
|
698
|
+
senders too, Windows not): one that takes both families (NetworkSettings.both_families)
|
|
699
|
+
is one socket on :: with it off (where this machine has no IPv6, an IPv4 one on
|
|
700
|
+
0.0.0.0), any other has it on, so :: takes IPv6 senders only, as a TCP listener does.
|
|
701
|
+
Raises BindError when the address and port can't be taken."""
|
|
702
|
+
both = net.both_families
|
|
703
|
+
address: Any
|
|
704
|
+
try:
|
|
705
|
+
if both:
|
|
706
|
+
family, address = socket.AF_INET6, ("::", net.port)
|
|
707
|
+
try:
|
|
708
|
+
sock = socket.socket(family, socket.SOCK_DGRAM)
|
|
709
|
+
except (OSError, AttributeError): # no IPv6 here
|
|
710
|
+
family, address = socket.AF_INET, ("0.0.0.0", net.port)
|
|
711
|
+
sock = socket.socket(family, socket.SOCK_DGRAM)
|
|
712
|
+
else:
|
|
713
|
+
infos = await asyncio.get_running_loop().getaddrinfo(
|
|
714
|
+
net.bind_ip, net.port, type=socket.SOCK_DGRAM,
|
|
715
|
+
flags=socket.AI_PASSIVE | socket.AI_NUMERICHOST)
|
|
716
|
+
family, kind, proto, _name, address = infos[0]
|
|
717
|
+
sock = socket.socket(family, kind, proto)
|
|
718
|
+
except OSError as e:
|
|
719
|
+
raise BindError.of(e) from e
|
|
720
|
+
try:
|
|
721
|
+
if family == socket.AF_INET6 and hasattr(socket, "IPPROTO_IPV6"):
|
|
722
|
+
sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0 if both else 1)
|
|
723
|
+
if net.udp_broadcast is True: # (udp_targets: a broadcast address)
|
|
724
|
+
sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
|
|
725
|
+
sock.bind(address)
|
|
726
|
+
sock.setblocking(False)
|
|
727
|
+
except OSError as e:
|
|
728
|
+
sock.close()
|
|
729
|
+
raise BindError.of(e) from e
|
|
730
|
+
return sock
|
|
731
|
+
|
|
732
|
+
|
|
733
|
+
async def start_server(client_connected_cb: Callable, host: Union[str, Sequence[str]], port: int,
|
|
734
|
+
*, ssl: Any = None,
|
|
735
|
+
allowed: Optional[Callable[[str], bool]] = None,
|
|
736
|
+
log: Optional[Callable[[str], None]] = None,
|
|
737
|
+
limit: int = 2 ** 16) -> Listener:
|
|
738
|
+
"""asyncio.start_server(client_connected_cb, host, port, ssl=ssl) (``host``: one
|
|
739
|
+
address, or several: listen_hosts), admitting each connection right after accept: the
|
|
740
|
+
process's budget, then (TLS) ``allowed`` (the mapping's allow-list) and the handshakes
|
|
741
|
+
in progress, before the handshake.
|
|
742
|
+
``client_connected_cb(reader, writer)`` gets TLS connections once their handshake is
|
|
743
|
+
done. ``log``: where refusals and failed handshakes are reported (the mapping's log).
|
|
744
|
+
Raises BindError when the address and port can't be taken."""
|
|
745
|
+
listener = Listener(client_connected_cb, ssl, allowed, log or (lambda _m: None), limit)
|
|
746
|
+
try:
|
|
747
|
+
server = await asyncio.get_running_loop().create_server(
|
|
748
|
+
lambda: _Gate(listener), host, port, start_serving=False)
|
|
749
|
+
except OSError as e:
|
|
750
|
+
raise BindError.of(e) from e
|
|
751
|
+
clash = _reserved_clash(server.sockets)
|
|
752
|
+
if clash:
|
|
753
|
+
server.close()
|
|
754
|
+
raise BindError.of(OSError(errno.EADDRINUSE, clash))
|
|
755
|
+
listener.server = server
|
|
756
|
+
await server.start_serving()
|
|
757
|
+
return listener
|
|
758
|
+
|
|
759
|
+
|
|
760
|
+
# Sockets that later listeners may not share an address and port with (reserve), with what
|
|
761
|
+
# holds them. Windows lets a listener on a wildcard address share a port with one on a
|
|
762
|
+
# specific address of its family (that one keeps its address's connections, the other gets
|
|
763
|
+
# the rest) and, without SO_EXCLUSIVEADDRUSE on the first, the reverse too; Linux refuses
|
|
764
|
+
# the second bind while the first listens. start_server refuses it on every OS, so a mapping
|
|
765
|
+
# on the admin UI's address and port fails (and is retried) instead of running next to it.
|
|
766
|
+
_reserved: weakref.WeakKeyDictionary[socket.socket, str] = weakref.WeakKeyDictionary()
|
|
767
|
+
|
|
768
|
+
|
|
769
|
+
def reserve(sockets: Sequence[socket.socket], holder: str) -> None:
|
|
770
|
+
"""Keep the addresses and ports of ``sockets`` (bound) from the listeners start_server
|
|
771
|
+
makes afterwards while they are open; ``holder`` names them in the error ("the admin
|
|
772
|
+
UI")."""
|
|
773
|
+
for sock in sockets:
|
|
774
|
+
_reserved[sock] = holder
|
|
775
|
+
|
|
776
|
+
|
|
777
|
+
def _reserved_clash(sockets: Sequence[Any]) -> str:
|
|
778
|
+
"""Why one of ``sockets`` (bound, not listening yet) can't listen: it shares the port of
|
|
779
|
+
a reserved socket on an overlapping address (the same one, or a wildcard of its family);
|
|
780
|
+
"" when none does."""
|
|
781
|
+
for sock in sockets:
|
|
782
|
+
try:
|
|
783
|
+
ip, port = sock.getsockname()[:2]
|
|
784
|
+
except OSError:
|
|
785
|
+
continue
|
|
786
|
+
for held, holder in list(_reserved.items()):
|
|
787
|
+
if held.fileno() == -1 or held.family != sock.family:
|
|
788
|
+
continue
|
|
789
|
+
try:
|
|
790
|
+
held_ip, held_port = held.getsockname()[:2]
|
|
791
|
+
except OSError:
|
|
792
|
+
continue
|
|
793
|
+
if held_port == port and _overlaps(held_ip, ip):
|
|
794
|
+
return (f"error while attempting to bind on address {host_port(ip, port)}: "
|
|
795
|
+
f"address already in use ({holder} listens on "
|
|
796
|
+
f"{host_port(held_ip, held_port)})")
|
|
797
|
+
return ""
|
|
798
|
+
|
|
799
|
+
|
|
800
|
+
def _overlaps(a: str, b: str) -> bool:
|
|
801
|
+
try:
|
|
802
|
+
x, y = ipaddress.ip_address(a), ipaddress.ip_address(b)
|
|
803
|
+
except ValueError:
|
|
804
|
+
return a == b
|
|
805
|
+
return x == y or x.is_unspecified or y.is_unspecified
|