ser2netweb 3.0.0__py3-none-any.whl

ser2netweb/storage.py

@@ -0,0 +1,130 @@
+"""What ser2netweb keeps on disk: the free space it needs, and the clock its retention goes by.
+
+Traces can fill a disk (up to trace_max_mb x (1 + trace_keep) per mapping), and a full
+disk keeps config.json, the logs and the audit log from being written. Below ``floor()``
+free space on the data folder's disk (max(256 MiB, 5% of the disk), at most 1 GiB, and at
+most a quarter of a small disk) a trace pauses (``check_space``, in its writer thread) and
+goes on by itself once there is 10% more than that again; /metrics reports the free space
+and what traces and logs take (``usage()``), and the optional disk_low notification says
+when it gets low (notify.Notifier).
+
+Retention deletes by age (the mapping logs' 15 days, Settings -> Retention). ``now()`` is
+the time it compares with: the system clock, but never ahead of the system clock at
+start-up plus the time this process has run (measured with a clock no step changes). A
+clock stepped ahead while ser2netweb runs (a typo, a bad time source) so deletes nothing
+early; one stepped back is followed at once (that only keeps more). After a restart the
+system clock counts again: by then the step is either corrected or a real one.
+"""
+from __future__ import annotations
+
+import errno
+import os
+import shutil
+import stat
+import sys
+import time
+
+FREE_FLOOR_MIN = 256 * 1024 * 1024
+FREE_FLOOR_SHARE = 0.05
+FREE_FLOOR_MAX = 1024 * 1024 * 1024
+FREE_RESUME = 1.1  # a trace paused for space goes on above floor() x this
+
+
+class LowSpace(OSError):
+    """Too little free space on the disk to write a trace (check_space)."""
+
+
+def space(path: str) -> tuple[int, int] | None:
+    """(free, total) bytes of the file system `path` is on (free: what this process may
+    use, without the part reserved for root), or None when that can't be read."""
+    try:
+        usage = shutil.disk_usage(path)
+    except (OSError, ValueError):
+        return None
+    return usage.free, usage.total
+
+
+def floor(total: int) -> int:
+    """The free space below which traces pause on a disk of `total` bytes."""
+    return min(max(FREE_FLOOR_MIN, min(int(total * FREE_FLOOR_SHARE), FREE_FLOOR_MAX)),
+               total // 4)
+
+
+def mb(n: float) -> str:
+    return f"{n / 1048576:.0f} MB"
+
+
+def size_text(n: float) -> str:
+    """A size as the UI shows it: '1.2 GB', '340 MB', '12 KB', '0 KB'."""
+    if n >= 1024 ** 3:
+        return f"{n / 1024 ** 3:.1f} GB"
+    return mb(n) if n >= 1048576 else f"{n / 1024:.0f} KB"
+
+
+def check_space(path: str, resuming: bool = False) -> None:
+    """Raise LowSpace when the disk `path` is on has less free space than floor() (with
+    `resuming`: than FREE_RESUME times that, so that a trace paused for it doesn't flap);
+    nothing when there is enough, or when that can't be told."""
+    found = space(path)
+    if found is None:
+        return
+    free, total = found
+    need = floor(total)
+    if free < (need * FREE_RESUME if resuming else need):
+        raise LowSpace(errno.ENOSPC, f"the disk is almost full ({mb(free)} free; traces "
+                                     f"pause below {mb(need)})")
+
+
+def folder_bytes(folder: str, prefixes=None) -> int:
+    """The size of the regular files in `folder` (not in its subfolders; with `prefixes`,
+    only those whose name begins with one of them). Blocking."""
+    total = 0
+    try:
+        with os.scandir(folder) as entries:
+            for entry in entries:
+                if prefixes is not None and not entry.name.startswith(tuple(prefixes)):
+                    continue
+                try:
+                    st = entry.stat(follow_symlinks=False)
+                except OSError:
+                    continue
+                if stat.S_ISREG(st.st_mode):
+                    total += st.st_size
+    except OSError:
+        pass
+    return total
+
+
+def usage(data_dir: str) -> dict:
+    """The data folder's disk and what ser2netweb keeps there: free and total bytes of the
+    disk (None when they can't be read), the floor traces pause below, and the bytes of the
+    traces (traces/) and of the logs (logs/, all.log*, audit.log*). Blocking: a few
+    directory listings."""
+    found = space(data_dir)
+    free, total = found if found is not None else (None, None)
+    return {"free": free, "total": total, "floor": floor(total) if total else None,
+            "traces": folder_bytes(os.path.join(data_dir, "traces")),
+            "logs": folder_bytes(os.path.join(data_dir, "logs"))
+            + folder_bytes(data_dir, ("all.log", "audit.log"))}
+
+
+def _elapsed() -> float:
+    """Seconds since an arbitrary start that no clock step changes: CLOCK_BOOTTIME on
+    Linux (it also counts a suspend), else time.monotonic()."""
+    if sys.platform != "win32":  # (no clock_gettime there)
+        boottime = getattr(time, "CLOCK_BOOTTIME", None)
+        if boottime is not None:
+            try:
+                return time.clock_gettime(boottime)
+            except OSError:
+                pass
+    return time.monotonic()
+
+
+_STARTED = (time.time(), _elapsed())
+
+
+def now() -> float:
+    """The time retention compares ages with (see the module docstring)."""
+    wall, elapsed = _STARTED
+    return min(time.time(), wall + (_elapsed() - elapsed))

ser2netweb/textfold.py

@@ -0,0 +1,51 @@
+"""How text compares where letter case or accents don't count, and how names sort.
+
+* fold(): one name, whatever its letter case (LDAP / OIDC group and user names, account
+  names) — Turkish İ, I, ı and i are one letter, accents count (Çorlu is not Corlu);
+* search_key(): what a search compares (the dashboard filter, the audit log search) —
+  also without accents and compatibility forms, so 'istanbul' finds 'İstanbul Depo' and
+  'corlu' finds 'Çorlu'. app.js folds what is typed into the dashboard filter the same
+  way (searchKey); the rows' text is folded here, on the server;
+* natural_key(): a sort key where digit runs count as numbers (COM2 before COM10,
+  'Hat 2' before 'Hat 10').
+
+str.lower() is no use for either: it turns 'İ' into 'i' + U+0307 and leaves 'ı' as it
+is, so KALİTE and Kalite, BAKIM and Bakım, or 'istanbul' and 'İstanbul' would differ.
+"""
+from __future__ import annotations
+
+import re
+import unicodedata
+
+
+def fold(text) -> str:
+    """How names compare where letter case doesn't count (LDAP and OIDC group names, LDAP
+    user names, an LDAP/OIDC sign-in name or a new account's name against the existing
+    ones): NFC, every run of white space (no-break spaces too) as one space, and letter
+    case folded so that Turkish İ, I, ı and i are one letter — str.lower() turns 'İ'
+    into 'i' + U+0307 and leaves 'ı' as it is, so KALİTE and Kalite, or BAKIM and Bakım,
+    would differ."""
+    s = " ".join(unicodedata.normalize("NFC", str(text)).split())
+    return s.replace("\u0130", "i").replace("\u0131", "i").casefold().replace("i\u0307", "i")
+
+
+def search_key(text) -> str:
+    """`text` as a search compares it: compatibility forms spelled out and accents left
+    off (NFKD without its combining marks: Ç is C, Ş is S, fi is fi, a no-break space a
+    space), then fold(). A search for search_key(typed) in search_key(text) finds
+    'İstanbul Depo' by 'istanbul', 'IŞIK KONTROL' by 'ışık' or 'isik', 'KAZAN DAİRESİ'
+    by 'dairesi'. app.js does the same to what is typed (searchKey)."""
+    s = unicodedata.normalize("NFKD", str(text))
+    return fold("".join(c for c in s if not unicodedata.category(c).startswith("M")))
+
+
+_DIGITS = re.compile(r"([0-9]+)")
+
+
+def natural_key(text) -> tuple:
+    """A sort key for names with numbers in them: digit runs compare as numbers (COM1,
+    COM2, COM10; /dev/ttyS2 before /dev/ttyS10), the rest as search_key() spells it
+    (letter case and accents don't count), and the text itself last, so equal keys
+    still sort one way."""
+    parts = _DIGITS.split(search_key(text))   # text, digits, text, ... (text first)
+    return (tuple(int(p) if i % 2 else p for i, p in enumerate(parts)), str(text))

ser2netweb/tlsreload.py

@@ -0,0 +1,37 @@
+"""TLS certificates that change while ser2netweb runs: new connections get a context built from
+the settings and files as they are now, connections already open keep theirs, and no
+listener is restarted (the admin UI's AdminGuard and a mapping's netguard.Listener read
+their context at each accept).
+
+``prepare(build)`` is the one place a context is made for that: ``build()`` makes it from
+the settings and their files (it raises when they can't be used) and runs in a worker
+thread; the caller hands the result to the listener only when it is usable.
+"""
+from __future__ import annotations
+
+import asyncio
+import ssl
+from dataclasses import dataclass
+from typing import Callable, Optional
+
+
+@dataclass
+class Prepared:
+    """What prepare() made: a context to use (None: no TLS), or why there is none."""
+    context: Optional[ssl.SSLContext] = None
+    error: str = ""   # the files can't be used (the change is refused, the context in use stays)
+    later: str = ""   # usable only from the next start (no listener in this process)
+    exc: Optional[BaseException] = None   # what made `error` (engine.tls.TlsFileError: the
+    #                                       Settings page explains it in its language)
+
+    @property
+    def usable(self) -> bool:
+        return not (self.error or self.later)
+
+
+async def prepare(build: Callable[[], Optional[ssl.SSLContext]]) -> Prepared:
+    """A new TLS context from ``build()`` (None for no TLS), made in a worker thread."""
+    try:
+        return Prepared(await asyncio.to_thread(build))
+    except (OSError, ValueError) as e:  # (engine.tls.TlsFileError is a ValueError)  # ssl.SSLError is an OSError
+        return Prepared(error=str(e) or type(e).__name__, exc=e)

ser2netweb/watchdog.py

@@ -0,0 +1,201 @@
+"""Event loop watchdog: a stuck event loop ends the process, so it is started again.
+
+Every bridge, the Modbus gateway, MQTT, the cluster and the web UI run on one asyncio
+event loop. When a call blocks it (a write to a device, file or console that never
+returns), all of them stop, but the process lives on: systemd, Docker and the Windows
+Service Control Manager only act when it exits, a failing /healthz only marks a container
+"unhealthy", and paho-mqtt's own thread keeps the MQTT sessions alive, so the 'online'
+status and the Sparkplug death certificate are never sent and SCADA sees stale values as
+live ones.
+
+A heartbeat task on the loop notes the time every BEAT_S seconds. A daemon thread checks
+it: once the loop has missed its heartbeat for ``limit`` seconds (SER2NETWEB_LOOP_WATCHDOG,
+default DEFAULT_LIMIT_S, 0 = off), it writes every thread's stack (faulthandler) to
+<data>/hang.log, which names the call that blocks, and ends the process with EXIT_CODE;
+the service manager (restart policy) starts it again, and the next start reports it.
+
+Why 90 s: a loop that is busy, however much, still runs its heartbeat within about a
+second, because its work is cut into short callbacks; only a loop that never gets back to
+them misses it. The longest legitimate holds known are seconds (a log write under heavy
+disk pressure, swapping on a small board), so 90 s is an order of magnitude more, and
+above what a Docker health check (3 tries 30 s apart) or a Kubernetes liveness probe
+waits. /healthz itself stays a plain liveness answer ("ok" while the loop runs), so a
+slow but working instance is never reported unhealthy for being busy.
+
+The thread measures with time.monotonic() and time.sleep() (not Event.wait). When its own
+sleep took much longer than asked, the whole process was frozen (the machine slept, the
+container was paused, a VM was suspended) and the loop could not have run either: it
+starts measuring again instead of ending a process that is fine. A loop that keeps
+running but spins (a callback that reschedules itself at once) still beats and is not
+caught; neither is a stall in native code that holds Python's global interpreter lock
+(it looks like a frozen process). Standard library only.
+"""
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import faulthandler
+import os
+import sys
+import threading
+import time
+from typing import Callable, Mapping, Optional
+
+from . import clock, envvars, sdnotify
+
+ENV = "SER2NETWEB_LOOP_WATCHDOG"  # (or SER2NET_LOOP_WATCHDOG, its 2.9.x name)
+DEFAULT_LIMIT_S = 90.0
+BEAT_S = 1.0         # the heartbeat's interval on the loop
+CHECK_S = 1.0        # how often the thread looks at it
+FROZEN_S = 5.0       # the thread's sleep took this much longer: the process was frozen
+SLOW_BEAT_S = 2.0    # a heartbeat this late is logged (at most once a minute)
+EXIT_CODE = 70       # EX_SOFTWARE
+EXIT_AFTER_S = 10.0  # the exit comes this long after the limit at the latest
+HANG_LOG = "hang.log"
+
+
+def limit_from_env(env: Optional[Mapping[str, str]] = None) -> tuple[float, str]:
+    """(seconds, a note for the start-up log or ""): SER2NETWEB_LOOP_WATCHDOG, 0 = off."""
+    raw = envvars.text(ENV, env)
+    if not raw:
+        return DEFAULT_LIMIT_S, ""
+    try:
+        value = float(raw)
+    except ValueError:
+        value = -1.0
+    if not 0 <= value <= 86400:  # (NaN too)
+        return DEFAULT_LIMIT_S, (f"WARNING: {envvars.source(ENV, env)}={raw!r} is not a "
+                                 f"number of seconds "
+                                 f"(0 = off, at most 86400) — the event loop watchdog uses "
+                                 f"{DEFAULT_LIMIT_S:g} s")
+    return value, ""
+
+
+def previous_hang(data_dir: str) -> str:
+    """When the previous run was ended by the watchdog (hang.log is there): its hang.log is
+    kept as hang.log.1 and the start-up log line about it is returned, else ""."""
+    path = os.path.join(data_dir, HANG_LOG)
+    try:
+        with open(path, encoding="utf-8", errors="replace") as fh:
+            first = fh.readline().strip()
+        os.replace(path, path + ".1")
+    except OSError:
+        return ""
+    return (f"WARNING: the previous run was ended by the event loop watchdog ({first}) — "
+            f"the threads' stacks are in {path}.1")
+
+
+class LoopWatchdog:
+    """The heartbeat (a task on the loop) and the thread that watches it. ``lag`` is how
+    late the last heartbeat was (/metrics), ``slow_beats`` how many were SLOW_BEAT_S late."""
+
+    def __init__(self, data_dir: str, log: Callable[[str], None], limit_s: float,
+                 exit_process: Callable[[int], None] = os._exit) -> None:
+        self.data_dir = data_dir
+        self.log = log
+        self.limit_s = limit_s
+        self.lag = 0.0
+        self.slow_beats = 0
+        self._exit = exit_process
+        self.beat = time.monotonic()
+        self._stop = threading.Event()
+        self._thread: Optional[threading.Thread] = None
+        self._task: Optional[asyncio.Task] = None
+        self._said = 0.0   # time.monotonic() of the last "slow heartbeat" line
+        self._ping_s = sdnotify.watchdog_interval()
+        self._pinged = 0.0
+
+    @property
+    def hang_log(self) -> str:
+        return os.path.join(self.data_dir, HANG_LOG)
+
+    def start(self) -> None:
+        """Begin the heartbeat (on the running loop) and, with a limit, the watching thread."""
+        self.beat = time.monotonic()
+        self._task = asyncio.get_running_loop().create_task(self._heartbeat(),
+                                                            name="loop-heartbeat")
+        if self.limit_s > 0:
+            self._thread = threading.Thread(target=self._watch, name="loop-watchdog",
+                                            daemon=True)
+            self._thread.start()
+
+    async def stop(self) -> None:
+        self._stop.set()
+        if self._task is not None:
+            self._task.cancel()
+            with contextlib.suppress(asyncio.CancelledError):
+                await self._task
+            self._task = None
+
+    async def _heartbeat(self) -> None:
+        try:
+            await self._beats()
+        finally:  # cancelled: the loop ends (also on an error before stop()), so does the thread
+            self._stop.set()
+
+    async def _beats(self) -> None:
+        while True:
+            asked = time.monotonic()
+            await asyncio.sleep(BEAT_S)
+            now = time.monotonic()
+            self.beat = now
+            self.lag = max(0.0, now - asked - BEAT_S)
+            if self.lag >= SLOW_BEAT_S:
+                self.slow_beats += 1
+                if now - self._said >= 60:
+                    self._said = now
+                    self.log(f"event loop: held up for {self.lag:.1f} s (the mappings and "
+                             "the web UI waited meanwhile)")
+            if self._ping_s is not None and now - self._pinged >= self._ping_s:
+                self._pinged = now
+                sdnotify.watchdog()
+
+    def _watch(self) -> None:
+        last = time.monotonic()
+        while not self._stop.is_set():
+            time.sleep(CHECK_S)
+            now = time.monotonic()
+            if now - last > CHECK_S + FROZEN_S:
+                self.beat = max(self.beat, now)  # the process was frozen: measure anew
+            last = now
+            if now - self.beat > self.limit_s and not self._stop.is_set():
+                self._fire(now - self.beat)
+                return
+
+    def _fire(self, stuck: float) -> None:
+        """Write every thread's stack to hang.log, then end the process (after
+        EXIT_AFTER_S at the latest, should writing block too)."""
+        backstop = threading.Timer(EXIT_AFTER_S, self._exit, (EXIT_CODE,))
+        backstop.daemon = True
+        backstop.start()
+        line = (f"{clock.stamp()} the event loop did not run for "
+                f"{stuck:.0f} s (limit {self.limit_s:g} s, {ENV})")
+        with contextlib.suppress(Exception):
+            with open(self.hang_log, "w", encoding="utf-8") as fh:
+                fh.write(line + "\n\n")
+                fh.flush()
+                faulthandler.dump_traceback(file=fh, all_threads=True)
+        with contextlib.suppress(Exception):
+            # straight to the file and to stderr: the log's own handlers may be what blocks
+            with open(os.path.join(self.data_dir, "all.log"), "a", encoding="utf-8") as fh:
+                fh.write(f"{line} — ser2netweb exits ({EXIT_CODE}) to be started again; the "
+                         f"threads' stacks are in {self.hang_log}\n")
+        with contextlib.suppress(Exception):
+            print(f"ser2netweb: {line} — exiting; see {self.hang_log}", file=sys.stderr, flush=True)
+        self._exit(EXIT_CODE)
+
+
+def start(data_dir: str, log: Callable[[str], None],
+          env: Optional[Mapping[str, str]] = None) -> LoopWatchdog:
+    """The watchdog for the running loop, started; its start-up line is logged."""
+    limit, note = limit_from_env(env)
+    if note:
+        log(note)
+    dog = LoopWatchdog(data_dir, log, limit)
+    dog.start()
+    if limit > 0:
+        log(f"event loop watchdog: on — ser2netweb exits (to be started again) when its event "
+            f"loop does not run for {limit:g} s ({ENV}=0: off)")
+    else:
+        log(f"event loop watchdog: off ({ENV}=0)")
+    return dog