factorysemantics-mes 0.3.1__tar.gz → 0.4.0__tar.gz

factorysemantics_mes-0.4.0/.github/scripts/two_nodes_on_one_box.py

@@ -0,0 +1,369 @@
+#!/usr/bin/env python
+"""Two nodes of one plant on one box, started and stopped by the fleet.
+
+Scott, 2026-10-15, about the release this is part of: *"must run on
+windows."* Nobody on this project has a Windows machine, so this script is
+how the two-node machinery is read on one: a pack that names its nodes, one
+PostgreSQL, one machine layer, and the three fleet verbs a person types.
+
+What it proves, in the order it does it:
+
+1. `fsmes pack check` accepts a pack that names two nodes.
+2. `fsmes fleet create` builds the plant once - two nodes are one plant and
+   one database - and records that this installation owns it.
+3. `fsmes fleet start` starts **both** nodes, each with its own node name,
+   its own port, its own pid file and its own log directory.
+4. Both nodes answer `/health`, each saying its own name.
+5. Every role - the reader, the floor, the housekeeping - is held by exactly
+   one of them, and `/health` on either node says the same thing about who
+   holds what, because the holders come from the database both nodes share.
+6. `fsmes fleet status` says two nodes answered and names what each holds.
+7. `fsmes fleet stop` stops both, and neither port answers afterwards.
+
+What it does **not** prove is the kill: the takeover, the lease units and the
+counters that must not be booked twice are proven by
+`tests/test_a_plant_on_two_nodes_survives_one_of_them_dying.py` and by
+`scripts/kill-a-node.sh`, both on Linux. `Popen.terminate()` on Windows does
+not go through the supervisor's own shutdown, so a killed node there is a
+different mechanism from a killed node here and deserves its own proof
+rather than a claim made by this script.
+
+It runs on either platform - it is proven on Linux before it is pushed, and
+it is the Windows runner it exists for.
+
+    python .github/scripts/two_nodes_on_one_box.py \
+        --database-url postgresql+psycopg://user:pw@127.0.0.1:5432/two_nodes
+
+Every path goes through `pathlib`, every child's output is read as UTF-8, and
+nothing here uses a shell, a signal or a process group: the three things that
+are not the same on both platforms.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+#: The variable the pack names for the thing that signs sessions. Both nodes
+#: read the same one: a person signed in on one node is signed in on the
+#: plant, and two of them would sign that person out every time the proxy
+#: sent them to the other node.
+#:
+#: Its name says `signer` and not `secret` on purpose. The pack names the
+#: variable and never the value, so this constant is a variable name and
+#: nothing private - but code scanning reads a name with `secret` in it as a
+#: credential, and a scanner that cries wolf on a CI script is a scanner
+#: nobody reads on the day it is right (`py/clear-text-storage-sensitive-data`,
+#: PR #178).
+SIGNING_VARIABLE = "FSMES_TWO_NODE_SIGNER"
+#: Short enough that a role is claimed while this script is watching, long
+#: enough that a busy runner does not drop a renewal. Every wait below is a
+#: multiple of it, never a number of seconds chosen for its own sake.
+LEASE_SECONDS = 6.0
+ROLES = ("reader", "floor", "housekeeping")
+#: This interpreter's own `fsmes`. Not the console script: on Windows that
+#: lives in a `Scripts` directory that is on PATH only if the installer put
+#: it there, and `python -m fsmes` is the same program either way.
+MES = [sys.executable, "-m", "fsmes"]
+
+
+def say(line: str = "") -> None:
+    print(line, flush=True)
+
+
+def fail(why: str) -> None:
+    say("")
+    say(f"NOT PROVEN: {why}")
+    raise SystemExit(1)
+
+
+# ------------------------------------------------------------------ the pack
+
+def pack_with_two_nodes(where: Path, *, database_url: str, port_a: int, port_b: int,
+                        opc_endpoint: str) -> Path:
+    """The lab's bottling pack with its nodes named, and nothing else changed.
+
+    Patched in the four places that make it this script's: where the machine
+    layer is, where the database is, that it runs on two nodes, and the name
+    of the variable its signing key lives in. The replay data is the
+    reference hour beside it, which generates in a moment.
+    """
+    pack = where / "bottling"
+    shutil.copytree(ROOT / "labs" / "multiplant" / "bottling", pack)
+    shutil.copy(ROOT / "labs" / "kepsim" / "line.json", pack / "line.json")
+
+    toml = (pack / "plant.toml").read_text(encoding="utf-8")
+    toml = toml.replace('opc_endpoint = "opc.tcp://127.0.0.1:4841/fsmes/bottling"',
+                        f'opc_endpoint = "{opc_endpoint}"')
+    toml = toml.replace("api_port     = 8010", f"api_port     = {port_a}")
+    toml = toml.replace("simulate     = true",
+                        f'simulate     = true\nsecret_key_env = "{SIGNING_VARIABLE}"\n\n'
+                        f'[[serve.nodes]]\nid = "a"\napi_port = {port_a}\n\n'
+                        f'[[serve.nodes]]\nid = "b"\napi_port = {port_b}\n')
+    # A pack a person writes names `database_password_file` and keeps the
+    # password out of the file that says what the plant is. This one splices
+    # the whole URL because that is what it was given, into a directory that
+    # is deleted at the end.
+    toml += (f'\n[storage]\ndatabase_url = "{database_url}"\n'
+             f'\n[redundancy]\nenabled = true\nlease_seconds = {LEASE_SECONDS}\n')
+    (pack / "plant.toml").write_text(toml, encoding="utf-8")
+    return pack
+
+
+# --------------------------------------------------------------- the asking
+
+def get(url: str, timeout: float = 4.0) -> tuple[int, str]:
+    try:
+        with urllib.request.urlopen(url, timeout=timeout) as answer:
+            return answer.status, answer.read().decode("utf-8", "replace")
+    except urllib.error.HTTPError as refused:
+        return refused.code, ""
+    except Exception:
+        return 0, ""
+
+
+def health(port: int) -> dict:
+    status, body = get(f"http://127.0.0.1:{port}/health")
+    if status != 200 or not body:
+        return {}
+    try:
+        return json.loads(body)
+    except ValueError:
+        return {}
+
+
+def answers(port: int, timeout: float) -> dict:
+    """Wait for one node's dashboard to answer, and say what it said."""
+    until = time.monotonic() + timeout
+    while time.monotonic() < until:
+        body = health(port)
+        if body:
+            return body
+        time.sleep(0.5)
+    return {}
+
+
+def holders(body: dict) -> dict:
+    """role -> the node holding it, as `/health` reports it."""
+    node = body.get("node") or {}
+    return {role: (node.get("holders") or {}).get(role) for role in ROLES}
+
+
+def wait_for_every_role(port: int, timeout: float) -> dict:
+    """Wait until every role has a holder, and say who holds what.
+
+    Each role is a race of its own - the reader, the floor and the
+    housekeeping are claimed by three different processes - so this waits for
+    all three to be claimed rather than for one node to claim all three.
+    """
+    until = time.monotonic() + timeout
+    held: dict = {}
+    while time.monotonic() < until:
+        held = holders(health(port))
+        if all(held.get(role) for role in ROLES):
+            return held
+        time.sleep(0.5)
+    return held
+
+
+# ------------------------------------------------------------- the commands
+
+def fsmes(*args: str, env: dict, where: Path | None = None, timeout: float = 600.0):
+    """One `fsmes` command, its output read as UTF-8 and printed."""
+    done = subprocess.run([*MES, *args], cwd=where or ROOT, env=env, capture_output=True,
+                          text=True, encoding="utf-8", errors="replace", timeout=timeout)
+    for line in (done.stdout or "").splitlines():
+        say(f"    | {line}")
+    if done.returncode != 0:
+        for line in (done.stderr or "").splitlines():
+            say(f"    ! {line}")
+    return done
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--database-url", required=True,
+                        help="a PostgreSQL both nodes share; the database must exist")
+    parser.add_argument("--port-a", type=int, default=8010)
+    parser.add_argument("--port-b", type=int, default=8011)
+    parser.add_argument("--opc-port", type=int, default=4841)
+    parser.add_argument("--speed", type=float, default=60.0,
+                        help="line seconds per wall second for the replay")
+    parser.add_argument("--workdir", default="",
+                        help="where the pack, the data and the logs go (default: a temp dir)")
+    parser.add_argument("--keep-running", action="store_true",
+                        help="leave both nodes up at the end, for a person to look at")
+    args = parser.parse_args()
+
+    if "postgresql" not in args.database_url:
+        fail("two nodes share one database, which a file database cannot be: "
+             "--database-url must be a PostgreSQL URL")
+
+    where = Path(args.workdir).resolve() if args.workdir else Path(tempfile.mkdtemp(
+        prefix="fsmes-two-nodes-"))
+    where.mkdir(parents=True, exist_ok=True)
+    root = where / "run"
+    root.mkdir(parents=True, exist_ok=True)
+    logs = where / "logs"
+    logs.mkdir(parents=True, exist_ok=True)
+    endpoint = f"opc.tcp://127.0.0.1:{args.opc_port}/fsmes/bottling-two-nodes"
+
+    env = dict(os.environ)
+    env[SIGNING_VARIABLE] = "the-same-signer-on-both-nodes"
+    env["PYTHONIOENCODING"] = "utf-8"
+
+    say(f"Two nodes of one plant on {sys.platform}, in {where}")
+    pack = pack_with_two_nodes(where, database_url=args.database_url, port_a=args.port_a,
+                              port_b=args.port_b, opc_endpoint=endpoint)
+    say("  the line data the replay plays")
+    if fsmes("sim-generate", str(pack / "line.json"), "--out", str(pack / "out"),
+             env=env).returncode != 0:
+        fail("the line data would not generate")
+
+    say("  fsmes pack check, on a pack that names two nodes")
+    if fsmes("pack", "check", str(pack), env=env).returncode != 0:
+        fail("the checker refused a pack naming two nodes")
+
+    replay = None
+    started_nodes = False
+    replay_log = logs / "replay.log"
+    try:
+        say("  the machine layer: one OPC replay for the whole plant")
+        sim_env = dict(env)
+        sim_env.update(MES_PLANT_NAME="bottling", MES_OPC_ENDPOINT=endpoint,
+                       MES_REPLAY_DIR=str(pack / "out"),
+                       MES_TAG_MAP_FILE=str(pack / "tag_map.json"),
+                       MES_SIM_SPEED=str(args.speed), MES_LOG_DIR=str(logs))
+        with replay_log.open("ab") as sink:
+            replay = subprocess.Popen([*MES, "run-opc-sim", "--replay"], cwd=ROOT, env=sim_env,
+                                      stdout=sink, stderr=subprocess.STDOUT)
+        until = time.monotonic() + 90
+        while time.monotonic() < until:
+            if "replay online" in replay_log.read_text(encoding="utf-8", errors="replace"):
+                break
+            if replay.poll() is not None:
+                fail(f"the machine layer exited ({replay.returncode}); see {replay_log}")
+            time.sleep(0.25)
+        else:
+            fail(f"the machine layer never came online; see {replay_log}")
+        say("    | replay online")
+
+        say("  fsmes fleet create, once, for one plant on two nodes")
+        if fsmes("fleet", "create", str(pack), "--root", str(root), env=env).returncode != 0:
+            fail("the plant would not build from a pack naming two nodes")
+
+        say("  fsmes fleet start, which starts every node the pack names")
+        started = fsmes("fleet", "start", "bottling", "--root", str(root),
+                        "--speed", str(args.speed), env=env)
+        started_nodes = True
+        if started.returncode != 0:
+            fail("fleet start refused a plant with two nodes")
+        for node in ("a", "b"):
+            if f"node {node}" not in started.stdout:
+                fail(f"fleet start said nothing about node {node}:\n{started.stdout}")
+
+        say("  both nodes answer /health, each with its own name")
+        seen = {}
+        for node, port in (("a", args.port_a), ("b", args.port_b)):
+            body = answers(port, timeout=180)
+            if not body:
+                fail(f"node {node} never answered on port {port}")
+            said = (body.get("node") or {}).get("name")
+            if said != node:
+                fail(f"the node on port {port} calls itself {said!r}, not {node!r}")
+            seen[node] = body
+            say(f"    | node {node} answered on {port}, redundancy "
+                f"{(body.get('node') or {}).get('redundancy')}")
+
+        say("  every role is held by exactly one node, and both nodes agree")
+        held = wait_for_every_role(args.port_a, timeout=LEASE_SECONDS * 20)
+        missing = [role for role in ROLES if not held.get(role)]
+        if missing:
+            fail(f"nobody held {', '.join(missing)} within 20 leases; "
+                 f"what was held: {held}")
+        for role, holder in held.items():
+            if holder not in ("a", "b"):
+                fail(f"{role} is held by {holder!r}, which is neither node")
+            say(f"    | {role} is held by node {holder}")
+        from_b = holders(health(args.port_b))
+        if from_b != held:
+            fail(f"the nodes disagree about who holds what: a says {held}, b says {from_b}")
+        say("    | node b says the same, because the holders are in the database")
+
+        say("  fsmes fleet status names both nodes and what they hold")
+        status = fsmes("fleet", "status", "bottling", "--root", str(root), env=env)
+        if status.returncode != 0:
+            fail("fleet status refused a plant with two nodes")
+        if not re.search(r"nodes\s+2, 2 answered", status.stdout):
+            fail(f"fleet status did not say two nodes answered:\n{status.stdout}")
+        for node in ("a", "b"):
+            if not re.search(rf"^\s+{node}\s+\S+.*holding ", status.stdout, re.MULTILINE):
+                fail(f"fleet status said nothing about node {node}:\n{status.stdout}")
+
+        say("  fsmes fleet stop stops both")
+        if args.keep_running:
+            say(f"    | --keep-running: both nodes are up on {args.port_a} and "
+                f"{args.port_b}, the machine layer is still playing on "
+                f"{endpoint} (pid {replay.pid}), and {root} is theirs. Stop "
+                f"them with `fsmes fleet stop bottling --root {root}`, and "
+                f"the machine layer by its pid.")
+            return 0
+        stopped = fsmes("fleet", "stop", "bottling", "--root", str(root), env=env)
+        if stopped.returncode != 0:
+            fail("fleet stop refused a plant with two nodes")
+        started_nodes = False
+        for node, port in (("a", args.port_a), ("b", args.port_b)):
+            if f"node {node}" not in stopped.stdout:
+                fail(f"fleet stop said nothing about node {node}:\n{stopped.stdout}")
+            until = time.monotonic() + 60
+            while time.monotonic() < until and health(port):
+                time.sleep(0.5)
+            if health(port):
+                fail(f"node {node} was still answering on {port} after fleet stop")
+            say(f"    | node {node} is gone from {port}")
+
+        say("")
+        say("PROVEN on this box: a pack that names two nodes is checked, built once, "
+            "started as two nodes that each answer for themselves, has every role held "
+            "by exactly one of them, and is stopped by one command. The kill is proven "
+            "on Linux and is not claimed here.")
+        return 0
+    finally:
+        # A failure before the stop step leaves two nodes answering, which on
+        # a runner is a job that never ends and on a person's box is two
+        # plants they did not ask for. `--keep-running` is the one case where
+        # they are meant to stay up.
+        if started_nodes and not args.keep_running:
+            fsmes("fleet", "stop", "bottling", "--root", str(root), env=env, timeout=120)
+        if replay is not None and replay.poll() is None and not args.keep_running:
+            # `--keep-running` keeps the machine layer too: two nodes with
+            # nothing to read are a plant whose machines all say
+            # *disconnected*, which is not what the person asked to look at.
+            replay.terminate()
+            try:
+                replay.wait(timeout=30)
+            except subprocess.TimeoutExpired:
+                replay.kill()
+        for log in sorted(logs.rglob("*.log")) + sorted((root / "logs").rglob("*.jsonl")):
+            say("")
+            say(f"--- the last 40 lines of {log}")
+            try:
+                for line in log.read_text(encoding="utf-8", errors="replace").splitlines()[-40:]:
+                    say(f"    {line}")
+            except OSError as unreadable:
+                say(f"    (unreadable: {unreadable})")
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

{factorysemantics_mes-0.3.1 → factorysemantics_mes-0.4.0}/.github/workflows/ci.yml

@@ -493,6 +493,128 @@ jobs:
             esac
           done
 
+  two-nodes-in-compose:
+    # The two-box demo, killed on purpose, on every pull request.
+    #
+    # `docker/docker-compose.redundant.yml` is one PostgreSQL, one machine
+    # layer, two nodes of the bottling plant and nginx in front of them, and
+    # `scripts/kill-a-node.sh` kills one node and checks the other carried
+    # on: the reader lease moved within three leases, the counters kept
+    # moving, every machine booked exactly what it counted, no reading was
+    # stored twice, and the one address kept answering. It is the same claim
+    # `postgres`'s two-node test makes, made through containers and a proxy
+    # instead of two processes - which is what anybody who tries this product
+    # will actually run.
+    #
+    # Linux only. `docker compose` on the Windows runner has no Linux daemon
+    # to run these images on, and a plant engineer's Windows box is covered by
+    # `half-dead-plant` and `two-nodes-on-windows` instead.
+    #
+    # Behind `image`: there is nothing to learn from a compose demo of an
+    # image that does not build, and the layer cache that job wrote makes the
+    # build here a cache hit rather than a second four-minute pip install.
+    # The timing this job asserts is in lease units measured against the
+    # database's own clock, never in seconds on a shared runner.
+    needs: image
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    steps:
+      - uses: actions/checkout@v7
+      - uses: docker/setup-buildx-action@v4
+      - name: Build the image the compose file names
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          file: docker/Dockerfile
+          platforms: linux/amd64
+          push: false
+          load: true
+          tags: factorysemantics-mes:two-nodes
+          # Read-only: `image` owns this scope, and two jobs writing one
+          # scope evict each other's layers.
+          cache-from: type=gha,scope=ci-image
+
+      - name: Kill one of two nodes and check the plant carried on
+        run: bash scripts/kill-a-node.sh
+
+      - name: What the containers said
+        # Always, pass or fail: a failover nobody can read the logs of is a
+        # green check or a red one and nothing else.
+        if: always()
+        run: |
+          docker compose -f docker/docker-compose.redundant.yml ps -a || true
+          docker compose -f docker/docker-compose.redundant.yml logs --no-color --tail 200 || true
+
+      - name: Put it away
+        if: always()
+        run: docker compose -f docker/docker-compose.redundant.yml down -v || true
+
+  two-nodes-on-windows:
+    # Scott, about this release: "must run on windows." Nobody on this
+    # project has a Windows machine, so this job is where the two-node
+    # machinery is read on one.
+    #
+    # It names its nodes in the pack, builds the plant once, starts it with
+    # `fsmes fleet start`, and checks that both nodes answer `/health` under
+    # their own names, that each of the three roles is held by exactly one of
+    # them, that `fleet status` names both, and that `fleet stop` stops both.
+    # The PostgreSQL is the one the runner image ships - installed and
+    # stopped, so the job starts it.
+    #
+    # What this job does NOT do is kill a node. `Popen.terminate()` on
+    # Windows does not go through the plant's own shutdown handler, so a
+    # killed node there is a different mechanism from a killed node on Linux
+    # and deserves a proof of its own rather than a claim made here. The
+    # lease, the takeover in lease units and the counters that must not be
+    # booked twice are proven by the `postgres` job and by
+    # `two-nodes-in-compose`, both Linux.
+    #
+    # Windows only: the Linux half of the same claim is the `postgres` job,
+    # which runs the whole two-node failover test, and running this script
+    # there as well would buy the weaker half of a stronger proof.
+    runs-on: windows-latest
+    timeout-minutes: 40
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: "3.12"
+
+      - name: Start the PostgreSQL this runner ships
+        shell: pwsh
+        run: |
+          $service = Get-Service -Name "postgresql*" | Select-Object -First 1
+          if (-not $service) { throw "this runner image has no PostgreSQL service" }
+          Write-Host "service $($service.Name) is $($service.Status)"
+          Set-Service -Name $service.Name -StartupType Manual
+          Start-Service -Name $service.Name
+          $env:PGPASSWORD = "root"
+          $ready = $false
+          foreach ($try in 1..30) {
+            & "$env:PGBIN\pg_isready.exe" -h 127.0.0.1 -p 5432 -U postgres
+            if ($LASTEXITCODE -eq 0) { $ready = $true; break }
+            Start-Sleep -Seconds 2
+          }
+          if (-not $ready) { throw "PostgreSQL never accepted a connection" }
+          & "$env:PGBIN\createdb.exe" -h 127.0.0.1 -p 5432 -U postgres two_nodes
+          if ($LASTEXITCODE -ne 0) { throw "could not create the database" }
+
+      - name: Install
+        run: python -m pip install --upgrade pip && python -m pip install -e ".[postgres]"
+
+      - name: Two nodes of one plant, started and stopped by the fleet
+        shell: bash
+        env:
+          # The runner image's own password for its own PostgreSQL. It is in
+          # the image's documentation, it is reachable from this job and from
+          # nothing else, and the database it opens is deleted with the
+          # runner.
+          PGPASSWORD: root
+        run: |
+          python .github/scripts/two_nodes_on_one_box.py \
+            --database-url "postgresql+psycopg://postgres:root@127.0.0.1:5432/two_nodes" \
+            --speed 60
+
   lab:
     # The lab's starter experiments, on a runner.
     #

{factorysemantics_mes-0.3.1 → factorysemantics_mes-0.4.0}/CHANGELOG.md

@@ -12,6 +12,163 @@ goes under Honesty with a migration line, so plant people can find it.
 
 Nothing yet.
 
+## [0.4.0] — 2026-10-16
+
+### Added
+
+- **A plant can run on two boxes and keep counting when one of them dies.**
+  Two or more **nodes** of one plant against one PostgreSQL, with the pack
+  saying so:
+
+  ```toml
+  [redundancy]
+  enabled       = true
+  lease_seconds = 10
+  ```
+
+  Three jobs that must run once — reading the machines, the simulated floor,
+  and pruning the history — are held one at a time by a row in a new
+  `plant_leases` table, claimed and renewed against the **database's** clock
+  and never the box's. A holder that cannot renew stops at once and comes
+  back as a standby; the other node has the job within a lease or two. The
+  dashboard on the surviving node keeps answering, a person signed in on one
+  node is signed in on the other (the pack names the variable holding the
+  signing key, `secret_key_env`, and a plant with `[redundancy]` on refuses
+  to start without it), and production is booked **exactly once** — the
+  counters are absolute, and a new `counter_baselines` table carries each
+  one's baseline in the same transaction as the booking it belongs to, so a
+  node taking over continues the count rather than starting it again.
+  `fsmes plant <name> run --node <id> --api-port <port>` names a node and
+  moves it off the plant's own port; `/health` grows a `node` block saying
+  which node answered, how long its lease is, what it holds and who holds
+  the rest; `fsmes plant <name> status` says the same as a line. PostgreSQL
+  only: `[redundancy]` with a SQLite URL refuses in one sentence, and the
+  single-box SQLite demo is untouched. Measured in lease units on one
+  machine, not in seconds. What it does **not** do is in
+  [one plant on two boxes](docs/operate/redundancy.md) and decision 0042:
+  the front door is yours (two nodes are two addresses), the database's own
+  standby is PostgreSQL's job, the PLC write path stays single, ERP sync and
+  the UNS publisher are still one copy each, and a conversation on the AI tab
+  starts over after a failover. (#177)
+
+- **The two-box demo, and the kill test anyone can run.**
+  `docker compose -f docker/docker-compose.redundant.yml up` brings up one
+  PostgreSQL, the simulated machines, two nodes of one plant and nginx in
+  front of both on one address; `bash scripts/kill-a-node.sh` brings it up,
+  waits for the counts to move, kills a node outright, watches the one
+  address until the other node holds the reader, and prints the takeover in
+  lease units with a check that nothing was booked twice and nothing was
+  lost. A CI job (`two-nodes-in-compose`) runs that same script on every
+  pull request. On this product's own box on 2026-10-16 the reader was on the
+  second node after 1.13 leases of 6 s, 0 of the requests asked during the
+  failover were refused, all 6 machines booked exactly what their counters
+  moved, and 0 readings were booked twice. (#178)
+
+- **The screens say which box holds what.** The line beside the plant's name
+  on the dashboard — *served by b · machines read by a* — used to be written
+  once when the page loaded, which left it naming a dead node until somebody
+  reloaded; it now follows a takeover on the page's own poll, within five
+  seconds and with no reload, and only on a plant that says it runs on more
+  than one node (a single-node plant starts no timer and makes no request it
+  did not make before). The [fleet console](docs/operate/fleet.md) shows such
+  a plant once with its nodes under it — how many there are and how many
+  answered, then one line each saying where that node is and what it holds in
+  words — and `fsmes fleet status`, `fsmes fleet list` and `/fleet.json`
+  carry the same. A node that did not answer is **unknown** there, never a
+  node holding nothing. (#178)
+
+- **A pack may name the nodes that run on one box**, which is the cheap way
+  to see all of this before you own the second box:
+
+  ```toml
+  [[serve.nodes]]
+  id       = "a"
+  api_port = 8010
+  ```
+
+  `fsmes fleet start`, `stop`, `status`, `list` and `apply` then act on every
+  node the pack names, each with its own pid file and its own log directory,
+  and `fsmes pack check` refuses the four ways a node list half-starts a
+  plant (two nodes sharing a port or a name, nodes named with `[redundancy]`
+  off, no signing key between them). A pack that names no nodes is the
+  ordinary plant on one address and behaves exactly as it did. (#178)
+
+- **`fsmes fleet start` starts the machines a two-node simulated plant
+  shares, and `fleet stop` stops them.** A simulated plant on one node starts
+  its own OPC replay; a pack that named nodes deliberately did not, because a
+  replay started inside a node dies with that node and the surviving reader
+  then sees the line start over — which reads as a PLC reset and books
+  nothing. Nobody started it instead: after the four commands in
+  [one plant on two boxes](docs/operate/redundancy.md) the console said
+  *6 of 6 disconnected* and nothing was booked. The plant now starts it once,
+  before either node, with a log directory of its own; it is left playing
+  when it is already playing, both when this installation's own pid file
+  names it and when anything else already answers on the endpoint the pack
+  names, so a replay you started by hand — or the one `docker compose` owns —
+  is never stopped by `fsmes fleet stop`. A plant whose pack says
+  `simulate = false` starts nothing here: its machine layer is a PLC somebody
+  else runs.
+
+- **`fsmes fleet stop <plant> --node <id>`** stops one node of a plant and
+  leaves the others and the shared machines running, for taking one box down
+  on purpose. It is a graceful stop — the node is asked, then told, and it
+  lets go of its leases on the way out — so it is not the proof that a plant
+  survives a box dying: that proof is a `kill`, and the docs say which is
+  which. A node the pack does not name is refused by name, with the nodes it
+  does have.
+
+### Fixed
+
+- **A plant whose first node died could not be stopped.** `fsmes fleet stop`
+  asked one address — the one in the ownership record — and counted a pid
+  file a plant with nodes never writes, so a plant running happily on its
+  second node refused to stop at all: *nothing in its pid file is alive, so
+  there may be nothing left to stop*, with three live processes. `stop`,
+  `apply` and `start` now ask every node the pack names, and a refusal counts
+  every node's pid file and says how many it read. (#178)
+
+- **The console's row for a plant running on its second node was `unknown`
+  in every column** — no label, no schema, no machines, no orders — because
+  the row was filled from the address in the pack, which was the node that
+  had died. A plant with nodes is now read at whichever node answers, and the
+  silent node is still named as silent on its own line underneath. The node
+  that answered is asked the same ownership question as the first address, so
+  a port answering as something else is refused rather than believed. (#178)
+
+- **`fsmes fleet apply` to a book with nothing in it leaves accounts in it.**
+  Pointing an existing pack at a new PostgreSQL ran the migrations and seeded
+  the plant's master data and stopped there: every sign-in answered *invalid
+  credentials*, and on a simulated plant both floor processes died saying
+  *could not sign in* with nothing saying why. `apply` now makes the accounts
+  `fsmes fleet create` makes, in the one case where it cannot be taking
+  anybody's decision away — a book with no account in it at all — and says on
+  a line of its own that it did. A plant that has accounts is left alone:
+  `apply` is not the command that brings back one you deleted. And the
+  floor's own line now names the likeliest cause, which is either a plant
+  that answered and refused the credentials or a plant that never answered
+  at all.
+
+### Honesty
+
+- **A restart no longer re-books what was already booked, on one box or
+  two.** Production is counted from absolute machine counters, and until now
+  the baseline each counter was measured from lived in the reader's memory:
+  a reader that restarted treated the counter's whole value as new
+  production. One migration, `b6f0c2a71d58`, adds `plant_leases` and
+  `counter_baselines`; nothing to do on a single-box plant, where the two
+  tables sit unused until `[redundancy]` is turned on.
+
+- **Windows: what is and is not proven.** A CI job
+  (`two-nodes-on-windows`) builds a two-node pack, checks it, creates it,
+  starts it and stops it, and holds that both nodes answer under their own
+  names and that each of the three roles is held by exactly one of them. The
+  **kill is not proven on Windows**: `Popen.terminate()` there does not go
+  through the plant's own shutdown handler, so a node killed that way stops
+  by a different mechanism than a node whose power is cut, and the takeover
+  that follows is a different thing to measure. The lease, the takeover in
+  lease units and the counts that must not be booked twice are proven on
+  Linux only.
+
 ## [0.3.1] — 2026-10-15
 
 ### Added
@@ -4907,7 +5064,8 @@ number in it comes from a simulated plant.
   MES was not watching. Unlabelled downtime is reported as unlabelled. These
   are the house rules and they are tested.
 
-[Unreleased]: https://github.com/factorysemantics/factorysemantics-mes/compare/v0.3.1...HEAD
+[Unreleased]: https://github.com/factorysemantics/factorysemantics-mes/compare/v0.4.0...HEAD
+[0.4.0]: https://github.com/factorysemantics/factorysemantics-mes/releases/tag/v0.4.0
 [0.3.1]: https://github.com/factorysemantics/factorysemantics-mes/releases/tag/v0.3.1
 [0.3.0]: https://github.com/factorysemantics/factorysemantics-mes/releases/tag/v0.3.0
 [0.2.0]: https://github.com/factorysemantics/factorysemantics-mes/releases/tag/v0.2.0