cswap-pin 0.1.332__tar.gz → 0.1.333__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cswap_pin-0.1.332 → cswap_pin-0.1.333}/PKG-INFO +1 -1
- {cswap_pin-0.1.332 → cswap_pin-0.1.333}/pyproject.toml +1 -1
- {cswap_pin-0.1.332 → cswap_pin-0.1.333}/src/cswap_pin/proxy.py +88 -2
- {cswap_pin-0.1.332 → cswap_pin-0.1.333}/tests/test_proxy.py +426 -0
- {cswap_pin-0.1.332 → cswap_pin-0.1.333}/uv.lock +1 -1
- {cswap_pin-0.1.332 → cswap_pin-0.1.333}/.githooks/pre-push +0 -0
- {cswap_pin-0.1.332 → cswap_pin-0.1.333}/.github/workflows/ci.yml +0 -0
- {cswap_pin-0.1.332 → cswap_pin-0.1.333}/.github/workflows/publish.yml +0 -0
- {cswap_pin-0.1.332 → cswap_pin-0.1.333}/.github/workflows/tag-on-version-bump.yml +0 -0
- {cswap_pin-0.1.332 → cswap_pin-0.1.333}/.gitignore +0 -0
- {cswap_pin-0.1.332 → cswap_pin-0.1.333}/README.md +0 -0
- {cswap_pin-0.1.332 → cswap_pin-0.1.333}/src/cswap_pin/__init__.py +0 -0
- {cswap_pin-0.1.332 → cswap_pin-0.1.333}/src/cswap_pin/_host.py +0 -0
- {cswap_pin-0.1.332 → cswap_pin-0.1.333}/tests/conftest.py +0 -0
- {cswap_pin-0.1.332 → cswap_pin-0.1.333}/tests/leakwatch.py +0 -0
- {cswap_pin-0.1.332 → cswap_pin-0.1.333}/tests/models/at_limit_conversion.pict +0 -0
- {cswap_pin-0.1.332 → cswap_pin-0.1.333}/tests/models/at_limit_conversion.tsv +0 -0
- {cswap_pin-0.1.332 → cswap_pin-0.1.333}/tests/test_meta.py +0 -0
- {cswap_pin-0.1.332 → cswap_pin-0.1.333}/tests/test_packaging.py +0 -0
- {cswap_pin-0.1.332 → cswap_pin-0.1.333}/tests/test_pin_run.py +0 -0
- {cswap_pin-0.1.332 → cswap_pin-0.1.333}/tests/test_proxy_server.py +0 -0
- {cswap_pin-0.1.332 → cswap_pin-0.1.333}/tests/test_publish_guard.py +0 -0
- {cswap_pin-0.1.332 → cswap_pin-0.1.333}/tests/test_session_name_adopt.py +0 -0
- {cswap_pin-0.1.332 → cswap_pin-0.1.333}/tools/thread_probe.py +0 -0
- {cswap_pin-0.1.332 → cswap_pin-0.1.333}/tools/upgrade_probe.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: cswap-pin
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.333
|
|
4
4
|
Summary: Keep Claude Code's Remote Control and Artifacts on one account while inference follows cswap's account swap
|
|
5
5
|
Project-URL: Homepage, https://github.com/codeslake/cswap-pin
|
|
6
6
|
Project-URL: Repository, https://github.com/codeslake/cswap-pin
|
|
@@ -12116,6 +12116,10 @@ class PortHolder:
|
|
|
12116
12116
|
if handed is not None:
|
|
12117
12117
|
self._handed_off = handed
|
|
12118
12118
|
return
|
|
12119
|
+
# THE DAEMON'S WAIT STARTS WITH THIS ASK: it polls `proxy.json` for
|
|
12120
|
+
# `_SPAWN_WAIT_S`, then falls back to exit 75 and a capped drain.
|
|
12121
|
+
deadline = time.monotonic() + _SPAWN_WAIT_S
|
|
12122
|
+
before = getattr(self, "_proc", None)
|
|
12119
12123
|
try:
|
|
12120
12124
|
self._spawn()
|
|
12121
12125
|
except (OSError, ValueError) as exc:
|
|
@@ -12123,6 +12127,83 @@ class PortHolder:
|
|
|
12123
12127
|
f"could not spawn a successor on replace request: {exc!r} "
|
|
12124
12128
|
f"— staying up on the current daemon"
|
|
12125
12129
|
)
|
|
12130
|
+
return
|
|
12131
|
+
# ONLY THE DAEMON'S OWN ASK IS WAITING ON A RECORD. `_wait_for_exit`'s
|
|
12132
|
+
# wedge repair (no signal) replaces a daemon that asked nothing and
|
|
12133
|
+
# terminates it next, and the supervisor takes the successor from there.
|
|
12134
|
+
if signum is not None and self._proc is not before:
|
|
12135
|
+
# RuntimeError IS HOW `Thread.start` REPORTS EAGAIN / RLIMIT_NPROC,
|
|
12136
|
+
# the exhaustion the `_spawn` guard above contains: escaping this
|
|
12137
|
+
# SIGNAL HANDLER it ends the holder with no `stop()`. Unwatched is
|
|
12138
|
+
# what the successor was before T2071, and the daemon's exit-75
|
|
12139
|
+
# path follows.
|
|
12140
|
+
try:
|
|
12141
|
+
threading.Thread(
|
|
12142
|
+
target=self._respawn_until_published,
|
|
12143
|
+
args=(self._proc, deadline), daemon=True).start()
|
|
12144
|
+
except RuntimeError as exc:
|
|
12145
|
+
_log_lifecycle(
|
|
12146
|
+
f"could not start the successor watch: {exc!r} "
|
|
12147
|
+
f"— the successor goes unwatched")
|
|
12148
|
+
|
|
12149
|
+
def _respawn_until_published(self, proc, deadline: float) -> None:
|
|
12150
|
+
"""Watch the successor ``proc`` that `_on_replace_request` just spawned
|
|
12151
|
+
and spawn it again if it dies before `proxy.json` names its pid (T2071).
|
|
12152
|
+
|
|
12153
|
+
NOBODY ELSE WATCHES IT. `_supervise` waits on the daemon that asked,
|
|
12154
|
+
which polls for a record until ``deadline``, then releases the
|
|
12155
|
+
listener, drains capped and exits 75 (4397803 chose that over a
|
|
12156
|
+
keep-serving retry, and it stays). A successor that died at once, on
|
|
12157
|
+
`ModuleNotFoundError` while `uv tool install --force` rebuilt the venv
|
|
12158
|
+
under it, was never retried before that: dkim Mac, 0.1.329 -> 0.1.331,
|
|
12159
|
+
"cut 5 in-flight request(s) after 30.0s".
|
|
12160
|
+
|
|
12161
|
+
ONLY A DEAD CHILD IS REPLACED, never a live one that has yet to
|
|
12162
|
+
publish: two daemons on one socket is the bug this class refuses to
|
|
12163
|
+
have. A clean exit is a decision (`_supervise_locked`) and ends the
|
|
12164
|
+
watch. A respawn is made under `_replace_lock`, and only while
|
|
12165
|
+
`self._proc` is still ``proc``: `stop()`, the supervisor (the daemon
|
|
12166
|
+
left first), a later ask or a hand-off of the socket wins. `self._failures` is not climbed: its
|
|
12167
|
+
cap would `degrade_now()` beside a daemon that still serves, and the
|
|
12168
|
+
ladder starts at the daemon's exit as it always did. `_backoff` paces
|
|
12169
|
+
the respawns (0.5 s, 1 s, 2 s, ...) and none starts that could not
|
|
12170
|
+
begin before ``deadline``, so a window longer than the wait still ends
|
|
12171
|
+
on the daemon's own fallback.
|
|
12172
|
+
"""
|
|
12173
|
+
import subprocess
|
|
12174
|
+
|
|
12175
|
+
certdir = getattr(self, "_certdir", None)
|
|
12176
|
+
attempt = 0
|
|
12177
|
+
while certdir is not None and not self._stop:
|
|
12178
|
+
rec = read_daemon_state(certdir)
|
|
12179
|
+
if rec and int(rec.get("pid") or 0) == proc.pid:
|
|
12180
|
+
return
|
|
12181
|
+
try:
|
|
12182
|
+
code = proc.wait(timeout=0.1)
|
|
12183
|
+
except subprocess.TimeoutExpired:
|
|
12184
|
+
if time.monotonic() >= deadline:
|
|
12185
|
+
return
|
|
12186
|
+
continue
|
|
12187
|
+
attempt += 1
|
|
12188
|
+
wait = self._backoff(attempt)
|
|
12189
|
+
if code == 0 or time.monotonic() + wait >= deadline:
|
|
12190
|
+
return
|
|
12191
|
+
_log_lifecycle(
|
|
12192
|
+
f"successor {proc.pid} exited (code {code}) before "
|
|
12193
|
+
f"publishing — spawning it again in {wait:.1f}s")
|
|
12194
|
+
time.sleep(wait)
|
|
12195
|
+
with self._replace_lock:
|
|
12196
|
+
if (self._stop or self._proc is not proc or self._handed_off
|
|
12197
|
+
or getattr(self, "_degraded", False)
|
|
12198
|
+
or time.monotonic() >= deadline):
|
|
12199
|
+
return
|
|
12200
|
+
try:
|
|
12201
|
+
self._spawn()
|
|
12202
|
+
except (OSError, ValueError) as exc:
|
|
12203
|
+
_log_lifecycle(f"could not spawn the successor again: "
|
|
12204
|
+
f"{exc!r}")
|
|
12205
|
+
return
|
|
12206
|
+
proc = self._proc
|
|
12126
12207
|
|
|
12127
12208
|
def _supervise(self) -> None:
|
|
12128
12209
|
"""ONE THREAD, ONE LOOP (T1193): ordinary supervision and the
|
|
@@ -12341,10 +12422,15 @@ class PortHolder:
|
|
|
12341
12422
|
# return — can move `self._proc` away from what this
|
|
12342
12423
|
# iteration captured.
|
|
12343
12424
|
if self._proc is not proc:
|
|
12425
|
+
# THE SUCCESSOR MAY HAVE DIED BEFORE ITS PREDECESSOR LEFT (T2071):
|
|
12426
|
+
# then nothing serves, and the next lap supervises its exit.
|
|
12427
|
+
died = getattr(self._proc, "poll", lambda: None)()
|
|
12344
12428
|
_log_lifecycle(
|
|
12345
12429
|
f"daemon {proc.pid} retired (exit {code}) after "
|
|
12346
|
-
f"handing over —
|
|
12347
|
-
f"{self.
|
|
12430
|
+
f"handing over — "
|
|
12431
|
+
+ (f"successor {self._proc.pid} already exited ({died}), "
|
|
12432
|
+
f"supervising that exit from here" if died is not None
|
|
12433
|
+
else f"successor already serving on port {self.port}")
|
|
12348
12434
|
)
|
|
12349
12435
|
return "continue"
|
|
12350
12436
|
# THE SOCKET WENT TO A CALLER'S LINEAGE AND ANOTHER DAEMON SERVES IT:
|
|
@@ -34758,3 +34758,429 @@ class TestTheDaemonRaisesItsFdLimit:
|
|
|
34758
34758
|
assert len(lines) == 2, out.stderr
|
|
34759
34759
|
assert "before soft=256 " in lines[0] and " after soft=1024 " in lines[0]
|
|
34760
34760
|
assert "before soft=2048 " in lines[1] and " after soft=2048 " in lines[1]
|
|
34761
|
+
|
|
34762
|
+
|
|
34763
|
+
class TestAHolderRespawnsASuccessorThatDiesBeforePublishing:
|
|
34764
|
+
"""T2071: a deploy's `uv tool install --force` rebuilds the venv in place, a
|
|
34765
|
+
code-watch tick lands inside it, and the successor the HOLDER spawns for the
|
|
34766
|
+
daemon's ask dies on `ModuleNotFoundError` before it serves. Nobody watched
|
|
34767
|
+
it: `_supervise` waits on the OLD daemon, which polls `proxy.json` for
|
|
34768
|
+
`_SPAWN_WAIT_S`, logs "no successor published within the wait", releases
|
|
34769
|
+
the listener, drains capped and cuts what is mid-reply (dkim Mac,
|
|
34770
|
+
0.1.329 -> 0.1.331).
|
|
34771
|
+
|
|
34772
|
+
REAL PROCESSES, on the seam `TestACallerOutsideTheLineageTakesTheSocketOver`
|
|
34773
|
+
uses: a shim package re-points `_DAEMON_MODULE` at itself, so every daemon
|
|
34774
|
+
and standby the holder spawns runs it. THE OUTAGE IS THE FILE SYSTEM'S, as
|
|
34775
|
+
in the incident: the shim directory is renamed away (the interpreter is
|
|
34776
|
+
there, the package is not) and back. A tick is forced by a flag file the
|
|
34777
|
+
shim's `_code_is_current` consumes once, so only the first daemon asks. A
|
|
34778
|
+
reply is held mid-body across all of it, past the capped drain.
|
|
34779
|
+
|
|
34780
|
+
THE LIMIT, and the last case pins it: the holder retries inside the
|
|
34781
|
+
daemon's `_SPAWN_WAIT_S`. A window longer than that still ends on the
|
|
34782
|
+
exit-75 fallback and still cuts. The daemon is not made to wait longer: its
|
|
34783
|
+
listener is released at the bound, and the holder cannot respawn until it
|
|
34784
|
+
exits (4397803).
|
|
34785
|
+
"""
|
|
34786
|
+
|
|
34787
|
+
EMAIL = "pin@example.com"
|
|
34788
|
+
CAP_S = 3.0 # the shim's `_HELD_DRAIN_SECONDS`, the fallback's drain cap
|
|
34789
|
+
|
|
34790
|
+
_SHIM = '''
|
|
34791
|
+
import functools, sys
|
|
34792
|
+
from pathlib import Path
|
|
34793
|
+
from cswap_pin import proxy as p
|
|
34794
|
+
p._DAEMON_MODULE = "t2071_cswap_pin.proxy"
|
|
34795
|
+
p._HELD_DRAIN_SECONDS = @CAP@
|
|
34796
|
+
p._SPAWN_WAIT_S = @WAIT@
|
|
34797
|
+
p._watch_own_code = functools.partial(p._watch_own_code, interval=0.3)
|
|
34798
|
+
_tick = Path("@TICK@")
|
|
34799
|
+
_current = p._code_is_current
|
|
34800
|
+
def _code_is_current(own):
|
|
34801
|
+
# THE TICK THAT FOUND THE CODE CHANGED (an unreadable tree reads as
|
|
34802
|
+
# changed): once, so a successor does not ask in its turn.
|
|
34803
|
+
if _tick.exists():
|
|
34804
|
+
_tick.unlink()
|
|
34805
|
+
return False
|
|
34806
|
+
return _current(own)
|
|
34807
|
+
p._code_is_current = _code_is_current
|
|
34808
|
+
a = sys.argv
|
|
34809
|
+
if a[1:2] == [p._HOLDER_MODULE_ARG]:
|
|
34810
|
+
p.holder_main(a[3], a[4], Path(a[5]), port=int(a[2]))
|
|
34811
|
+
elif a[1:2] == [p._STANDBY_MODULE_ARG]:
|
|
34812
|
+
p.standby_main(a[2], a[3], Path(a[4]))
|
|
34813
|
+
else:
|
|
34814
|
+
p.daemon_main(a[1], a[2], Path(a[3]))
|
|
34815
|
+
'''
|
|
34816
|
+
|
|
34817
|
+
def test_all(self, request, tmp_path_factory):
|
|
34818
|
+
run_cases(self, request, tmp_path_factory)
|
|
34819
|
+
|
|
34820
|
+
class _Origin:
|
|
34821
|
+
"""A plain-HTTP origin whose reply is mid-body until ``release``."""
|
|
34822
|
+
|
|
34823
|
+
def __init__(self):
|
|
34824
|
+
self.release = threading.Event()
|
|
34825
|
+
self.srv = socket.socket()
|
|
34826
|
+
self.srv.bind(("127.0.0.1", 0))
|
|
34827
|
+
self.srv.listen(4)
|
|
34828
|
+
self.port = self.srv.getsockname()[1]
|
|
34829
|
+
threading.Thread(target=self._serve, daemon=True).start()
|
|
34830
|
+
|
|
34831
|
+
def _serve(self):
|
|
34832
|
+
try:
|
|
34833
|
+
c, _ = self.srv.accept()
|
|
34834
|
+
data = b""
|
|
34835
|
+
while b"\r\n\r\n" not in data:
|
|
34836
|
+
data += c.recv(4096)
|
|
34837
|
+
c.sendall(b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n"
|
|
34838
|
+
b"\r\n5\r\nfirst\r\n")
|
|
34839
|
+
self.release.wait(60)
|
|
34840
|
+
c.sendall(b"6\r\nsecond\r\n0\r\n\r\n")
|
|
34841
|
+
c.close()
|
|
34842
|
+
except OSError:
|
|
34843
|
+
pass
|
|
34844
|
+
|
|
34845
|
+
@staticmethod
|
|
34846
|
+
def _until(what, fn, timeout):
|
|
34847
|
+
deadline = time.monotonic() + timeout
|
|
34848
|
+
while time.monotonic() < deadline:
|
|
34849
|
+
got = fn()
|
|
34850
|
+
if got:
|
|
34851
|
+
return got
|
|
34852
|
+
time.sleep(0.05)
|
|
34853
|
+
raise AssertionError(f"timed out after {timeout}s: {what}")
|
|
34854
|
+
|
|
34855
|
+
@staticmethod
|
|
34856
|
+
def _read_reply(client, until, timeout=30):
|
|
34857
|
+
"""Everything the client receives until ``until`` is in it, or EOF."""
|
|
34858
|
+
client.settimeout(timeout)
|
|
34859
|
+
got = b""
|
|
34860
|
+
while until not in got:
|
|
34861
|
+
try:
|
|
34862
|
+
chunk = client.recv(4096)
|
|
34863
|
+
except OSError:
|
|
34864
|
+
break
|
|
34865
|
+
if not chunk:
|
|
34866
|
+
break
|
|
34867
|
+
got += chunk
|
|
34868
|
+
return got
|
|
34869
|
+
|
|
34870
|
+
@contextlib.contextmanager
|
|
34871
|
+
def _held_reply_across_a_window(self, monkeypatch, window_s, wait_s=10.0):
|
|
34872
|
+
"""A real holder + daemon + standby under a short scratch dir (started
|
|
34873
|
+
through a launcher that exits), a reply held mid-body, a code-watch
|
|
34874
|
+
tick, and ``window_s`` seconds in which the successor's module cannot
|
|
34875
|
+
be found. Yields ``(t, rest)`` once the old daemon has settled and the
|
|
34876
|
+
held reply was released past its capped drain: ``rest`` is what the
|
|
34877
|
+
client got after that."""
|
|
34878
|
+
import shutil
|
|
34879
|
+
import subprocess
|
|
34880
|
+
import sys
|
|
34881
|
+
import tempfile
|
|
34882
|
+
|
|
34883
|
+
from conftest import _reap_pin_processes
|
|
34884
|
+
from cswap_pin import proxy as pp
|
|
34885
|
+
|
|
34886
|
+
base = Path(tempfile.mkdtemp(prefix="r", dir="/tmp")).resolve()
|
|
34887
|
+
certdir = base / "pin-proxy"
|
|
34888
|
+
certdir.mkdir()
|
|
34889
|
+
pkg = base / "shim" / "t2071_cswap_pin"
|
|
34890
|
+
pkg.mkdir(parents=True)
|
|
34891
|
+
(pkg / "__init__.py").write_text("")
|
|
34892
|
+
tick = base / "tick"
|
|
34893
|
+
(pkg / "proxy.py").write_text(
|
|
34894
|
+
self._SHIM.replace("@TICK@", str(tick))
|
|
34895
|
+
.replace("@WAIT@", str(wait_s)).replace("@CAP@", str(self.CAP_S)))
|
|
34896
|
+
away = pkg.with_name("t2071_cswap_pin.gone")
|
|
34897
|
+
origin = self._Origin()
|
|
34898
|
+
timer = client = None
|
|
34899
|
+
try:
|
|
34900
|
+
pp.ensure_ca(certdir, "api.anthropic.com")
|
|
34901
|
+
pp.save_pin(base, self.EMAIL, "org-1")
|
|
34902
|
+
monkeypatch.setenv("CSWAP_PIN_ALLOW_DIRECT", "1")
|
|
34903
|
+
env = dict(os.environ)
|
|
34904
|
+
env["PYTHONPATH"] = os.pathsep.join(
|
|
34905
|
+
[str(base / "shim")]
|
|
34906
|
+
+ ([env["PYTHONPATH"]] if env.get("PYTHONPATH") else []))
|
|
34907
|
+
subprocess.run(
|
|
34908
|
+
[sys.executable, "-c",
|
|
34909
|
+
TestACallerOutsideTheLineageTakesTheSocketOver._LAUNCH,
|
|
34910
|
+
str(certdir / "daemon.log"), sys.executable, "-m",
|
|
34911
|
+
"t2071_cswap_pin.proxy", pp._HOLDER_MODULE_ARG, "0", "2",
|
|
34912
|
+
self.EMAIL, str(certdir)], env=env, check=True, timeout=30)
|
|
34913
|
+
|
|
34914
|
+
def serving():
|
|
34915
|
+
st = pp.read_daemon_state(certdir)
|
|
34916
|
+
body = st and pp._health_body(int(st["port"]))
|
|
34917
|
+
return st if body and body.get("pid") == st["pid"] else None
|
|
34918
|
+
|
|
34919
|
+
st = self._until("the first daemon never served", serving, 40)
|
|
34920
|
+
log = lambda: (certdir / "daemon.log").read_text(errors="replace")
|
|
34921
|
+
t = types.SimpleNamespace(
|
|
34922
|
+
certdir=certdir, port=int(st["port"]), daemon=int(st["pid"]),
|
|
34923
|
+
log=log, serving=serving)
|
|
34924
|
+
|
|
34925
|
+
client = socket.create_connection(("127.0.0.1", t.port), timeout=20)
|
|
34926
|
+
client.sendall(
|
|
34927
|
+
f"GET http://127.0.0.1:{origin.port}/s HTTP/1.1\r\n"
|
|
34928
|
+
f"Host: 127.0.0.1:{origin.port}\r\n\r\n".encode())
|
|
34929
|
+
assert b"first" in self._read_reply(client, b"first"), (
|
|
34930
|
+
"premise: the reply is mid-body")
|
|
34931
|
+
# THE VENV BEING REBUILT: the package is missing, the interpreter
|
|
34932
|
+
# is not. Back when the install finishes.
|
|
34933
|
+
pkg.rename(away)
|
|
34934
|
+
tick.touch()
|
|
34935
|
+
timer = threading.Timer(window_s, lambda: away.rename(pkg))
|
|
34936
|
+
timer.daemon = True
|
|
34937
|
+
timer.start()
|
|
34938
|
+
self._until(
|
|
34939
|
+
"the old daemon neither handed over nor fell back",
|
|
34940
|
+
lambda: ("while we keep serving" in log()
|
|
34941
|
+
or "no successor published" in log()), wait_s + 20)
|
|
34942
|
+
# HELD PAST THE CAPPED DRAIN, so a fallback that drains capped
|
|
34943
|
+
# cuts it, and a handover that does not is seen not to.
|
|
34944
|
+
time.sleep(self.CAP_S + 1.5)
|
|
34945
|
+
origin.release.set()
|
|
34946
|
+
yield t, self._read_reply(client, b"0\r\n\r\n")
|
|
34947
|
+
finally:
|
|
34948
|
+
if timer is not None:
|
|
34949
|
+
timer.cancel()
|
|
34950
|
+
if client is not None:
|
|
34951
|
+
client.close()
|
|
34952
|
+
origin.release.set()
|
|
34953
|
+
origin.srv.close()
|
|
34954
|
+
_reap_pin_processes(certdir)
|
|
34955
|
+
shutil.rmtree(base, ignore_errors=True)
|
|
34956
|
+
|
|
34957
|
+
def _one_daemon_is_left(self, t, pid):
|
|
34958
|
+
from cswap_pin import proxy as pp
|
|
34959
|
+
|
|
34960
|
+
self._until(
|
|
34961
|
+
f"exactly daemon {pid} is left for the certdir, not "
|
|
34962
|
+
f"{pp._pin_daemon_pids(t.certdir)}",
|
|
34963
|
+
lambda: pp._pin_daemon_pids(t.certdir) == [pid], 20)
|
|
34964
|
+
|
|
34965
|
+
def _a_successor_serves_and_the_reply_completed(self, t, rest):
|
|
34966
|
+
log = t.log()
|
|
34967
|
+
assert "finding module specification" in log, (
|
|
34968
|
+
"premise: the first successor died on the missing module\n" + log)
|
|
34969
|
+
assert "no successor published within the wait" not in log, (
|
|
34970
|
+
"the daemon gave up on a successor the holder could have "
|
|
34971
|
+
"respawned\n" + log)
|
|
34972
|
+
assert not re.search(r"cut \d+ in-flight", log), log
|
|
34973
|
+
assert "asked the holder to replace us while we keep serving" in log, (
|
|
34974
|
+
"the gapless arm did not run\n" + log)
|
|
34975
|
+
assert rest.endswith(b"second\r\n0\r\n\r\n"), (
|
|
34976
|
+
f"the held reply did not complete: {rest!r}\n{log}")
|
|
34977
|
+
st = t.serving()
|
|
34978
|
+
assert st and int(st["pid"]) != t.daemon, (
|
|
34979
|
+
"no successor is serving\n" + log)
|
|
34980
|
+
assert int(st["port"]) == t.port, "the port moved"
|
|
34981
|
+
self._one_daemon_is_left(t, int(st["pid"]))
|
|
34982
|
+
|
|
34983
|
+
def case_a_successor_that_dies_in_a_short_window_is_respawned(
|
|
34984
|
+
self, monkeypatch):
|
|
34985
|
+
with self._held_reply_across_a_window(monkeypatch, 1.0) as (t, rest):
|
|
34986
|
+
self._a_successor_serves_and_the_reply_completed(t, rest)
|
|
34987
|
+
|
|
34988
|
+
def case_a_window_of_several_seconds_takes_several_respawns(
|
|
34989
|
+
self, monkeypatch):
|
|
34990
|
+
"""The window is ~3 s: the successor is born into the missing module
|
|
34991
|
+
again and again (backoff 0.5 s, 1 s, 2 s) until one of them finds it."""
|
|
34992
|
+
with self._held_reply_across_a_window(monkeypatch, 3.0) as (t, rest):
|
|
34993
|
+
self._a_successor_serves_and_the_reply_completed(t, rest)
|
|
34994
|
+
died = t.log().count("finding module specification")
|
|
34995
|
+
assert died >= 2, (
|
|
34996
|
+
f"{died} successor(s) died: the window was not several "
|
|
34997
|
+
f"respawns wide\n" + t.log())
|
|
34998
|
+
|
|
34999
|
+
def case_a_window_longer_than_the_wait_still_cuts(self, monkeypatch):
|
|
35000
|
+
"""THE LIMIT. The wait is 3 s here and the window 5 s: no respawn can
|
|
35001
|
+
publish in time, the daemon falls back to exit 75 and its capped drain,
|
|
35002
|
+
and the held reply is cut. The holder then restarts a daemon once the
|
|
35003
|
+
module is back, so the port is served again."""
|
|
35004
|
+
with self._held_reply_across_a_window(
|
|
35005
|
+
monkeypatch, 5.0, wait_s=3.0) as (t, rest):
|
|
35006
|
+
log = t.log()
|
|
35007
|
+
at = [log.find(m) for m in (
|
|
35008
|
+
"finding module specification",
|
|
35009
|
+
"no successor published within the wait", "cut 1 in-flight")]
|
|
35010
|
+
assert all(i >= 0 for i in at) and at == sorted(at), (
|
|
35011
|
+
"the daemon did not fall back and cut, in that order\n" + log)
|
|
35012
|
+
assert not rest.endswith(b"second\r\n0\r\n\r\n"), rest
|
|
35013
|
+
again = self._until(
|
|
35014
|
+
"the holder never served the port again",
|
|
35015
|
+
lambda: (lambda st: st if st and int(st["pid"]) != t.daemon
|
|
35016
|
+
else None)(t.serving()), 40)
|
|
35017
|
+
assert int(again["port"]) == t.port
|
|
35018
|
+
self._one_daemon_is_left(t, int(again["pid"]))
|
|
35019
|
+
|
|
35020
|
+
class _Child:
|
|
35021
|
+
"""A successor as the watch sees it: dead with ``code``, or alive."""
|
|
35022
|
+
|
|
35023
|
+
def __init__(self, pid, code=None):
|
|
35024
|
+
self.pid, self.code = pid, code
|
|
35025
|
+
|
|
35026
|
+
def wait(self, timeout=None):
|
|
35027
|
+
import subprocess
|
|
35028
|
+
|
|
35029
|
+
if self.code is None:
|
|
35030
|
+
time.sleep(timeout or 0)
|
|
35031
|
+
raise subprocess.TimeoutExpired("daemon", timeout)
|
|
35032
|
+
return self.code
|
|
35033
|
+
|
|
35034
|
+
def poll(self):
|
|
35035
|
+
return self.code
|
|
35036
|
+
|
|
35037
|
+
def _holder(self, certdir, kids, backoff=0.0):
|
|
35038
|
+
"""A holder whose k-th `_spawn` makes ``kids[k]`` the successor."""
|
|
35039
|
+
from cswap_pin import proxy as pp
|
|
35040
|
+
|
|
35041
|
+
class _Holder(pp.PortHolder):
|
|
35042
|
+
_backoff = staticmethod(lambda failures: backoff)
|
|
35043
|
+
|
|
35044
|
+
def __init__(self):
|
|
35045
|
+
self._stop = False
|
|
35046
|
+
self._certdir = certdir
|
|
35047
|
+
self._replace_lock = threading.RLock()
|
|
35048
|
+
self._proc = kids[0]
|
|
35049
|
+
self.spawned = 0
|
|
35050
|
+
self.port = 1
|
|
35051
|
+
|
|
35052
|
+
def _spawn(self):
|
|
35053
|
+
self.spawned += 1
|
|
35054
|
+
self._proc = kids[self.spawned]
|
|
35055
|
+
|
|
35056
|
+
return _Holder()
|
|
35057
|
+
|
|
35058
|
+
def case_the_watch_spawns_a_dead_successor_again_until_one_publishes(
|
|
35059
|
+
self, tmp_path):
|
|
35060
|
+
from cswap_pin import proxy as pp
|
|
35061
|
+
|
|
35062
|
+
kids = [self._Child(10, 1), self._Child(11, 1), self._Child(12)]
|
|
35063
|
+
pp.write_daemon_state(tmp_path, 1, 12, "fp")
|
|
35064
|
+
h = self._holder(tmp_path, kids)
|
|
35065
|
+
h._respawn_until_published(kids[0], time.monotonic() + 10)
|
|
35066
|
+
assert h.spawned == 2 and h._proc is kids[2], h.spawned
|
|
35067
|
+
|
|
35068
|
+
def case_the_watch_never_spawns_over_a_live_successor(self, tmp_path):
|
|
35069
|
+
"""A successor that is alive and has not published (a slow boot) is
|
|
35070
|
+
left alone to the deadline: two daemons on one socket."""
|
|
35071
|
+
kids = [self._Child(10), self._Child(11)]
|
|
35072
|
+
h = self._holder(tmp_path, kids)
|
|
35073
|
+
h._respawn_until_published(kids[0], time.monotonic() + 0.4)
|
|
35074
|
+
assert h.spawned == 0 and h._proc is kids[0]
|
|
35075
|
+
|
|
35076
|
+
def case_the_respawns_end_inside_the_daemons_wait(self, tmp_path):
|
|
35077
|
+
"""With `_backoff`'s ladder (0.5 s, 1 s, 2 s) a deadline 2 s out takes
|
|
35078
|
+
the first two respawns and not the third, which could not begin in
|
|
35079
|
+
time: the daemon's own fallback follows, as it always did."""
|
|
35080
|
+
from cswap_pin import proxy as pp
|
|
35081
|
+
|
|
35082
|
+
kids = [self._Child(10 + n, 1) for n in range(8)]
|
|
35083
|
+
h = self._holder(tmp_path, kids)
|
|
35084
|
+
h._backoff = pp.PortHolder._backoff
|
|
35085
|
+
started = time.monotonic()
|
|
35086
|
+
h._respawn_until_published(kids[0], started + 2.0)
|
|
35087
|
+
assert h.spawned == 2, h.spawned
|
|
35088
|
+
assert time.monotonic() - started < 2.5
|
|
35089
|
+
|
|
35090
|
+
def case_the_watch_leaves_a_successor_that_is_no_longer_current(
|
|
35091
|
+
self, tmp_path):
|
|
35092
|
+
"""The supervisor respawned it (the daemon left first), a later ask
|
|
35093
|
+
moved `self._proc`, or `stop()` ran: this spawns nothing."""
|
|
35094
|
+
kids = [self._Child(10, 1), self._Child(11, 1)]
|
|
35095
|
+
h = self._holder(tmp_path, kids)
|
|
35096
|
+
h._proc = kids[1]
|
|
35097
|
+
h._respawn_until_published(kids[0], time.monotonic() + 5)
|
|
35098
|
+
assert h.spawned == 0
|
|
35099
|
+
h._proc = kids[0]
|
|
35100
|
+
h._stop = True
|
|
35101
|
+
h._respawn_until_published(kids[0], time.monotonic() + 5)
|
|
35102
|
+
assert h.spawned == 0
|
|
35103
|
+
|
|
35104
|
+
def case_the_watch_starts_only_on_the_daemons_own_ask(
|
|
35105
|
+
self, tmp_path, monkeypatch):
|
|
35106
|
+
import signal
|
|
35107
|
+
|
|
35108
|
+
watched = []
|
|
35109
|
+
kids = [self._Child(10), self._Child(11), self._Child(12)]
|
|
35110
|
+
h = self._holder(tmp_path, kids)
|
|
35111
|
+
h._respawn_until_published = lambda proc, deadline: watched.append(proc)
|
|
35112
|
+
h._on_replace_request(None, None) # the holder's own wedge repair
|
|
35113
|
+
assert h.spawned == 1 and watched == []
|
|
35114
|
+
h._on_replace_request(signal.SIGUSR1, None)
|
|
35115
|
+
self._until("the daemon's ask started no watch",
|
|
35116
|
+
lambda: watched, 5)
|
|
35117
|
+
|
|
35118
|
+
def case_a_successor_that_died_is_not_logged_as_serving(
|
|
35119
|
+
self, tmp_path, monkeypatch):
|
|
35120
|
+
"""The daemon left (exit 75) after a successor that had already died:
|
|
35121
|
+
the line said "successor already serving" while nothing served."""
|
|
35122
|
+
from cswap_pin import proxy as pp
|
|
35123
|
+
|
|
35124
|
+
lines = []
|
|
35125
|
+
monkeypatch.setattr(pp, "_log_lifecycle", lines.append)
|
|
35126
|
+
old, dead, alive = self._Child(9, 75), self._Child(10, 1), self._Child(11)
|
|
35127
|
+
h = self._holder(tmp_path, [dead])
|
|
35128
|
+
assert h._supervise_locked(old, 75) == "continue"
|
|
35129
|
+
h = self._holder(tmp_path, [alive])
|
|
35130
|
+
assert h._supervise_locked(old, 0) == "continue"
|
|
35131
|
+
assert "already exited (1)" in lines[0] and "serving" not in lines[0]
|
|
35132
|
+
assert "successor already serving" in lines[1], "the control"
|
|
35133
|
+
|
|
35134
|
+
def case_a_respawn_that_waited_on_the_lock_past_the_deadline_does_not_start(
|
|
35135
|
+
self, tmp_path):
|
|
35136
|
+
"""The docstring's promise: none starts after the daemon's wait. A
|
|
35137
|
+
respawn that sat on `_replace_lock` (an ask, `stop()`) until the
|
|
35138
|
+
deadline passed has nobody left to publish to."""
|
|
35139
|
+
kids = [self._Child(10, 1), self._Child(11)]
|
|
35140
|
+
h = self._holder(tmp_path, kids)
|
|
35141
|
+
held = threading.Event()
|
|
35142
|
+
|
|
35143
|
+
def hold():
|
|
35144
|
+
with h._replace_lock:
|
|
35145
|
+
held.set()
|
|
35146
|
+
time.sleep(0.6)
|
|
35147
|
+
|
|
35148
|
+
threading.Thread(target=hold, daemon=True).start()
|
|
35149
|
+
assert held.wait(5)
|
|
35150
|
+
h._respawn_until_published(kids[0], time.monotonic() + 0.3)
|
|
35151
|
+
assert h.spawned == 0 and h._proc is kids[0]
|
|
35152
|
+
|
|
35153
|
+
def case_the_watch_leaves_a_socket_that_went_to_a_callers_lineage(
|
|
35154
|
+
self, tmp_path):
|
|
35155
|
+
"""A later ask handed the socket to a caller outside this lineage
|
|
35156
|
+
(`_handed_off`): a successor spawned here would be a denied-lineage
|
|
35157
|
+
daemon on a socket that is another lineage's now."""
|
|
35158
|
+
kids = [self._Child(10, 1), self._Child(11)]
|
|
35159
|
+
h = self._holder(tmp_path, kids)
|
|
35160
|
+
h._handed_off = True
|
|
35161
|
+
h._respawn_until_published(kids[0], time.monotonic() + 0.5)
|
|
35162
|
+
assert h.spawned == 0 and h._proc is kids[0]
|
|
35163
|
+
|
|
35164
|
+
def case_a_watch_that_cannot_start_leaves_the_holder_up(
|
|
35165
|
+
self, tmp_path, monkeypatch):
|
|
35166
|
+
"""`Thread.start` raises RuntimeError under EAGAIN / RLIMIT_NPROC, the
|
|
35167
|
+
exhaustion the `_spawn` guard above it already contains. Escaping this
|
|
35168
|
+
SIGNAL HANDLER it would end the holder's main thread with no `stop()`,
|
|
35169
|
+
and the daemon would exit 75 to a dead holder."""
|
|
35170
|
+
import signal
|
|
35171
|
+
|
|
35172
|
+
from cswap_pin import proxy as pp
|
|
35173
|
+
|
|
35174
|
+
lines = []
|
|
35175
|
+
monkeypatch.setattr(pp, "_log_lifecycle", lines.append)
|
|
35176
|
+
kids = [self._Child(10), self._Child(11)]
|
|
35177
|
+
h = self._holder(tmp_path, kids)
|
|
35178
|
+
|
|
35179
|
+
def no_thread(self):
|
|
35180
|
+
raise RuntimeError("can't start new thread")
|
|
35181
|
+
|
|
35182
|
+
with monkeypatch.context() as m:
|
|
35183
|
+
m.setattr(threading.Thread, "start", no_thread)
|
|
35184
|
+
h._on_replace_request(signal.SIGUSR1, None)
|
|
35185
|
+
assert h.spawned == 1 and h._proc is kids[1] and not h._stop
|
|
35186
|
+
assert any("could not start the successor watch" in ln for ln in lines)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|