alissa-tools-github-devloop 0.8.61__tar.gz → 0.8.62__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {alissa_tools_github_devloop-0.8.61/src/main/alissa_tools_github_devloop.egg-info → alissa_tools_github_devloop-0.8.62}/PKG-INFO +1 -1
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa/tools/github/devloop/alissa.py +10 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa/tools/github/devloop/fleet_vitals.py +51 -2
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa/tools/github/devloop/loop.py +53 -2
- alissa_tools_github_devloop-0.8.62/src/main/alissa/tools/github/devloop/self_upgrade.py +622 -0
- alissa_tools_github_devloop-0.8.62/src/main/alissa/tools/github/devloop/version +1 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62/src/main/alissa_tools_github_devloop.egg-info}/PKG-INFO +1 -1
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa_tools_github_devloop.egg-info/SOURCES.txt +1 -0
- alissa_tools_github_devloop-0.8.61/src/main/alissa/tools/github/devloop/version +0 -1
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/LICENSE +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/MANIFEST.in +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/NOTICE +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/README.md +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/requirements.txt +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/setup.cfg +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/setup.py +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa/tools/github/devloop/__init__.py +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa/tools/github/devloop/__main__.py +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa/tools/github/devloop/alissa_client.py +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa/tools/github/devloop/bows.py +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa/tools/github/devloop/config.py +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa/tools/github/devloop/ghclient.py +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa/tools/github/devloop/hubs.py +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa/tools/github/devloop/hungtests.py +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa/tools/github/devloop/integration_check.py +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa/tools/github/devloop/janitor.py +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa/tools/github/devloop/loop_events.py +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa/tools/github/devloop/parking.py +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa/tools/github/devloop/proc.py +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa/tools/github/devloop/prompts.py +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa/tools/github/devloop/state.py +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa/tools/github/devloop/version.py +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa/tools/github/devloop/webui/__init__.py +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa/tools/github/devloop/webui/__main__.py +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa/tools/github/devloop/webui/auth.py +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa/tools/github/devloop/webui/page.py +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa/tools/github/devloop/webui/server.py +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa/tools/github/devloop/webui/sources.py +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa/tools/github/devloop/webui/sysinfo.py +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa_tools_github_devloop.egg-info/dependency_links.txt +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa_tools_github_devloop.egg-info/entry_points.txt +0 -0
- {alissa_tools_github_devloop-0.8.61 → alissa_tools_github_devloop-0.8.62}/src/main/alissa_tools_github_devloop.egg-info/top_level.txt +0 -0
|
@@ -267,6 +267,16 @@ class Alissa:
|
|
|
267
267
|
)
|
|
268
268
|
return [entry for entry in data if isinstance(entry, dict)]
|
|
269
269
|
|
|
270
|
+
def list_queues(self) -> "object":
|
|
271
|
+
"""`alissa tmux queue ls --json`, parsed, or None when the CLI does
|
|
272
|
+
not answer (`parking.queued_items` reads None as unknown, never
|
|
273
|
+
empty). Read by the self-upgrade idle check only."""
|
|
274
|
+
try:
|
|
275
|
+
return json.loads(run(["alissa", "tmux", "queue", "ls", "--json"], timeout=60))
|
|
276
|
+
except (CommandError, OSError, ValueError) as exc:
|
|
277
|
+
log.debug("could not list the alissa tmux queue (%s)", exc)
|
|
278
|
+
return None
|
|
279
|
+
|
|
270
280
|
def tail_session(self, name: str, lines: int = 20) -> str:
|
|
271
281
|
"""The last `lines` of ONE managed session's terminal, or "" when the
|
|
272
282
|
CLI cannot answer.
|
|
@@ -58,6 +58,15 @@ it; the console's out-of-pass push leaves it out (that process never
|
|
|
58
58
|
resolves the feed authority nor refreshes the allowlist, so its answer would
|
|
59
59
|
be a different one). A Studio whose strict schema predates the block gets
|
|
60
60
|
the `idle` treatment: resent without it, offered again after BLOCK_REPROBE_S.
|
|
61
|
+
|
|
62
|
+
Fleet auto-upgrade (issue #206, Studio AU-1) makes the ANSWER matter: every
|
|
63
|
+
answer carries an `upgrade` block (`self_upgrade.parse_upgrade`; absent on an
|
|
64
|
+
older Studio). The pusher keeps the last one read (`FleetVitalsPusher.upgrade`)
|
|
65
|
+
and hands each answer to the daemon's `SelfUpgrader`, which drains and
|
|
66
|
+
upgrades on it. The request side gains `upgradeFailed: {version, reason}`,
|
|
67
|
+
the seat's report that a version would not start and was rolled back: sent
|
|
68
|
+
once, on the next snapshot after the rollback, and treated like the other
|
|
69
|
+
optional blocks by a Studio that refuses it.
|
|
61
70
|
"""
|
|
62
71
|
|
|
63
72
|
from __future__ import annotations
|
|
@@ -71,6 +80,7 @@ from datetime import datetime, timezone
|
|
|
71
80
|
from typing import Callable, Mapping, Sequence
|
|
72
81
|
|
|
73
82
|
from . import parking
|
|
83
|
+
from . import self_upgrade as self_upgrade_mod
|
|
74
84
|
from .alissa_client import AlissaAuthError, AlissaClient, AlissaError
|
|
75
85
|
from .config import REPOS_BOWS, Config
|
|
76
86
|
from .version import version as running_version
|
|
@@ -102,7 +112,7 @@ FAILED = "failed"
|
|
|
102
112
|
# The blocks Studio's schema gained after the base contract, each in its own
|
|
103
113
|
# lane: `idle` (seat parking L3) and `config` (provisioner L5). A strict 400
|
|
104
114
|
# naming one of them is an old Studio, not a bad snapshot.
|
|
105
|
-
OPTIONAL_BLOCKS = ("idle", "config")
|
|
115
|
+
OPTIONAL_BLOCKS = ("idle", "config", "upgradeFailed")
|
|
106
116
|
|
|
107
117
|
# How long a snapshot leaves an optional block out after Studio refused it as
|
|
108
118
|
# an unknown key, before it offers the block again. An hour: one extra POST
|
|
@@ -450,6 +460,7 @@ class FleetVitalsPusher:
|
|
|
450
460
|
memory_reader: "Callable[[], Mapping | None]" = sysinfo.memory_usage,
|
|
451
461
|
clock: "Callable[[], float]" = time.time,
|
|
452
462
|
config_reader: "Callable[[], Mapping | None] | None" = None,
|
|
463
|
+
upgrader: "self_upgrade_mod.SelfUpgrader | None" = None,
|
|
453
464
|
):
|
|
454
465
|
self.config = config
|
|
455
466
|
self.client = client
|
|
@@ -466,6 +477,12 @@ class FleetVitalsPusher:
|
|
|
466
477
|
# as an unknown key). Absent = offer it.
|
|
467
478
|
self._refused_until: "dict[str, float]" = {}
|
|
468
479
|
self.last: "dict | None" = None
|
|
480
|
+
# The daemon's self-upgrade state (issue #206), or None -- the
|
|
481
|
+
# console's pusher -- to read the answer and act on nothing.
|
|
482
|
+
self.upgrader = upgrader
|
|
483
|
+
# The last `upgrade` block a vitals answer carried (issue #206), kept
|
|
484
|
+
# across answers that lack one.
|
|
485
|
+
self.upgrade: "self_upgrade_mod.UpgradeAnswer | None" = None
|
|
469
486
|
|
|
470
487
|
def push_pass(self, roster: "Sequence[Mapping] | None") -> str:
|
|
471
488
|
"""Build this pass's snapshot and POST it once. Returns PUSHED or
|
|
@@ -514,12 +531,43 @@ class FleetVitalsPusher:
|
|
|
514
531
|
"pushed this pass; the next pass builds a fresh one", exc,
|
|
515
532
|
)
|
|
516
533
|
return FAILED
|
|
534
|
+
report = self._upgrade_report()
|
|
535
|
+
if report is not None:
|
|
536
|
+
snapshot["upgradeFailed"] = report
|
|
517
537
|
for key, until in self._refused_until.items():
|
|
518
538
|
if now < until:
|
|
519
539
|
snapshot.pop(key, None)
|
|
520
540
|
self.last = snapshot
|
|
521
541
|
return self._post(snapshot)
|
|
522
542
|
|
|
543
|
+
def _upgrade_report(self) -> "dict | None":
|
|
544
|
+
"""The `upgradeFailed` report owed to Studio, or None. A reader that
|
|
545
|
+
fails costs the report this pass, never the snapshot."""
|
|
546
|
+
if self.upgrader is None:
|
|
547
|
+
return None
|
|
548
|
+
try:
|
|
549
|
+
return self.upgrader.report()
|
|
550
|
+
except Exception as exc:
|
|
551
|
+
log.warning("fleet vitals: could not read the upgrade report (%s)", exc)
|
|
552
|
+
return None
|
|
553
|
+
|
|
554
|
+
def _answered(self, answer: object, snapshot: "Mapping") -> None:
|
|
555
|
+
"""Read one successful answer: keep its `upgrade` block and hand it
|
|
556
|
+
to the upgrader, then mark a carried `upgradeFailed` delivered.
|
|
557
|
+
Never raises: the push already succeeded."""
|
|
558
|
+
try:
|
|
559
|
+
parsed = self_upgrade_mod.parse_upgrade(answer)
|
|
560
|
+
if parsed is not None:
|
|
561
|
+
self.upgrade = parsed
|
|
562
|
+
if self.upgrader is None:
|
|
563
|
+
return
|
|
564
|
+
self.upgrader.observe(answer)
|
|
565
|
+
sent = snapshot.get("upgradeFailed")
|
|
566
|
+
if isinstance(sent, Mapping):
|
|
567
|
+
self.upgrader.reported(sent)
|
|
568
|
+
except Exception as exc:
|
|
569
|
+
log.warning("fleet vitals: could not read the upgrade answer (%s)", exc)
|
|
570
|
+
|
|
523
571
|
@staticmethod
|
|
524
572
|
def _refused_blocks(exc: AlissaError, snapshot: "Mapping") -> "list[str]":
|
|
525
573
|
"""The optional blocks of `snapshot` a 400 names: a Studio whose
|
|
@@ -536,7 +584,7 @@ class FleetVitalsPusher:
|
|
|
536
584
|
|
|
537
585
|
def _post(self, snapshot: "dict") -> str:
|
|
538
586
|
try:
|
|
539
|
-
self.client.post_fleet_vitals(snapshot)
|
|
587
|
+
answer = self.client.post_fleet_vitals(snapshot)
|
|
540
588
|
except AlissaAuthError as exc:
|
|
541
589
|
if exc.code == NOT_ACTIVATED:
|
|
542
590
|
self._warn_not_activated(exc)
|
|
@@ -561,6 +609,7 @@ class FleetVitalsPusher:
|
|
|
561
609
|
return self._post(trimmed)
|
|
562
610
|
self._warn_failed(exc)
|
|
563
611
|
return FAILED
|
|
612
|
+
self._answered(answer, snapshot)
|
|
564
613
|
sessions = snapshot.get("sessions") or {}
|
|
565
614
|
log.debug(
|
|
566
615
|
"fleet vitals: pushed (%s live / %s managed session(s), %d inbox "
|
|
@@ -470,6 +470,7 @@ from . import loop_events as loop_events_mod
|
|
|
470
470
|
from . import parking as parking_mod
|
|
471
471
|
from . import proc as proc_mod
|
|
472
472
|
from . import prompts as prompts_mod
|
|
473
|
+
from . import self_upgrade as self_upgrade_mod
|
|
473
474
|
from .alissa import MANAGED_PREFIX, SHELL_COMMANDS, Alissa
|
|
474
475
|
from .alissa_client import AlissaClient
|
|
475
476
|
from .bows import BowRepoSource
|
|
@@ -513,6 +514,7 @@ from .ghclient import (
|
|
|
513
514
|
)
|
|
514
515
|
from .proc import CommandError, run
|
|
515
516
|
from .state import State
|
|
517
|
+
from .version import version as running_version
|
|
516
518
|
|
|
517
519
|
log = logging.getLogger(__name__)
|
|
518
520
|
|
|
@@ -4625,6 +4627,7 @@ class DevWatcher:
|
|
|
4625
4627
|
repo_source: "BowRepoSource | None" = None,
|
|
4626
4628
|
loop_events: "loop_events_mod.LoopEventEmitter | None" = None,
|
|
4627
4629
|
fleet_vitals: "fleet_vitals_mod.FleetVitalsPusher | None" = None,
|
|
4630
|
+
self_upgrade: "self_upgrade_mod.SelfUpgrader | None" = None,
|
|
4628
4631
|
):
|
|
4629
4632
|
self.config = config
|
|
4630
4633
|
self.github = github or GitHub(config.developer_login)
|
|
@@ -4674,14 +4677,23 @@ class DevWatcher:
|
|
|
4674
4677
|
# fleet_vitals_enabled is off -- the loop-events shape exactly: no
|
|
4675
4678
|
# client built, no request ever made, same endpoint and token,
|
|
4676
4679
|
# injectable so a test never opens a socket.
|
|
4680
|
+
# Fleet auto-upgrade (issue #206): the drain and self-upgrade state,
|
|
4681
|
+
# fed by the vitals answer. Built always (it reads nothing until an
|
|
4682
|
+
# answer arrives, so a seat without vitals never drains on it).
|
|
4683
|
+
self.self_upgrade = self_upgrade or self_upgrade_mod.SelfUpgrader(
|
|
4684
|
+
running_version.value, dry_run=config.dry_run,
|
|
4685
|
+
)
|
|
4677
4686
|
self.fleet_vitals = fleet_vitals or (
|
|
4678
4687
|
fleet_vitals_mod.FleetVitalsPusher(
|
|
4679
4688
|
config, AlissaClient(base=config.alissa_endpoint),
|
|
4680
4689
|
config_reader=self._vitals_config,
|
|
4690
|
+
upgrader=self.self_upgrade,
|
|
4681
4691
|
)
|
|
4682
4692
|
if config.fleet_vitals_enabled
|
|
4683
4693
|
else None
|
|
4684
4694
|
)
|
|
4695
|
+
if self.fleet_vitals is not None and getattr(self.fleet_vitals, "upgrader", None) is None:
|
|
4696
|
+
self.fleet_vitals.upgrader = self.self_upgrade
|
|
4685
4697
|
# The reaper's name-slug -> (owner, repo) index (see `_anchor_repo`),
|
|
4686
4698
|
# rebuilt only when the allowlist itself changes rather than per
|
|
4687
4699
|
# session per sweep.
|
|
@@ -5313,7 +5325,23 @@ class DevWatcher:
|
|
|
5313
5325
|
this spawn is between its gate and its enqueue sees the claim and
|
|
5314
5326
|
refuses itself, so a park can never race a spawn. A ledger that
|
|
5315
5327
|
cannot answer stands the spawn down too -- a spawn that cannot prove
|
|
5316
|
-
the seat is not drained does not happen.
|
|
5328
|
+
the seat is not drained does not happen.
|
|
5329
|
+
|
|
5330
|
+
The fleet auto-upgrade drain (issue #206) is checked FIRST: while
|
|
5331
|
+
Studio's answer says drain, or a self-hosted seat is waiting to
|
|
5332
|
+
upgrade, no spawn path starts a session. The item defers like a
|
|
5333
|
+
parking drain and the state change is logged once by the upgrader,
|
|
5334
|
+
not once per item."""
|
|
5335
|
+
hold = self.self_upgrade.hold()
|
|
5336
|
+
if hold is not None:
|
|
5337
|
+
detail = self.self_upgrade.hold_detail()
|
|
5338
|
+
log.debug("%s: not spawning %s — %s", slug, name, detail)
|
|
5339
|
+
return Decision(
|
|
5340
|
+
Action.DEFERRED,
|
|
5341
|
+
f"spawns held — {detail}; running sessions finish normally",
|
|
5342
|
+
attempt, round=round_,
|
|
5343
|
+
defer_token=loop_events_mod.DEFER_DRAINED,
|
|
5344
|
+
)
|
|
5317
5345
|
try:
|
|
5318
5346
|
drain = self.state.claim_spawn(name)
|
|
5319
5347
|
except sqlite3.Error as exc:
|
|
@@ -13584,11 +13612,12 @@ class DevWatcher:
|
|
|
13584
13612
|
revert_results=revert_results,
|
|
13585
13613
|
)
|
|
13586
13614
|
vitals = self._push_fleet_vitals(roster)
|
|
13615
|
+
upgrade = self._after_pass_upgrade(roster)
|
|
13587
13616
|
log.info(
|
|
13588
13617
|
"poll summary: %d candidate(s) — %d spawned, %d in-flight, "
|
|
13589
13618
|
"%d skipped, %d escalated, %d capped, %d deferred; "
|
|
13590
13619
|
"%d worker session(s) reaped, %d foreign; %d parked on a prompt, "
|
|
13591
|
-
"%d answered%s%s; vitals: %s",
|
|
13620
|
+
"%d answered%s%s; vitals: %s%s",
|
|
13592
13621
|
len(results),
|
|
13593
13622
|
counts[Action.SPAWNED],
|
|
13594
13623
|
counts[Action.IN_FLIGHT],
|
|
@@ -13603,9 +13632,26 @@ class DevWatcher:
|
|
|
13603
13632
|
" (spawns HELD)" if sweep.hold else "",
|
|
13604
13633
|
f"; deferred: {budget.headroom}" if budget.headroom else "",
|
|
13605
13634
|
vitals,
|
|
13635
|
+
f"; upgrade: {upgrade}" if upgrade != self_upgrade_mod.NONE else "",
|
|
13606
13636
|
)
|
|
13607
13637
|
return results
|
|
13608
13638
|
|
|
13639
|
+
def _after_pass_upgrade(self, roster: SessionLiveness) -> str:
|
|
13640
|
+
"""The pass's self-upgrade step (issue #206), after the vitals push
|
|
13641
|
+
read this pass's answer: `exit` once the seat has been idle for two
|
|
13642
|
+
polls (no live worker session, no undispatched queue item) with a
|
|
13643
|
+
self-hosted upgrade wanted (the request file is then
|
|
13644
|
+
written and `run_forever` returns), `wait` while draining towards
|
|
13645
|
+
one, `none` otherwise. Never raises."""
|
|
13646
|
+
try:
|
|
13647
|
+
return self.self_upgrade.after_pass(
|
|
13648
|
+
roster.entries,
|
|
13649
|
+
queued=lambda: parking_mod.queued_items(self.alissa.list_queues()),
|
|
13650
|
+
)
|
|
13651
|
+
except Exception as exc:
|
|
13652
|
+
log.warning("self-upgrade: the after-pass check failed (%s)", exc)
|
|
13653
|
+
return self_upgrade_mod.NONE
|
|
13654
|
+
|
|
13609
13655
|
def _push_fleet_vitals(self, roster: SessionLiveness) -> str:
|
|
13610
13656
|
"""The pass's fleet-vitals push (issue #116): one console snapshot
|
|
13611
13657
|
to Studio, built from the pass's own roster and the ledgers the pass
|
|
@@ -14017,6 +14063,11 @@ class DevWatcher:
|
|
|
14017
14063
|
try:
|
|
14018
14064
|
self.poll_once()
|
|
14019
14065
|
backoff = self.config.poll_interval
|
|
14066
|
+
if self.self_upgrade.exit_requested:
|
|
14067
|
+
# Issue #206: exit 0 so the container restarts and
|
|
14068
|
+
# the entrypoint installs the target at boot.
|
|
14069
|
+
log.info("stopping for the self-upgrade")
|
|
14070
|
+
return
|
|
14020
14071
|
except RateLimited as exc:
|
|
14021
14072
|
backoff = min(backoff * 2, 900)
|
|
14022
14073
|
log.warning("rate limited (%s) — backing off %ds", exc, backoff)
|
|
@@ -0,0 +1,622 @@
|
|
|
1
|
+
"""Self-upgrade -- the devloop half of fleet auto-upgrade (MDF M6, AU-4,
|
|
2
|
+
issue #206; Studio's contract: studio.alissa.app docs/design/
|
|
3
|
+
fleet-auto-upgrade.md, AU-1).
|
|
4
|
+
|
|
5
|
+
Every ``POST /v1/loop/fleet-vitals`` answer carries an ``upgrade`` block::
|
|
6
|
+
|
|
7
|
+
{"mode": "managed" | "self_hosted",
|
|
8
|
+
"target": {"version", "digest"} | null,
|
|
9
|
+
"autoUpgrade": bool, "paused": bool, "drain": bool}
|
|
10
|
+
|
|
11
|
+
This module reads it and acts on it in four steps:
|
|
12
|
+
|
|
13
|
+
1. **Read.** `parse_upgrade` turns the answer into an `UpgradeAnswer`, or
|
|
14
|
+
None when the block is absent or malformed (an older Studio). The
|
|
15
|
+
`SelfUpgrader` keeps the last good answer in memory. An answer without
|
|
16
|
+
the block leaves the last one in place.
|
|
17
|
+
2. **Drain.** `SelfUpgrader.hold` names why no new worker session may start:
|
|
18
|
+
``drain`` is true (either mode), or a self-hosted seat has a target to
|
|
19
|
+
move to. A seat has a target to move to when `autoUpgrade` is on, the
|
|
20
|
+
pause switch is off, `target.version` is strictly newer than the running
|
|
21
|
+
version (numeric ``X.Y.Z``; an older or equal pin, or a version that does
|
|
22
|
+
not parse, is never a target), and that version is not in the skip list. Running sessions are never
|
|
23
|
+
touched. One INFO line per change of state, not per poll.
|
|
24
|
+
3. **Exit at idle.** Self-hosted only (a managed seat is swapped by the
|
|
25
|
+
backend, AU-2). After two consecutive polls with no live
|
|
26
|
+
``ali-(develop|fix|maintain|revert)-`` session and no undispatched item
|
|
27
|
+
in the alissa tmux queue, `after_pass` writes the
|
|
28
|
+
request file (`REQUEST_FILE` under `upgrade_dir()`) and asks the daemon
|
|
29
|
+
to exit with code 0. The container's restart policy brings it back, and
|
|
30
|
+
the entrypoint installs the target at boot.
|
|
31
|
+
4. **Confirm.** The first vitals POST answered on the new version marks the
|
|
32
|
+
request confirmed. The entrypoint rolls back an unconfirmed upgrade. It
|
|
33
|
+
reinstalls `previous`, adds the target to the file's skip list and leaves
|
|
34
|
+
an ``upgradeFailed`` report. `report` hands that report to the next
|
|
35
|
+
snapshot, and `reported` marks it sent once a POST carrying it succeeds.
|
|
36
|
+
5. **Unserved.** A process that finds an unconfirmed request for a target
|
|
37
|
+
it does not run, written by this same version (``previous``), knows its
|
|
38
|
+
predecessor's exit produced no upgrade: the boot step never ran (a
|
|
39
|
+
non-container or non-root start, a boot block that failed before its
|
|
40
|
+
first write) or installed somewhere the daemon does not import from.
|
|
41
|
+
It skips the target and reports it, so the seat never loops drain →
|
|
42
|
+
exit → restart.
|
|
43
|
+
|
|
44
|
+
Never while paused, never in managed mode, never with a live session: those
|
|
45
|
+
three are checked here, at the moment of the exit, not only when the hold
|
|
46
|
+
starts.
|
|
47
|
+
|
|
48
|
+
The request file is shared with ``docker/claude/entrypoint.sh`` (the
|
|
49
|
+
`self_upgrade_boot` block). Both sides keep every key they do not own, so
|
|
50
|
+
neither can drop the other's state. Its keys:
|
|
51
|
+
|
|
52
|
+
* ``target`` -- ``{"version", "digest"}``, written by the daemon.
|
|
53
|
+
* ``previous`` -- the version the daemon ran when it asked.
|
|
54
|
+
* ``requestedAt`` -- ISO-8601 UTC.
|
|
55
|
+
* ``confirmed`` / ``confirmedAt`` -- written by the daemon after the first
|
|
56
|
+
answered poll on the target.
|
|
57
|
+
* ``boots`` / ``firstBootAt`` / ``firstBootTs`` / ``lastError`` -- the
|
|
58
|
+
entrypoint's attempt record for this target. ``firstBootTs`` is the epoch
|
|
59
|
+
seconds its 10-minute window reads; ``firstBootAt`` the same as ISO-8601.
|
|
60
|
+
* ``rollbackReason`` -- why the entrypoint rolled back, kept across the
|
|
61
|
+
retries of a ``rollback_failed`` reinstall.
|
|
62
|
+
* ``state`` -- ``requested`` (daemon), then ``installed``,
|
|
63
|
+
``install_failed``, ``rollback_failed`` (retried at the next boot),
|
|
64
|
+
``rolled_back`` or ``not_newer`` (a target not newer than the installed
|
|
65
|
+
version, never installed) (entrypoint), then ``confirmed`` or
|
|
66
|
+
``unserved`` (daemon). A ``not_newer`` request this version wrote is
|
|
67
|
+
unserved too: the daemon runs an older copy than the boot step sees.
|
|
68
|
+
* ``skip`` -- versions never to move to again. A newer target clears it.
|
|
69
|
+
* ``upgradeFailed`` -- ``{"version", "reason"}``, the report owed to Studio.
|
|
70
|
+
``upgradeFailedReported`` is set once a POST carried it.
|
|
71
|
+
"""
|
|
72
|
+
|
|
73
|
+
from __future__ import annotations
|
|
74
|
+
|
|
75
|
+
import json
|
|
76
|
+
import logging
|
|
77
|
+
import os
|
|
78
|
+
import re
|
|
79
|
+
import tempfile
|
|
80
|
+
import time
|
|
81
|
+
from dataclasses import dataclass
|
|
82
|
+
from datetime import datetime, timezone
|
|
83
|
+
from typing import Callable, Mapping, Sequence
|
|
84
|
+
|
|
85
|
+
from . import parking
|
|
86
|
+
|
|
87
|
+
log = logging.getLogger(__name__)
|
|
88
|
+
|
|
89
|
+
SEAT = "devloop"
|
|
90
|
+
MANAGED = "managed"
|
|
91
|
+
SELF_HOSTED = "self_hosted"
|
|
92
|
+
MODES = (MANAGED, SELF_HOSTED)
|
|
93
|
+
|
|
94
|
+
# The shared request file. The directory follows ALISSA_UPGRADE_DIR so a
|
|
95
|
+
# test or a non-container run can point it elsewhere. The entrypoint reads
|
|
96
|
+
# the same variable.
|
|
97
|
+
UPGRADE_DIR_ENV = "ALISSA_UPGRADE_DIR"
|
|
98
|
+
DEFAULT_UPGRADE_DIR = "/workspace/.alissa-upgrade"
|
|
99
|
+
REQUEST_FILE = f"{SEAT}.json"
|
|
100
|
+
|
|
101
|
+
# How many consecutive polls with no live worker session count as idle.
|
|
102
|
+
IDLE_POLLS = 2
|
|
103
|
+
|
|
104
|
+
# The worker sessions that make a seat busy, by their bare (un-prefixed)
|
|
105
|
+
# name: the four edges' sessions.
|
|
106
|
+
WORKER_SESSION = re.compile(r"^(?:develop|fix|maintain|revert)-")
|
|
107
|
+
|
|
108
|
+
# The hold reasons `hold` returns, and the poll summary quotes.
|
|
109
|
+
HOLD_DRAIN = "drain"
|
|
110
|
+
HOLD_UPGRADE = "upgrade"
|
|
111
|
+
|
|
112
|
+
# The request states in which an upgrade is still on its way: the only ones
|
|
113
|
+
# the daemon confirms or calls unserved. Every other state is final, or a
|
|
114
|
+
# failure the entrypoint owns.
|
|
115
|
+
PENDING_STATES = ("requested", "installed")
|
|
116
|
+
|
|
117
|
+
# The states in which a request this version wrote left the seat where it
|
|
118
|
+
# was: still pending, or refused at boot as not newer than the installed
|
|
119
|
+
# version (the daemon imports an older copy than the root pass sees). Any
|
|
120
|
+
# of them on a later process means the exit did not move the seat.
|
|
121
|
+
UNSERVED_STATES = PENDING_STATES + ("not_newer",)
|
|
122
|
+
|
|
123
|
+
# `after_pass` outcomes.
|
|
124
|
+
EXIT = "exit"
|
|
125
|
+
WAIT = "wait"
|
|
126
|
+
NONE = "none"
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def upgrade_dir(environ: "Mapping[str, str] | None" = None) -> str:
|
|
130
|
+
env = os.environ if environ is None else environ
|
|
131
|
+
return (env.get(UPGRADE_DIR_ENV) or "").strip() or DEFAULT_UPGRADE_DIR
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def iso(ts: float) -> str:
|
|
135
|
+
return datetime.fromtimestamp(int(ts), tz=timezone.utc).strftime(
|
|
136
|
+
"%Y-%m-%dT%H:%M:%SZ"
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def version_key(version: str) -> "tuple[int, ...] | None":
|
|
141
|
+
"""`X.Y.Z` as a comparable tuple, or None for anything else."""
|
|
142
|
+
parts = str(version).strip().split(".")
|
|
143
|
+
if len(parts) != 3 or not all(p.isdigit() for p in parts):
|
|
144
|
+
return None
|
|
145
|
+
return tuple(int(p) for p in parts)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
@dataclass(frozen=True)
|
|
149
|
+
class UpgradeAnswer:
|
|
150
|
+
"""One vitals answer's ``upgrade`` block."""
|
|
151
|
+
|
|
152
|
+
mode: str
|
|
153
|
+
target_version: "str | None"
|
|
154
|
+
target_digest: "str | None"
|
|
155
|
+
auto_upgrade: bool
|
|
156
|
+
paused: bool
|
|
157
|
+
drain: bool
|
|
158
|
+
|
|
159
|
+
@property
|
|
160
|
+
def target(self) -> "dict | None":
|
|
161
|
+
if self.target_version is None:
|
|
162
|
+
return None
|
|
163
|
+
return {"version": self.target_version, "digest": self.target_digest}
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def parse_upgrade(payload: object) -> "UpgradeAnswer | None":
|
|
167
|
+
"""The ``upgrade`` block of a vitals answer, or None.
|
|
168
|
+
|
|
169
|
+
None when the answer is not an object or carries no ``upgrade`` (a Studio
|
|
170
|
+
older than AU-1), or when the block is not one this daemon can act on
|
|
171
|
+
(an unknown `mode`, a non-boolean flag). A malformed block is never read
|
|
172
|
+
as a partial one: a guessed `paused: false` would let a seat upgrade
|
|
173
|
+
through the operator's switch. A `target` without a string version is
|
|
174
|
+
read as no target."""
|
|
175
|
+
if not isinstance(payload, Mapping):
|
|
176
|
+
return None
|
|
177
|
+
block = payload.get("upgrade")
|
|
178
|
+
if not isinstance(block, Mapping):
|
|
179
|
+
return None
|
|
180
|
+
mode = block.get("mode")
|
|
181
|
+
flags = [block.get(k) for k in ("autoUpgrade", "paused", "drain")]
|
|
182
|
+
if mode not in MODES or not all(isinstance(f, bool) for f in flags):
|
|
183
|
+
return None
|
|
184
|
+
version = digest = None
|
|
185
|
+
target = block.get("target")
|
|
186
|
+
if isinstance(target, Mapping):
|
|
187
|
+
raw = target.get("version")
|
|
188
|
+
if isinstance(raw, str) and raw.strip():
|
|
189
|
+
version = raw.strip()
|
|
190
|
+
raw_digest = target.get("digest")
|
|
191
|
+
digest = raw_digest if isinstance(raw_digest, str) else None
|
|
192
|
+
auto_upgrade, paused, drain = flags
|
|
193
|
+
return UpgradeAnswer(
|
|
194
|
+
mode=str(mode), target_version=version, target_digest=digest,
|
|
195
|
+
auto_upgrade=bool(auto_upgrade), paused=bool(paused), drain=bool(drain),
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def live_worker_sessions(roster: "Sequence[Mapping] | None") -> "list[str] | None":
|
|
200
|
+
"""The live worker sessions in an ``alissa tmux ls`` listing, or None
|
|
201
|
+
when the listing failed (unknown is never idle). Live follows the
|
|
202
|
+
fleet-vitals rule (`parking.roster_counts`): the row's `live` flag, else
|
|
203
|
+
`status != "gone"`."""
|
|
204
|
+
if roster is None:
|
|
205
|
+
return None
|
|
206
|
+
names: "list[str]" = []
|
|
207
|
+
for row in roster:
|
|
208
|
+
if not isinstance(row, Mapping):
|
|
209
|
+
continue
|
|
210
|
+
if not bool(row.get("live", row.get("status") != "gone")):
|
|
211
|
+
continue
|
|
212
|
+
for key in ("name", "session"):
|
|
213
|
+
bare = parking.bare_session(row.get(key))
|
|
214
|
+
if WORKER_SESSION.match(bare):
|
|
215
|
+
names.append(bare)
|
|
216
|
+
break
|
|
217
|
+
return names
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
# -- the request file ---------------------------------------------------------
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def read_request(path: str) -> "dict | None":
|
|
224
|
+
"""The request file, or None when it is absent. An unreadable or
|
|
225
|
+
non-object file is None too, with a WARNING: the daemon must not crash
|
|
226
|
+
on a file the entrypoint half-wrote."""
|
|
227
|
+
try:
|
|
228
|
+
with open(path, encoding="utf-8") as fh:
|
|
229
|
+
data = json.load(fh)
|
|
230
|
+
except FileNotFoundError:
|
|
231
|
+
return None
|
|
232
|
+
except (OSError, ValueError) as exc:
|
|
233
|
+
log.warning("self-upgrade: could not read %s (%s) — ignoring it", path, exc)
|
|
234
|
+
return None
|
|
235
|
+
if not isinstance(data, dict):
|
|
236
|
+
log.warning("self-upgrade: %s is not a JSON object — ignoring it", path)
|
|
237
|
+
return None
|
|
238
|
+
return data
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def write_request(path: str, data: "Mapping") -> None:
|
|
242
|
+
"""Write the request file atomically (a temp file in the same directory,
|
|
243
|
+
then a rename), so the entrypoint never reads half a file. Raises
|
|
244
|
+
OSError."""
|
|
245
|
+
directory = os.path.dirname(path) or "."
|
|
246
|
+
os.makedirs(directory, exist_ok=True)
|
|
247
|
+
fd, tmp = tempfile.mkstemp(prefix=".devloop-", suffix=".json", dir=directory)
|
|
248
|
+
try:
|
|
249
|
+
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
|
250
|
+
json.dump(dict(data), fh, indent=2, sort_keys=True)
|
|
251
|
+
fh.write("\n")
|
|
252
|
+
os.replace(tmp, path)
|
|
253
|
+
except BaseException:
|
|
254
|
+
try:
|
|
255
|
+
os.unlink(tmp)
|
|
256
|
+
except OSError:
|
|
257
|
+
pass
|
|
258
|
+
raise
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def _target_version(data: "Mapping") -> "str | None":
|
|
262
|
+
target = data.get("target")
|
|
263
|
+
version = target.get("version") if isinstance(target, Mapping) else None
|
|
264
|
+
return version if isinstance(version, str) else None
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _count(queued: "Callable[[], int | None]") -> "int | None":
|
|
268
|
+
"""`queued()`, with a failure read as unknown (never idle)."""
|
|
269
|
+
try:
|
|
270
|
+
return queued()
|
|
271
|
+
except Exception as exc:
|
|
272
|
+
log.warning("self-upgrade: could not count the alissa queue (%s)", exc)
|
|
273
|
+
return None
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def skip_list(data: "Mapping | None") -> "list[str]":
|
|
277
|
+
if not data:
|
|
278
|
+
return []
|
|
279
|
+
skip = data.get("skip")
|
|
280
|
+
if not isinstance(skip, list):
|
|
281
|
+
return []
|
|
282
|
+
return [str(v) for v in skip if isinstance(v, str)]
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
# -- the upgrader -------------------------------------------------------------
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
class SelfUpgrader:
|
|
289
|
+
"""The daemon's self-upgrade state: the last answer, the hold, the idle
|
|
290
|
+
streak and the request file.
|
|
291
|
+
|
|
292
|
+
Every method is best-effort and never raises. The daemon's job is
|
|
293
|
+
spawning sessions, and an upgrade that cannot be done must leave the
|
|
294
|
+
seat running."""
|
|
295
|
+
|
|
296
|
+
def __init__(
|
|
297
|
+
self,
|
|
298
|
+
running: str,
|
|
299
|
+
*,
|
|
300
|
+
directory: "str | None" = None,
|
|
301
|
+
clock: "Callable[[], float]" = time.time,
|
|
302
|
+
dry_run: bool = False,
|
|
303
|
+
):
|
|
304
|
+
self.running = running
|
|
305
|
+
self.path = os.path.join(directory or upgrade_dir(), REQUEST_FILE)
|
|
306
|
+
self._clock = clock
|
|
307
|
+
self.dry_run = dry_run
|
|
308
|
+
self.answer: "UpgradeAnswer | None" = None
|
|
309
|
+
self._hold: "str | None" = None
|
|
310
|
+
self._idle_polls = 0
|
|
311
|
+
# Targets this process could not request (the file would not
|
|
312
|
+
# write): never held for again in this process, or the seat would
|
|
313
|
+
# drain for ever without ever exiting.
|
|
314
|
+
self._unwritable: "set[str]" = set()
|
|
315
|
+
self.exit_requested = False
|
|
316
|
+
|
|
317
|
+
# -- reading the answer --
|
|
318
|
+
|
|
319
|
+
def observe(self, payload: object) -> "UpgradeAnswer | None":
|
|
320
|
+
"""Read one vitals answer. Keeps the last good block (an answer
|
|
321
|
+
without one changes nothing), confirms a pending upgrade the first
|
|
322
|
+
time an answer arrives on the target version, and clears the skip
|
|
323
|
+
list when a newer target arrives. Returns the block read, or None."""
|
|
324
|
+
answer = parse_upgrade(payload)
|
|
325
|
+
if answer is None:
|
|
326
|
+
if payload is not None:
|
|
327
|
+
log.debug("self-upgrade: the vitals answer carries no upgrade block")
|
|
328
|
+
return None
|
|
329
|
+
self.answer = answer
|
|
330
|
+
try:
|
|
331
|
+
self._confirm()
|
|
332
|
+
self._unserved()
|
|
333
|
+
self._clear_skip(answer)
|
|
334
|
+
except OSError as exc:
|
|
335
|
+
log.warning("self-upgrade: could not update %s (%s)", self.path, exc)
|
|
336
|
+
self._update_hold()
|
|
337
|
+
return answer
|
|
338
|
+
|
|
339
|
+
def _confirm(self) -> None:
|
|
340
|
+
data = read_request(self.path)
|
|
341
|
+
if not data or data.get("confirmed"):
|
|
342
|
+
return
|
|
343
|
+
if data.get("state") not in PENDING_STATES:
|
|
344
|
+
# A failed or rolled-back target is never confirmed, even when
|
|
345
|
+
# a rollback that could not reinstall left the daemon on it.
|
|
346
|
+
return
|
|
347
|
+
version = _target_version(data)
|
|
348
|
+
if version != self.running or version in skip_list(data):
|
|
349
|
+
return
|
|
350
|
+
data["confirmed"] = True
|
|
351
|
+
data["confirmedAt"] = iso(self._clock())
|
|
352
|
+
data["state"] = "confirmed"
|
|
353
|
+
write_request(self.path, data)
|
|
354
|
+
log.info(
|
|
355
|
+
"self-upgrade: confirmed — this daemon runs %s and Studio answered "
|
|
356
|
+
"its first poll (upgraded from %s)", self.running,
|
|
357
|
+
data.get("previous") or "unknown",
|
|
358
|
+
)
|
|
359
|
+
|
|
360
|
+
def _unserved(self) -> None:
|
|
361
|
+
"""Skip and report a target whose last request never moved this
|
|
362
|
+
seat: the file still holds an unconfirmed, pending request that
|
|
363
|
+
THIS version wrote (`previous == running`) for another version,
|
|
364
|
+
or a request of this version's that the boot step refused as
|
|
365
|
+
`not_newer`. Asking again would only repeat the drain, the exit and
|
|
366
|
+
the restart for ever. A file that cannot be written puts the target out of
|
|
367
|
+
reach for this process instead."""
|
|
368
|
+
data = read_request(self.path)
|
|
369
|
+
if not data or data.get("confirmed"):
|
|
370
|
+
return
|
|
371
|
+
state = data.get("state")
|
|
372
|
+
version = _target_version(data)
|
|
373
|
+
if state not in UNSERVED_STATES or not version or version == self.running:
|
|
374
|
+
return
|
|
375
|
+
if data.get("previous") != self.running:
|
|
376
|
+
return
|
|
377
|
+
skip = skip_list(data)
|
|
378
|
+
if version in skip:
|
|
379
|
+
return
|
|
380
|
+
if state == "requested":
|
|
381
|
+
reason = "the boot step never acted on the request (no install was tried)"
|
|
382
|
+
elif state == "not_newer":
|
|
383
|
+
reason = (
|
|
384
|
+
"the boot step refused it: not newer than the installed "
|
|
385
|
+
f"version, but the daemon still runs {self.running}"
|
|
386
|
+
)
|
|
387
|
+
else:
|
|
388
|
+
reason = f"installed at boot, but the daemon still runs {self.running}"
|
|
389
|
+
data.update({
|
|
390
|
+
"state": "unserved", "skip": skip + [version], "lastError": reason,
|
|
391
|
+
"upgradeFailed": {"version": version, "reason": reason},
|
|
392
|
+
"upgradeFailedReported": False,
|
|
393
|
+
})
|
|
394
|
+
try:
|
|
395
|
+
write_request(self.path, data)
|
|
396
|
+
except OSError as exc:
|
|
397
|
+
self._unwritable.add(version)
|
|
398
|
+
log.warning(
|
|
399
|
+
"self-upgrade: the request for %s was never served (%s) and "
|
|
400
|
+
"%s could not be updated (%s) — this process will not try %s "
|
|
401
|
+
"again", version, reason, self.path, exc, version,
|
|
402
|
+
)
|
|
403
|
+
return
|
|
404
|
+
log.warning(
|
|
405
|
+
"self-upgrade: the request for %s → %s was never served: %s — "
|
|
406
|
+
"%s is now on the skip list and reported to Studio; this seat "
|
|
407
|
+
"keeps running %s", self.running, version, reason, version,
|
|
408
|
+
self.running,
|
|
409
|
+
)
|
|
410
|
+
|
|
411
|
+
def _clear_skip(self, answer: UpgradeAnswer) -> None:
|
|
412
|
+
version = answer.target_version
|
|
413
|
+
new = version_key(version or "")
|
|
414
|
+
if new is None:
|
|
415
|
+
return
|
|
416
|
+
data = read_request(self.path)
|
|
417
|
+
skip = skip_list(data)
|
|
418
|
+
if not skip or version in skip:
|
|
419
|
+
return
|
|
420
|
+
keys = [k for k in (version_key(v) for v in skip) if k is not None]
|
|
421
|
+
if keys and new <= max(keys):
|
|
422
|
+
return
|
|
423
|
+
assert data is not None
|
|
424
|
+
data["skip"] = []
|
|
425
|
+
write_request(self.path, data)
|
|
426
|
+
log.info(
|
|
427
|
+
"self-upgrade: target %s is newer than every skipped version "
|
|
428
|
+
"(%s) — skip list cleared", version, ", ".join(skip),
|
|
429
|
+
)
|
|
430
|
+
|
|
431
|
+
# -- the hold --
|
|
432
|
+
|
|
433
|
+
def _skipped(self, version: str) -> bool:
|
|
434
|
+
return version in skip_list(read_request(self.path))
|
|
435
|
+
|
|
436
|
+
def wanted(self) -> "str | None":
|
|
437
|
+
"""The version a self-hosted seat should move to now, or None."""
|
|
438
|
+
a = self.answer
|
|
439
|
+
if a is None or a.mode != SELF_HOSTED:
|
|
440
|
+
return None
|
|
441
|
+
if not a.auto_upgrade or a.paused:
|
|
442
|
+
return None
|
|
443
|
+
version = a.target_version
|
|
444
|
+
if not version:
|
|
445
|
+
return None
|
|
446
|
+
# Only a strictly newer target. A pin briefly older than what runs
|
|
447
|
+
# (a merge rolls the seat before the platform pin moves) must never
|
|
448
|
+
# drain and downgrade the seat. Unparseable on either side: never.
|
|
449
|
+
new, cur = version_key(version), version_key(self.running)
|
|
450
|
+
if new is None or cur is None or new <= cur:
|
|
451
|
+
return None
|
|
452
|
+
if version in self._unwritable or self._skipped(version):
|
|
453
|
+
return None
|
|
454
|
+
return version
|
|
455
|
+
|
|
456
|
+
def hold(self) -> "str | None":
|
|
457
|
+
"""Why no new worker session may start now (`HOLD_DRAIN`,
|
|
458
|
+
`HOLD_UPGRADE`), or None. Read by every spawn path."""
|
|
459
|
+
return self._hold
|
|
460
|
+
|
|
461
|
+
def hold_detail(self) -> str:
|
|
462
|
+
"""The hold in words, for a deferral's reason."""
|
|
463
|
+
if self._hold == HOLD_DRAIN:
|
|
464
|
+
return "Studio asked this seat to drain for a managed upgrade"
|
|
465
|
+
if self._hold == HOLD_UPGRADE:
|
|
466
|
+
return (
|
|
467
|
+
f"draining to self-upgrade {self.running} → "
|
|
468
|
+
f"{self.answer.target_version if self.answer else '?'}"
|
|
469
|
+
)
|
|
470
|
+
return ""
|
|
471
|
+
|
|
472
|
+
def _update_hold(self) -> None:
|
|
473
|
+
if self.answer is not None and self.answer.drain:
|
|
474
|
+
new = HOLD_DRAIN
|
|
475
|
+
elif self.wanted() is not None:
|
|
476
|
+
new = HOLD_UPGRADE
|
|
477
|
+
else:
|
|
478
|
+
new = None
|
|
479
|
+
if new == self._hold:
|
|
480
|
+
return
|
|
481
|
+
old, self._hold = self._hold, new
|
|
482
|
+
if new is None:
|
|
483
|
+
self._idle_polls = 0
|
|
484
|
+
log.info(
|
|
485
|
+
"self-upgrade: drain lifted (was: %s) — new worker sessions may "
|
|
486
|
+
"start again", old,
|
|
487
|
+
)
|
|
488
|
+
else:
|
|
489
|
+
log.info(
|
|
490
|
+
"self-upgrade: draining — %s; no new worker session starts, "
|
|
491
|
+
"running ones finish normally", self.hold_detail(),
|
|
492
|
+
)
|
|
493
|
+
|
|
494
|
+
# -- the exit --
|
|
495
|
+
|
|
496
|
+
def after_pass(
|
|
497
|
+
self,
|
|
498
|
+
roster: "Sequence[Mapping] | None",
|
|
499
|
+
queued: "Callable[[], int | None] | None" = None,
|
|
500
|
+
) -> str:
|
|
501
|
+
"""Called once per completed pass, after the vitals push. Returns
|
|
502
|
+
EXIT when the daemon should exit now for the upgrade (the request is
|
|
503
|
+
written), WAIT while it is draining towards one, NONE otherwise.
|
|
504
|
+
|
|
505
|
+
`roster` is the pass's ``alissa tmux ls`` listing. `queued` counts
|
|
506
|
+
the undispatched items in the alissa tmux queue (None: unknown). It
|
|
507
|
+
is read only when the roster is idle: an item queued before the
|
|
508
|
+
hold came on is in neither listing yet, and a restart would kill the
|
|
509
|
+
session it dispatches to."""
|
|
510
|
+
# Re-checked here, not only at the hold's start: a skip list the
|
|
511
|
+
# entrypoint just wrote, or a target that moved, must stop the exit.
|
|
512
|
+
self._update_hold()
|
|
513
|
+
version = self.wanted()
|
|
514
|
+
if version is None or self.answer is None:
|
|
515
|
+
self._idle_polls = 0
|
|
516
|
+
return NONE
|
|
517
|
+
live = live_worker_sessions(roster)
|
|
518
|
+
if live is None or live:
|
|
519
|
+
if self._idle_polls:
|
|
520
|
+
log.info(
|
|
521
|
+
"self-upgrade: not idle any more (%s) — idle count reset",
|
|
522
|
+
"the session listing failed" if live is None
|
|
523
|
+
else ", ".join(live),
|
|
524
|
+
)
|
|
525
|
+
self._idle_polls = 0
|
|
526
|
+
return WAIT
|
|
527
|
+
waiting = 0 if queued is None else _count(queued)
|
|
528
|
+
if waiting is None or waiting:
|
|
529
|
+
if self._idle_polls:
|
|
530
|
+
log.info(
|
|
531
|
+
"self-upgrade: not idle any more (%s) — idle count reset",
|
|
532
|
+
"the alissa queue listing failed" if waiting is None
|
|
533
|
+
else f"{waiting} undispatched item(s) in the alissa queue",
|
|
534
|
+
)
|
|
535
|
+
self._idle_polls = 0
|
|
536
|
+
return WAIT
|
|
537
|
+
self._idle_polls += 1
|
|
538
|
+
if self._idle_polls < IDLE_POLLS:
|
|
539
|
+
log.info(
|
|
540
|
+
"self-upgrade: idle on poll %d of %d before the exit to %s",
|
|
541
|
+
self._idle_polls, IDLE_POLLS, version,
|
|
542
|
+
)
|
|
543
|
+
return WAIT
|
|
544
|
+
if self.dry_run:
|
|
545
|
+
log.info(
|
|
546
|
+
"[dry-run] self-upgrade: idle — would write %s and exit to "
|
|
547
|
+
"upgrade %s → %s", self.path, self.running, version,
|
|
548
|
+
)
|
|
549
|
+
return WAIT
|
|
550
|
+
try:
|
|
551
|
+
self._request(self.answer)
|
|
552
|
+
except OSError as exc:
|
|
553
|
+
self._unwritable.add(version)
|
|
554
|
+
log.warning(
|
|
555
|
+
"self-upgrade: could not write %s (%s) — NOT upgrading to %s; "
|
|
556
|
+
"the drain is lifted and this process will not try %s again",
|
|
557
|
+
self.path, exc, version, version,
|
|
558
|
+
)
|
|
559
|
+
self._update_hold()
|
|
560
|
+
return NONE
|
|
561
|
+
self.exit_requested = True
|
|
562
|
+
log.info(
|
|
563
|
+
"self-upgrade: idle on %d consecutive polls — wrote %s; exiting "
|
|
564
|
+
"with code 0 so the container restarts and installs %s (from %s)",
|
|
565
|
+
IDLE_POLLS, self.path, version, self.running,
|
|
566
|
+
)
|
|
567
|
+
return EXIT
|
|
568
|
+
|
|
569
|
+
def _request(self, answer: UpgradeAnswer) -> None:
|
|
570
|
+
old = read_request(self.path) or {}
|
|
571
|
+
data: dict = {
|
|
572
|
+
"target": answer.target,
|
|
573
|
+
"previous": self.running,
|
|
574
|
+
"requestedAt": iso(self._clock()),
|
|
575
|
+
"state": "requested",
|
|
576
|
+
"confirmed": False,
|
|
577
|
+
"boots": 0,
|
|
578
|
+
"skip": skip_list(old),
|
|
579
|
+
}
|
|
580
|
+
# A report not yet delivered survives a new request.
|
|
581
|
+
if old.get("upgradeFailed") and not old.get("upgradeFailedReported"):
|
|
582
|
+
data["upgradeFailed"] = old["upgradeFailed"]
|
|
583
|
+
data["upgradeFailedReported"] = False
|
|
584
|
+
write_request(self.path, data)
|
|
585
|
+
|
|
586
|
+
# -- the failure report --
|
|
587
|
+
|
|
588
|
+
def report(self) -> "dict | None":
|
|
589
|
+
"""The ``upgradeFailed`` report the next snapshot owes Studio, or
|
|
590
|
+
None. Only a well-formed `{version, reason}` is offered: the request
|
|
591
|
+
schema is strict."""
|
|
592
|
+
data = read_request(self.path)
|
|
593
|
+
if not data or data.get("upgradeFailedReported"):
|
|
594
|
+
return None
|
|
595
|
+
failed = data.get("upgradeFailed")
|
|
596
|
+
if not isinstance(failed, Mapping):
|
|
597
|
+
return None
|
|
598
|
+
version, reason = failed.get("version"), failed.get("reason")
|
|
599
|
+
if not isinstance(version, str) or not re.fullmatch(r"[A-Za-z0-9._-]{1,128}", version):
|
|
600
|
+
return None
|
|
601
|
+
reason = " ".join(str(reason or "rolled back").split())[:300] or "rolled back"
|
|
602
|
+
return {"version": version, "reason": reason}
|
|
603
|
+
|
|
604
|
+
def reported(self, sent: "Mapping") -> None:
|
|
605
|
+
"""Mark the report delivered, after a POST carrying `sent`
|
|
606
|
+
succeeded."""
|
|
607
|
+
data = read_request(self.path)
|
|
608
|
+
if not data:
|
|
609
|
+
return
|
|
610
|
+
failed = data.get("upgradeFailed")
|
|
611
|
+
if not isinstance(failed, Mapping) or failed.get("version") != sent.get("version"):
|
|
612
|
+
return
|
|
613
|
+
data["upgradeFailedReported"] = True
|
|
614
|
+
try:
|
|
615
|
+
write_request(self.path, data)
|
|
616
|
+
except OSError as exc:
|
|
617
|
+
log.warning("self-upgrade: could not update %s (%s)", self.path, exc)
|
|
618
|
+
return
|
|
619
|
+
log.info(
|
|
620
|
+
"self-upgrade: reported to Studio that %s would not start "
|
|
621
|
+
"(rolled back to %s)", sent.get("version"), self.running,
|
|
622
|
+
)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
0.8.62
|
|
@@ -21,6 +21,7 @@ src/main/alissa/tools/github/devloop/loop_events.py
|
|
|
21
21
|
src/main/alissa/tools/github/devloop/parking.py
|
|
22
22
|
src/main/alissa/tools/github/devloop/proc.py
|
|
23
23
|
src/main/alissa/tools/github/devloop/prompts.py
|
|
24
|
+
src/main/alissa/tools/github/devloop/self_upgrade.py
|
|
24
25
|
src/main/alissa/tools/github/devloop/state.py
|
|
25
26
|
src/main/alissa/tools/github/devloop/version
|
|
26
27
|
src/main/alissa/tools/github/devloop/version.py
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
0.8.61
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|