tuneplane-node 0.3.15__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tuneplane_node/__init__.py +1 -0
- tuneplane_node/allocator.py +596 -0
- tuneplane_node/cli.py +189 -0
- tuneplane_node/daemon.py +497 -0
- tuneplane_node/inventory.py +163 -0
- tuneplane_node/join.py +199 -0
- tuneplane_node/runtime.py +611 -0
- tuneplane_node/settings.py +77 -0
- tuneplane_node/wire.py +85 -0
- tuneplane_node-0.3.15.dist-info/METADATA +12 -0
- tuneplane_node-0.3.15.dist-info/RECORD +13 -0
- tuneplane_node-0.3.15.dist-info/WHEEL +4 -0
- tuneplane_node-0.3.15.dist-info/entry_points.txt +2 -0
tuneplane_node/join.py
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
"""Joining a Fleet, and staying in it.
|
|
2
|
+
|
|
3
|
+
Two calls and a loop. What makes this small is where the authorization lives:
|
|
4
|
+
the Join Token is the whole of it (ADR-0013), so there is no login, no
|
|
5
|
+
credential file to provision ahead of time, and nothing for the operator to
|
|
6
|
+
have an account for. What comes back is this node's identity and its long-lived
|
|
7
|
+
credential, and those are what every later request carries.
|
|
8
|
+
|
|
9
|
+
The credential is returned exactly once. Losing it means asking an administrator
|
|
10
|
+
for another Join Token and joining again as a new node, which is the correct
|
|
11
|
+
outcome: a machine that lost its identity is not the same machine to a control
|
|
12
|
+
plane that has to reconcile containers against it.
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import asyncio
|
|
17
|
+
import json
|
|
18
|
+
import logging
|
|
19
|
+
from dataclasses import dataclass
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Any, Optional
|
|
22
|
+
|
|
23
|
+
import httpx
|
|
24
|
+
|
|
25
|
+
from tuneplane_node.wire import PROTOCOL
|
|
26
|
+
|
|
27
|
+
log = logging.getLogger(__name__)
|
|
28
|
+
|
|
29
|
+
#: Where the node keeps what registration gave it. Beside the agent's own state
|
|
30
|
+
#: rather than in the storage root, because a node that cannot see the storage
|
|
31
|
+
#: root still has to be able to say who it is.
|
|
32
|
+
CREDENTIAL_FILE = "node.json"
|
|
33
|
+
|
|
34
|
+
#: How often a node says it is still there. Well inside the console's tightest
|
|
35
|
+
#: liveness threshold (serve, seconds) so that a single missed beat is not read
|
|
36
|
+
#: as a machine going away.
|
|
37
|
+
HEARTBEAT_INTERVAL_S = 15.0
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass(frozen=True)
|
|
41
|
+
class NodeIdentity:
|
|
42
|
+
#: Where the control plane answers. Named `server` because that is the word
|
|
43
|
+
#: an operator arriving from k3s, Nomad or Consul reaches for; the thing it
|
|
44
|
+
#: points at is this platform's console.
|
|
45
|
+
server: str
|
|
46
|
+
fleet_id: str
|
|
47
|
+
node_id: str
|
|
48
|
+
credential: str
|
|
49
|
+
|
|
50
|
+
def to_dict(self) -> dict[str, Any]:
|
|
51
|
+
return {
|
|
52
|
+
"server": self.server,
|
|
53
|
+
"fleet_id": self.fleet_id,
|
|
54
|
+
"node_id": self.node_id,
|
|
55
|
+
"credential": self.credential,
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class JoinError(RuntimeError):
|
|
60
|
+
"""Registration was refused, with the console's reason.
|
|
61
|
+
|
|
62
|
+
Carries the console's own wording rather than a summary: the refusals are
|
|
63
|
+
written to be acted on ("mount it, or join a fleet that pulls from object
|
|
64
|
+
storage"), and rewording them here would lose that.
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def identity_path(state_dir: Path) -> Path:
|
|
69
|
+
return Path(state_dir) / CREDENTIAL_FILE
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def load_identity(state_dir: Path) -> Optional[NodeIdentity]:
|
|
73
|
+
path = identity_path(state_dir)
|
|
74
|
+
if not path.exists():
|
|
75
|
+
return None
|
|
76
|
+
try:
|
|
77
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
78
|
+
return NodeIdentity(
|
|
79
|
+
server=str(data["server"]),
|
|
80
|
+
fleet_id=str(data["fleet_id"]),
|
|
81
|
+
node_id=str(data["node_id"]),
|
|
82
|
+
credential=str(data["credential"]),
|
|
83
|
+
)
|
|
84
|
+
except (ValueError, KeyError, OSError) as exc:
|
|
85
|
+
raise JoinError(f"{path} is not a readable node identity: {exc}") from exc
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def save_identity(state_dir: Path, identity: NodeIdentity) -> Path:
|
|
89
|
+
path = identity_path(state_dir)
|
|
90
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
91
|
+
path.write_text(json.dumps(identity.to_dict(), indent=2) + "\n", encoding="utf-8")
|
|
92
|
+
# It is a bearer credential; nothing else on the box needs to read it.
|
|
93
|
+
path.chmod(0o600)
|
|
94
|
+
return path
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
async def join(
|
|
98
|
+
server: str,
|
|
99
|
+
*,
|
|
100
|
+
fleet_id: str,
|
|
101
|
+
token: str,
|
|
102
|
+
address: str,
|
|
103
|
+
inventory: dict[str, Any],
|
|
104
|
+
node_version: str,
|
|
105
|
+
connection: str = "direct",
|
|
106
|
+
timeout: float = 30.0,
|
|
107
|
+
transport=None,
|
|
108
|
+
) -> tuple[NodeIdentity, list[str]]:
|
|
109
|
+
"""Redeem a Join Token for this node's identity, plus anything the console warns about.
|
|
110
|
+
|
|
111
|
+
The warnings are returned rather than saved: they describe the state of the
|
|
112
|
+
fleet at this moment, and the identity file is meant to outlive that.
|
|
113
|
+
|
|
114
|
+
transport is injectable for tests, the same way `NodeClient` takes one.
|
|
115
|
+
"""
|
|
116
|
+
base = server.rstrip("/")
|
|
117
|
+
payload = {
|
|
118
|
+
"token": token,
|
|
119
|
+
"address": address,
|
|
120
|
+
"connection": connection,
|
|
121
|
+
"node_version": node_version,
|
|
122
|
+
"protocol": PROTOCOL,
|
|
123
|
+
"inventory": inventory,
|
|
124
|
+
}
|
|
125
|
+
async with httpx.AsyncClient(timeout=timeout, transport=transport) as client:
|
|
126
|
+
response = await client.post(f"{base}/api/fleets/{fleet_id}/nodes", json=payload)
|
|
127
|
+
if response.status_code != 201:
|
|
128
|
+
raise JoinError(_detail(response))
|
|
129
|
+
body = response.json()
|
|
130
|
+
identity = NodeIdentity(
|
|
131
|
+
server=base,
|
|
132
|
+
fleet_id=fleet_id,
|
|
133
|
+
node_id=str(body["id"]),
|
|
134
|
+
credential=str(body["credential"]),
|
|
135
|
+
)
|
|
136
|
+
return identity, [str(w) for w in (body.get("warnings") or [])]
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
async def beat(
|
|
140
|
+
identity: NodeIdentity,
|
|
141
|
+
*,
|
|
142
|
+
inventory: dict[str, Any],
|
|
143
|
+
node_version: str,
|
|
144
|
+
address: str = "",
|
|
145
|
+
timeout: float = 15.0,
|
|
146
|
+
transport=None,
|
|
147
|
+
) -> dict[str, Any]:
|
|
148
|
+
"""Tell the console this node is still there, and re-report what it has."""
|
|
149
|
+
url = f"{identity.server}/api/fleets/{identity.fleet_id}/nodes/{identity.node_id}/heartbeat"
|
|
150
|
+
payload = {
|
|
151
|
+
"node_version": node_version,
|
|
152
|
+
"protocol": PROTOCOL,
|
|
153
|
+
"address": address,
|
|
154
|
+
"inventory": inventory,
|
|
155
|
+
}
|
|
156
|
+
async with httpx.AsyncClient(timeout=timeout, transport=transport) as client:
|
|
157
|
+
response = await client.post(
|
|
158
|
+
url, json=payload, headers={"Authorization": f"Bearer {identity.credential}"}
|
|
159
|
+
)
|
|
160
|
+
if response.status_code != 200:
|
|
161
|
+
raise JoinError(_detail(response))
|
|
162
|
+
return response.json()
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
async def heartbeat_forever(
|
|
166
|
+
identity: NodeIdentity,
|
|
167
|
+
settings,
|
|
168
|
+
*,
|
|
169
|
+
node_version: str,
|
|
170
|
+
address: str = "",
|
|
171
|
+
interval_s: float = HEARTBEAT_INTERVAL_S,
|
|
172
|
+
) -> None:
|
|
173
|
+
"""Beat until cancelled. A failed beat is logged and retried, never fatal.
|
|
174
|
+
|
|
175
|
+
Losing contact with the console is not a reason to stop running work: the
|
|
176
|
+
containers on this machine are the truth about what is running, and the
|
|
177
|
+
console reconciles against them when it comes back. That is the same rule
|
|
178
|
+
ADR-002 states from the other side -- an unreachable node is not a dead job.
|
|
179
|
+
"""
|
|
180
|
+
from tuneplane_node.inventory import collect
|
|
181
|
+
|
|
182
|
+
while True:
|
|
183
|
+
try:
|
|
184
|
+
inventory = (await collect(settings)).to_dict()
|
|
185
|
+
await beat(identity, inventory=inventory, node_version=node_version, address=address)
|
|
186
|
+
except asyncio.CancelledError:
|
|
187
|
+
raise
|
|
188
|
+
except Exception as exc: # noqa: BLE001
|
|
189
|
+
log.warning("heartbeat to %s failed: %s", identity.server, exc)
|
|
190
|
+
await asyncio.sleep(interval_s)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _detail(response: httpx.Response) -> str:
|
|
194
|
+
try:
|
|
195
|
+
body = response.json()
|
|
196
|
+
except ValueError:
|
|
197
|
+
return f"{response.status_code}: {response.text[:200]}"
|
|
198
|
+
detail = body.get("detail") if isinstance(body, dict) else None
|
|
199
|
+
return str(detail or f"{response.status_code}: {response.text[:200]}")
|