offbox 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- offbox/__init__.py +2 -0
- offbox/__main__.py +4 -0
- offbox/_data/host/cloud-init.yaml.j2 +15 -0
- offbox/_data/host/host-init.sh +230 -0
- offbox/_data/images/universal/Dockerfile +101 -0
- offbox/_data/images/universal/offbox-init +75 -0
- offbox/_data/images/universal/offbox-run +131 -0
- offbox/_data/providers/gcp/.terraform.lock.hcl +52 -0
- offbox/_data/providers/gcp/main.tf +67 -0
- offbox/_data/providers/gcp/outputs.tf +7 -0
- offbox/_data/providers/gcp/provider.toml +9 -0
- offbox/_data/providers/gcp/variables.tf +33 -0
- offbox/_data/providers/gcp/versions.tf +12 -0
- offbox/_data/providers/hetzner/.terraform.lock.hcl +51 -0
- offbox/_data/providers/hetzner/main.tf +32 -0
- offbox/_data/providers/hetzner/outputs.tf +7 -0
- offbox/_data/providers/hetzner/provider.toml +9 -0
- offbox/_data/providers/hetzner/variables.tf +28 -0
- offbox/_data/providers/hetzner/versions.tf +12 -0
- offbox/_data/providers/ssh/provider.toml +4 -0
- offbox/cli/__init__.py +117 -0
- offbox/cli/config.py +39 -0
- offbox/cli/env.py +109 -0
- offbox/cli/exec.py +50 -0
- offbox/cli/host.py +313 -0
- offbox/cli/misc.py +132 -0
- offbox/cli/ports.py +44 -0
- offbox/config.py +189 -0
- offbox/console.py +63 -0
- offbox/doctor.py +174 -0
- offbox/drivers/__init__.py +0 -0
- offbox/drivers/base.py +40 -0
- offbox/drivers/none.py +114 -0
- offbox/drivers/tofu.py +59 -0
- offbox/errors.py +45 -0
- offbox/execute.py +772 -0
- offbox/host/__init__.py +89 -0
- offbox/host/activity.py +58 -0
- offbox/host/auto.py +322 -0
- offbox/host/cloudinit.py +44 -0
- offbox/host/create.py +198 -0
- offbox/host/destroy.py +151 -0
- offbox/host/init.py +196 -0
- offbox/host/ready.py +106 -0
- offbox/host/timer.py +227 -0
- offbox/host/usage.py +123 -0
- offbox/host/warnings.py +61 -0
- offbox/locking.py +27 -0
- offbox/logging.py +154 -0
- offbox/logs.py +135 -0
- offbox/naming.py +67 -0
- offbox/paths.py +55 -0
- offbox/ports.py +610 -0
- offbox/project.py +187 -0
- offbox/providers.py +117 -0
- offbox/py.typed +0 -0
- offbox/runner.py +291 -0
- offbox/runtime.py +887 -0
- offbox/setup.py +181 -0
- offbox/share.py +286 -0
- offbox/ssh.py +179 -0
- offbox/state.py +157 -0
- offbox/status.py +378 -0
- offbox/sync.py +380 -0
- offbox/timings.py +69 -0
- offbox/tools.py +227 -0
- offbox/uninstall.py +114 -0
- offbox-0.1.0.dist-info/METADATA +140 -0
- offbox-0.1.0.dist-info/RECORD +72 -0
- offbox-0.1.0.dist-info/WHEEL +4 -0
- offbox-0.1.0.dist-info/entry_points.txt +2 -0
- offbox-0.1.0.dist-info/licenses/LICENSE +21 -0
offbox/sync.py
ADDED
|
@@ -0,0 +1,380 @@
|
|
|
1
|
+
import csv
|
|
2
|
+
import io
|
|
3
|
+
import logging
|
|
4
|
+
import subprocess
|
|
5
|
+
from collections.abc import Callable
|
|
6
|
+
from datetime import datetime
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Literal
|
|
9
|
+
|
|
10
|
+
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
|
|
11
|
+
|
|
12
|
+
from offbox import console
|
|
13
|
+
from offbox.config import Model, SyncConfig
|
|
14
|
+
from offbox.errors import CommandFailed, PreconditionError
|
|
15
|
+
from offbox.naming import Worktree, validate_env_name
|
|
16
|
+
from offbox.paths import Paths
|
|
17
|
+
from offbox.providers import validate_name
|
|
18
|
+
from offbox.runner import Result, Runner
|
|
19
|
+
from offbox.state import HostState
|
|
20
|
+
from offbox.timings import measured
|
|
21
|
+
from offbox.tools import MUTAGEN_FIX, MutagenMissing, cached_mutagen, download_prompt, install_mutagen
|
|
22
|
+
|
|
23
|
+
logger = logging.getLogger(__name__)
|
|
24
|
+
|
|
25
|
+
DEFAULT_IGNORES = (
|
|
26
|
+
"node_modules",
|
|
27
|
+
".venv",
|
|
28
|
+
"venv",
|
|
29
|
+
"__pycache__",
|
|
30
|
+
"*.pyc",
|
|
31
|
+
".pytest_cache",
|
|
32
|
+
".mypy_cache",
|
|
33
|
+
".ruff_cache",
|
|
34
|
+
"dist",
|
|
35
|
+
"build",
|
|
36
|
+
".next",
|
|
37
|
+
".nuxt",
|
|
38
|
+
"target",
|
|
39
|
+
".terraform",
|
|
40
|
+
".DS_Store",
|
|
41
|
+
".git",
|
|
42
|
+
)
|
|
43
|
+
SyncState = Literal[
|
|
44
|
+
"disconnected",
|
|
45
|
+
"halted-on-root-emptied",
|
|
46
|
+
"halted-on-root-deletion",
|
|
47
|
+
"halted-on-root-type-change",
|
|
48
|
+
"connecting-alpha",
|
|
49
|
+
"connecting-beta",
|
|
50
|
+
"watching",
|
|
51
|
+
"scanning",
|
|
52
|
+
"waiting-for-rescan",
|
|
53
|
+
"reconciling",
|
|
54
|
+
"staging-alpha",
|
|
55
|
+
"staging-beta",
|
|
56
|
+
"transitioning",
|
|
57
|
+
"saving",
|
|
58
|
+
"unknown",
|
|
59
|
+
]
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class MutagenModel(BaseModel):
|
|
63
|
+
model_config = ConfigDict(extra="ignore", populate_by_name=True, hide_input_in_errors=True)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class IgnoreConfig(MutagenModel):
|
|
67
|
+
paths: list[str] = Field(default_factory=list[str])
|
|
68
|
+
vcs: bool | None = None
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class Permissions(MutagenModel):
|
|
72
|
+
file_mode: str | None = Field(default=None, alias="defaultFileMode")
|
|
73
|
+
directory_mode: str | None = Field(default=None, alias="defaultDirectoryMode")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class SyncProblem(MutagenModel):
|
|
77
|
+
path: str
|
|
78
|
+
error: str
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class Conflict(MutagenModel):
|
|
82
|
+
root: str
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class Endpoint(MutagenModel):
|
|
86
|
+
protocol: Literal["local", "ssh", "docker"]
|
|
87
|
+
path: str
|
|
88
|
+
host: str | None = None
|
|
89
|
+
user: str | None = None
|
|
90
|
+
port: int | None = None
|
|
91
|
+
environment: dict[str, str] = Field(default_factory=dict[str, str])
|
|
92
|
+
parameters: dict[str, str] = Field(default_factory=dict[str, str])
|
|
93
|
+
connected: bool
|
|
94
|
+
permissions: Permissions = Field(default_factory=Permissions)
|
|
95
|
+
scan_problems: list[SyncProblem] = Field(default_factory=list[SyncProblem], alias="scanProblems")
|
|
96
|
+
transition_problems: list[SyncProblem] = Field(default_factory=list[SyncProblem], alias="transitionProblems")
|
|
97
|
+
excluded_scan_problems: int = Field(default=0, ge=0, alias="excludedScanProblems")
|
|
98
|
+
excluded_transition_problems: int = Field(default=0, ge=0, alias="excludedTransitionProblems")
|
|
99
|
+
|
|
100
|
+
@property
|
|
101
|
+
def problem_count(self) -> int:
|
|
102
|
+
return len(self.scan_problems) + len(self.transition_problems) + self.excluded_scan_problems + self.excluded_transition_problems
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class SyncSession(MutagenModel):
|
|
106
|
+
identifier: str = Field(pattern=r"^[A-Za-z0-9_-]+$")
|
|
107
|
+
version: Literal[1]
|
|
108
|
+
creation_time: datetime = Field(alias="creationTime")
|
|
109
|
+
creating_version: str = Field(alias="creatingVersion")
|
|
110
|
+
name: str = ""
|
|
111
|
+
labels: dict[str, str] = Field(default_factory=dict[str, str])
|
|
112
|
+
alpha: Endpoint
|
|
113
|
+
beta: Endpoint
|
|
114
|
+
paused: bool
|
|
115
|
+
status: SyncState
|
|
116
|
+
mode: str | None = None
|
|
117
|
+
ignore: IgnoreConfig = Field(default_factory=IgnoreConfig)
|
|
118
|
+
conflicts: list[Conflict] = Field(default_factory=list[Conflict])
|
|
119
|
+
excluded_conflicts: int = Field(default=0, ge=0, alias="excludedConflicts")
|
|
120
|
+
last_error: str | None = Field(default=None, alias="lastError")
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
class SyncHealth(Model):
|
|
124
|
+
state: SyncState
|
|
125
|
+
paused: bool
|
|
126
|
+
conflicts: int
|
|
127
|
+
problems: int
|
|
128
|
+
last_error: str | None = None
|
|
129
|
+
ok: bool
|
|
130
|
+
fix: str | None = None
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def health(session: SyncSession) -> SyncHealth:
|
|
134
|
+
conflicts = len(session.conflicts) + session.excluded_conflicts
|
|
135
|
+
problems = session.alpha.problem_count + session.beta.problem_count
|
|
136
|
+
fix: str | None = None
|
|
137
|
+
if session.paused:
|
|
138
|
+
fix = "Run offbox up to resume source sync."
|
|
139
|
+
elif conflicts:
|
|
140
|
+
paths = ", ".join(conflict.root or "<root>" for conflict in session.conflicts) or "excluded paths"
|
|
141
|
+
fix = f"Resolve sync conflicts locally ({paths}), then run mutagen sync reset {session.identifier} and offbox up."
|
|
142
|
+
elif session.status.startswith("halted-on-root-"):
|
|
143
|
+
fix = f"Restore the worktree root, then run mutagen sync reset {session.identifier} and offbox up."
|
|
144
|
+
elif session.status == "unknown":
|
|
145
|
+
fix = "Check the Mutagen daemon and install a supported Mutagen 0.18.x version, then run offbox up."
|
|
146
|
+
elif problems:
|
|
147
|
+
details = [
|
|
148
|
+
problem for endpoint in (session.alpha, session.beta) for problem in (*endpoint.scan_problems, *endpoint.transition_problems)
|
|
149
|
+
]
|
|
150
|
+
paths = ", ".join(problem.path or "<root>" for problem in details) or "excluded paths"
|
|
151
|
+
fix = f"Correct sync access problems ({paths}) and run offbox up."
|
|
152
|
+
elif session.last_error or not session.alpha.connected or not session.beta.connected:
|
|
153
|
+
fix = "Check host connectivity with offbox host ssh, then run offbox up."
|
|
154
|
+
return SyncHealth(
|
|
155
|
+
state=session.status,
|
|
156
|
+
paused=session.paused,
|
|
157
|
+
conflicts=conflicts,
|
|
158
|
+
problems=problems,
|
|
159
|
+
last_error=session.last_error,
|
|
160
|
+
ok=fix is None,
|
|
161
|
+
fix=fix,
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def ignores(config: SyncConfig) -> list[str]:
|
|
166
|
+
if any(not pattern or any(character in pattern for character in "\x00\r\n") for pattern in config.ignore):
|
|
167
|
+
raise PreconditionError("Invalid sync ignore pattern", "Set sync.ignore to nonempty, single-line patterns without NUL characters.")
|
|
168
|
+
return [*(pattern for pattern in (*DEFAULT_IGNORES, *config.ignore) if pattern != ".git"), ".git"]
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _csv_pattern(pattern: str) -> str:
|
|
172
|
+
output = io.StringIO()
|
|
173
|
+
csv.writer(output, lineterminator="\n").writerow([pattern])
|
|
174
|
+
return output.getvalue()[:-1]
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def create_command(work: Worktree, host: HostState, config: SyncConfig) -> list[str]:
|
|
178
|
+
argv = [
|
|
179
|
+
"mutagen",
|
|
180
|
+
"sync",
|
|
181
|
+
"create",
|
|
182
|
+
"--no-global-configuration",
|
|
183
|
+
"--name",
|
|
184
|
+
f"offbox-{work.name}",
|
|
185
|
+
"--label",
|
|
186
|
+
f"offbox-env={work.name}",
|
|
187
|
+
"--mode",
|
|
188
|
+
"two-way-safe",
|
|
189
|
+
"--ignore-vcs",
|
|
190
|
+
"--default-file-mode-beta=0644",
|
|
191
|
+
"--default-directory-mode-beta=0755",
|
|
192
|
+
]
|
|
193
|
+
for pattern in ignores(config):
|
|
194
|
+
argv.extend(["--ignore", _csv_pattern(pattern)])
|
|
195
|
+
return [*argv, str(work.root), f"offbox-{host.name}:{work.work}"]
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
class SyncManager:
|
|
199
|
+
def __init__(self, runner: Runner, *, cache: Path | None = None, offer: Callable[[str], bool] | None = None) -> None:
|
|
200
|
+
self.runner = runner
|
|
201
|
+
self.cache = cache
|
|
202
|
+
self.offer = offer
|
|
203
|
+
self.executable = "mutagen"
|
|
204
|
+
|
|
205
|
+
def _locate(self, install: bool) -> str:
|
|
206
|
+
cache = self.cache or Paths.discover().cache
|
|
207
|
+
cached = cached_mutagen(cache)
|
|
208
|
+
if cached is not None:
|
|
209
|
+
return str(cached)
|
|
210
|
+
offer = self.offer or (console.confirm if console.interactive() else None)
|
|
211
|
+
if install and offer is not None and offer(download_prompt(cache)):
|
|
212
|
+
return str(install_mutagen(cache))
|
|
213
|
+
raise MutagenMissing("Mutagen is not installed", MUTAGEN_FIX)
|
|
214
|
+
|
|
215
|
+
def _run(self, argv: list[str], *, timeout: float = 30, install: bool = True) -> Result:
|
|
216
|
+
command = [self.executable, *argv[1:]]
|
|
217
|
+
try:
|
|
218
|
+
result = self.runner.run(command, check=False, timeout=timeout)
|
|
219
|
+
except PreconditionError as exc:
|
|
220
|
+
if isinstance(exc.__cause__, FileNotFoundError) and self.executable == "mutagen":
|
|
221
|
+
self.executable = self._locate(install)
|
|
222
|
+
return self._run(argv, timeout=timeout, install=install)
|
|
223
|
+
if isinstance(exc.__cause__, subprocess.TimeoutExpired):
|
|
224
|
+
raise PreconditionError("Source sync timed out", "Check sync conflicts and host connectivity, then run offbox up.") from exc
|
|
225
|
+
raise
|
|
226
|
+
if result.returncode:
|
|
227
|
+
failure = CommandFailed(command, result.returncode, result.stderr)
|
|
228
|
+
failure.fix = "Check the Mutagen daemon and SSH connectivity, then run offbox up."
|
|
229
|
+
raise failure
|
|
230
|
+
return result
|
|
231
|
+
|
|
232
|
+
def status(self, work: Worktree, host: HostState | None = None, *, install: bool = True) -> SyncSession | None:
|
|
233
|
+
result = self._run(
|
|
234
|
+
["mutagen", "sync", "list", "--label-selector", f"offbox-env={work.name}", "--template", "{{ json . }}"], install=install
|
|
235
|
+
)
|
|
236
|
+
try:
|
|
237
|
+
sessions = TypeAdapter(list[SyncSession]).validate_json(result.stdout)
|
|
238
|
+
except ValidationError as exc:
|
|
239
|
+
raise PreconditionError("Invalid Mutagen session response", "Install Mutagen 0.18.x and inspect the sync daemon.") from exc
|
|
240
|
+
if not sessions:
|
|
241
|
+
return None
|
|
242
|
+
if len(sessions) != 1:
|
|
243
|
+
raise PreconditionError(
|
|
244
|
+
f"Several sync sessions claim {work.name}", "Inspect mutagen sync list and remove duplicate managed sessions."
|
|
245
|
+
)
|
|
246
|
+
session = sessions[0]
|
|
247
|
+
if (
|
|
248
|
+
session.name != f"offbox-{work.name}"
|
|
249
|
+
or session.labels.get("offbox-env") != work.name
|
|
250
|
+
or session.alpha.user
|
|
251
|
+
or session.alpha.host
|
|
252
|
+
or session.alpha.port
|
|
253
|
+
or session.alpha.environment
|
|
254
|
+
or session.alpha.parameters
|
|
255
|
+
or session.beta.user
|
|
256
|
+
or session.beta.port
|
|
257
|
+
or session.beta.environment
|
|
258
|
+
or session.beta.parameters
|
|
259
|
+
or session.alpha.protocol != "local"
|
|
260
|
+
or Path(session.alpha.path) != work.root
|
|
261
|
+
or session.beta.protocol != "ssh"
|
|
262
|
+
or session.beta.path != work.work
|
|
263
|
+
or (host is not None and session.beta.host != f"offbox-{host.name}")
|
|
264
|
+
):
|
|
265
|
+
raise PreconditionError(
|
|
266
|
+
"Sync session has conflicting ownership or endpoints", "Inspect mutagen sync list before changing this environment."
|
|
267
|
+
)
|
|
268
|
+
return session
|
|
269
|
+
|
|
270
|
+
@measured("Sync setup")
|
|
271
|
+
def ensure(self, work: Worktree, host: HostState, config: SyncConfig) -> None:
|
|
272
|
+
session = self.status(work, host)
|
|
273
|
+
if session is None:
|
|
274
|
+
self._run(create_command(work, host, config))
|
|
275
|
+
return
|
|
276
|
+
if (
|
|
277
|
+
session.mode != "two-way-safe"
|
|
278
|
+
or session.ignore.vcs is not True
|
|
279
|
+
or session.ignore.paths != ignores(config)
|
|
280
|
+
or session.beta.permissions.file_mode != "0644"
|
|
281
|
+
or session.beta.permissions.directory_mode != "0755"
|
|
282
|
+
):
|
|
283
|
+
raise PreconditionError(
|
|
284
|
+
"Sync configuration changed",
|
|
285
|
+
f"Run mutagen sync terminate {session.identifier}, then offbox up to apply the new configuration.",
|
|
286
|
+
)
|
|
287
|
+
if session.paused:
|
|
288
|
+
self._run(["mutagen", "sync", "resume", session.identifier])
|
|
289
|
+
|
|
290
|
+
@measured("Sync flush")
|
|
291
|
+
def flush(self, work: Worktree, host: HostState | None = None) -> None:
|
|
292
|
+
try:
|
|
293
|
+
self._run(["mutagen", "sync", "flush", "--label-selector", f"offbox-env={work.name}"], timeout=120)
|
|
294
|
+
except CommandFailed:
|
|
295
|
+
logger.info("label flush failed for %s; inspecting the session", work.name, exc_info=True)
|
|
296
|
+
self._recover_and_flush(work, host)
|
|
297
|
+
current = self.status(work, host)
|
|
298
|
+
if current is None:
|
|
299
|
+
raise PreconditionError("No source sync session", "Run offbox up in this worktree.")
|
|
300
|
+
self._raise_health(current)
|
|
301
|
+
|
|
302
|
+
def _recover_and_flush(self, work: Worktree, host: HostState | None) -> None:
|
|
303
|
+
session = self.status(work, host)
|
|
304
|
+
if session is None:
|
|
305
|
+
raise PreconditionError("No source sync session", "Run offbox up in this worktree.")
|
|
306
|
+
if session.paused:
|
|
307
|
+
self._run(["mutagen", "sync", "resume", session.identifier])
|
|
308
|
+
elif (
|
|
309
|
+
session.conflicts
|
|
310
|
+
or session.excluded_conflicts
|
|
311
|
+
or session.status.startswith("halted-on-root-")
|
|
312
|
+
or session.alpha.problem_count
|
|
313
|
+
or session.beta.problem_count
|
|
314
|
+
):
|
|
315
|
+
self._raise_health(session)
|
|
316
|
+
self._run(["mutagen", "sync", "flush", session.identifier], timeout=120)
|
|
317
|
+
|
|
318
|
+
def _raise_health(self, session: SyncSession) -> None:
|
|
319
|
+
report = health(session)
|
|
320
|
+
if not report.ok:
|
|
321
|
+
detail = session.last_error or f"{report.conflicts} conflicts, {report.problems} problems"
|
|
322
|
+
raise PreconditionError(f"Source sync is {session.status}: {detail}", report.fix)
|
|
323
|
+
|
|
324
|
+
def _cleanup(self, action: Literal["pause", "terminate"], work: Worktree, host: HostState | None) -> None:
|
|
325
|
+
try:
|
|
326
|
+
session = self.status(work, host, install=False)
|
|
327
|
+
if session is not None and (action == "terminate" or not session.paused):
|
|
328
|
+
self._run(["mutagen", "sync", action, session.identifier], install=False)
|
|
329
|
+
except MutagenMissing:
|
|
330
|
+
console.warn(f"Mutagen is not installed; continuing cleanup without sync {action}.")
|
|
331
|
+
|
|
332
|
+
@measured("Sync pause")
|
|
333
|
+
def pause(self, work: Worktree, host: HostState | None = None) -> None:
|
|
334
|
+
self._cleanup("pause", work, host)
|
|
335
|
+
|
|
336
|
+
@measured("Sync cleanup")
|
|
337
|
+
def terminate(self, work: Worktree, host: HostState | None = None) -> None:
|
|
338
|
+
self._cleanup("terminate", work, host)
|
|
339
|
+
|
|
340
|
+
def terminate_all(self) -> None:
|
|
341
|
+
try:
|
|
342
|
+
result = self._run(["mutagen", "sync", "list", "--label-selector", "offbox-env", "--template", "{{ json . }}"], install=False)
|
|
343
|
+
except MutagenMissing:
|
|
344
|
+
console.warn("Mutagen is not installed; continuing local cleanup without terminating sync sessions.")
|
|
345
|
+
return
|
|
346
|
+
try:
|
|
347
|
+
sessions = TypeAdapter(list[SyncSession]).validate_json(result.stdout)
|
|
348
|
+
except ValidationError as exc:
|
|
349
|
+
raise PreconditionError("Invalid Mutagen session response", "Check mutagen sync list and retry uninstall.") from exc
|
|
350
|
+
owned: list[SyncSession] = []
|
|
351
|
+
for session in sessions:
|
|
352
|
+
name = session.labels.get("offbox-env")
|
|
353
|
+
if name is None or session.name != f"offbox-{name}":
|
|
354
|
+
continue
|
|
355
|
+
validate_env_name(name)
|
|
356
|
+
if session.beta.host is not None and session.beta.host.startswith("offbox-"):
|
|
357
|
+
validate_name(session.beta.host[len("offbox-") :])
|
|
358
|
+
if (
|
|
359
|
+
session.alpha.protocol != "local"
|
|
360
|
+
or not Path(session.alpha.path).is_absolute()
|
|
361
|
+
or session.beta.protocol != "ssh"
|
|
362
|
+
or not (session.beta.host or "").startswith("offbox-")
|
|
363
|
+
or session.beta.path != f"/srv/offbox/envs/{name}/work"
|
|
364
|
+
or session.alpha.user
|
|
365
|
+
or session.alpha.host
|
|
366
|
+
or session.alpha.port
|
|
367
|
+
or session.alpha.environment
|
|
368
|
+
or session.alpha.parameters
|
|
369
|
+
or session.beta.user
|
|
370
|
+
or session.beta.port
|
|
371
|
+
or session.beta.environment
|
|
372
|
+
or session.beta.parameters
|
|
373
|
+
):
|
|
374
|
+
raise PreconditionError(
|
|
375
|
+
f"Cannot verify ownership of sync session {session.identifier}",
|
|
376
|
+
f"Inspect mutagen sync list; terminate {session.identifier} explicitly only if it belongs to offbox.",
|
|
377
|
+
)
|
|
378
|
+
owned.append(session)
|
|
379
|
+
for session in owned:
|
|
380
|
+
self._run(["mutagen", "sync", "terminate", session.identifier])
|
offbox/timings.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
from collections.abc import Callable, Generator
|
|
2
|
+
from contextlib import contextmanager
|
|
3
|
+
from contextvars import ContextVar
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
from functools import wraps
|
|
6
|
+
from time import perf_counter
|
|
7
|
+
|
|
8
|
+
from offbox import console
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass
|
|
12
|
+
class _Timing:
|
|
13
|
+
seconds: float = 0
|
|
14
|
+
calls: int = 0
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass
|
|
18
|
+
class _Session:
|
|
19
|
+
started: float
|
|
20
|
+
stages: dict[str, _Timing] = field(default_factory=dict[str, _Timing])
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
_active: ContextVar[_Session | None] = ContextVar("offbox_timings", default=None)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@contextmanager
|
|
27
|
+
def collect(enabled: bool) -> Generator[None]:
|
|
28
|
+
if not enabled:
|
|
29
|
+
yield
|
|
30
|
+
return
|
|
31
|
+
session = _Session(started=perf_counter())
|
|
32
|
+
token = _active.set(session)
|
|
33
|
+
try:
|
|
34
|
+
yield
|
|
35
|
+
finally:
|
|
36
|
+
elapsed = perf_counter() - session.started
|
|
37
|
+
_active.reset(token)
|
|
38
|
+
console.info("Timings (CLI startup excluded):")
|
|
39
|
+
for label, value in session.stages.items():
|
|
40
|
+
calls = f" ({value.calls} calls)" if value.calls > 1 else ""
|
|
41
|
+
console.info(f" {label}: {value.seconds:.3f}s{calls}")
|
|
42
|
+
console.info(f" Total: {elapsed:.3f}s")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@contextmanager
|
|
46
|
+
def stage(label: str) -> Generator[None]:
|
|
47
|
+
session = _active.get()
|
|
48
|
+
if session is None:
|
|
49
|
+
yield
|
|
50
|
+
return
|
|
51
|
+
started = perf_counter()
|
|
52
|
+
try:
|
|
53
|
+
yield
|
|
54
|
+
finally:
|
|
55
|
+
value = session.stages.setdefault(label, _Timing())
|
|
56
|
+
value.seconds += perf_counter() - started
|
|
57
|
+
value.calls += 1
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def measured[**P, R](label: str) -> Callable[[Callable[P, R]], Callable[P, R]]:
|
|
61
|
+
def decorate(function: Callable[P, R]) -> Callable[P, R]:
|
|
62
|
+
@wraps(function)
|
|
63
|
+
def call(*args: P.args, **kwargs: P.kwargs) -> R:
|
|
64
|
+
with stage(label):
|
|
65
|
+
return function(*args, **kwargs)
|
|
66
|
+
|
|
67
|
+
return call
|
|
68
|
+
|
|
69
|
+
return decorate
|
offbox/tools.py
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
import hashlib
|
|
2
|
+
import io
|
|
3
|
+
import os
|
|
4
|
+
import platform
|
|
5
|
+
import re
|
|
6
|
+
import shutil
|
|
7
|
+
import tarfile
|
|
8
|
+
import tempfile
|
|
9
|
+
import zipfile
|
|
10
|
+
from collections.abc import Callable, Mapping
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from offbox import console
|
|
14
|
+
from offbox.config import Model
|
|
15
|
+
from offbox.errors import PreconditionError
|
|
16
|
+
from offbox.runner import Runner
|
|
17
|
+
from offbox.timings import measured
|
|
18
|
+
|
|
19
|
+
TOFU_VERSION = "1.10.6"
|
|
20
|
+
MUTAGEN_MIN = (0, 18)
|
|
21
|
+
MUTAGEN_VERSION = "0.18.1"
|
|
22
|
+
MUTAGEN_FIX = "Run offbox setup to download Mutagen, or install Mutagen 0.18.x."
|
|
23
|
+
TOFU_SHA256 = {
|
|
24
|
+
("darwin", "amd64"): "85115c8ec23b48b425259060347ff932355dad48056fa83ade05f7d890dc43ed",
|
|
25
|
+
("darwin", "arm64"): "6da8ee57f20113c302baeddc1f0c6340589fa1f7af04d41282e406cfa53c0b59",
|
|
26
|
+
("linux", "amd64"): "15b7bed76420b50da3e121769c43341df8cd57d751ca14e6dbe9c850124c6dac",
|
|
27
|
+
("linux", "arm64"): "a32f653d686a8cad9b9be82101eb5b5e834fbfa8d095842fa1820c5d27fad967",
|
|
28
|
+
}
|
|
29
|
+
MUTAGEN_SHA256 = {
|
|
30
|
+
("darwin", "amd64"): "7d06f7d8fcfe90bc7e55cc834a2f2f20c2e0af9ea9bc35911fc4341ad56a9bbf",
|
|
31
|
+
("darwin", "arm64"): "6f810416d9e5fc4fd5e18431146f8b3c5a2056ba5a24f76c1e66da86eb3257e2",
|
|
32
|
+
("linux", "amd64"): "7735286c778cc438418209f24d03a64f3a0151c8065ef0fe079cfaf093af6f8f",
|
|
33
|
+
("linux", "arm64"): "bcba735aebf8cbc11da9b3742118a665599ac697fa06bc5751cac8dcd540db8a",
|
|
34
|
+
}
|
|
35
|
+
_MUTAGEN_MEMBERS = ("mutagen", "mutagen-agents.tar.gz")
|
|
36
|
+
_VERSION = re.compile(r"(?:OpenSSH_|(?:OpenTofu|Terraform)\s+v?|(?:git|mutagen)\s+version\s+v?)(\d+)\.(\d+)(?:\.(\d+))?", re.IGNORECASE)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class Tool(Model):
|
|
40
|
+
name: str
|
|
41
|
+
path: Path
|
|
42
|
+
version: str
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class MutagenMissing(PreconditionError):
|
|
46
|
+
pass
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def download(url: str, timeout: float) -> bytes:
|
|
50
|
+
import urllib.request
|
|
51
|
+
|
|
52
|
+
try:
|
|
53
|
+
with urllib.request.urlopen(url, timeout=timeout) as response:
|
|
54
|
+
content: object = response.read()
|
|
55
|
+
if not isinstance(content, bytes):
|
|
56
|
+
raise PreconditionError(f"Invalid download response from {url}", "Retry, or install the tool on PATH.")
|
|
57
|
+
return content
|
|
58
|
+
except OSError as exc:
|
|
59
|
+
raise PreconditionError(f"Cannot download {url}", "Check your network and retry, or install the tool on PATH.") from exc
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _atomic_write(path: Path, content: bytes, mode: int) -> None:
|
|
63
|
+
temporary: Path | None = None
|
|
64
|
+
try:
|
|
65
|
+
with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as output:
|
|
66
|
+
temporary = Path(output.name)
|
|
67
|
+
output.write(content)
|
|
68
|
+
output.flush()
|
|
69
|
+
os.fsync(output.fileno())
|
|
70
|
+
temporary.chmod(mode)
|
|
71
|
+
os.replace(temporary, path)
|
|
72
|
+
finally:
|
|
73
|
+
if temporary is not None:
|
|
74
|
+
temporary.unlink(missing_ok=True)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _platform(system: str | None, machine: str | None) -> tuple[str, str]:
|
|
78
|
+
arch_name = (machine or platform.machine()).lower()
|
|
79
|
+
return (system or platform.system()).lower(), {"x86_64": "amd64", "aarch64": "arm64"}.get(arch_name, arch_name)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def install_tofu(
|
|
83
|
+
cache: Path,
|
|
84
|
+
*,
|
|
85
|
+
system: str | None = None,
|
|
86
|
+
machine: str | None = None,
|
|
87
|
+
fetch: Callable[[str, float], bytes] = download,
|
|
88
|
+
) -> Path:
|
|
89
|
+
os_name, arch_name = _platform(system, machine)
|
|
90
|
+
checksum = TOFU_SHA256.get((os_name, arch_name))
|
|
91
|
+
if checksum is None:
|
|
92
|
+
raise PreconditionError(f"Unsupported OpenTofu platform: {os_name}/{arch_name}", "Install OpenTofu >=1.8 on PATH.")
|
|
93
|
+
directory = cache / "tools" / "tofu" / TOFU_VERSION
|
|
94
|
+
archive = directory / f"tofu_{TOFU_VERSION}_{os_name}_{arch_name}.zip"
|
|
95
|
+
try:
|
|
96
|
+
directory.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
97
|
+
content = (
|
|
98
|
+
archive.read_bytes()
|
|
99
|
+
if archive.exists()
|
|
100
|
+
else fetch(
|
|
101
|
+
f"https://github.com/opentofu/opentofu/releases/download/v{TOFU_VERSION}/{archive.name}",
|
|
102
|
+
60.0,
|
|
103
|
+
)
|
|
104
|
+
)
|
|
105
|
+
if hashlib.sha256(content).hexdigest() != checksum:
|
|
106
|
+
raise PreconditionError("OpenTofu checksum mismatch", f"Remove {archive} and retry, or install OpenTofu on PATH.")
|
|
107
|
+
with zipfile.ZipFile(io.BytesIO(content)) as package:
|
|
108
|
+
binary = package.read("tofu")
|
|
109
|
+
executable = directory / "tofu"
|
|
110
|
+
_atomic_write(archive, content, 0o600)
|
|
111
|
+
_atomic_write(executable, binary, 0o700)
|
|
112
|
+
return executable
|
|
113
|
+
except (OSError, zipfile.BadZipFile, KeyError) as exc:
|
|
114
|
+
raise PreconditionError("Cannot install verified OpenTofu", f"Clear {directory} and retry, or install OpenTofu on PATH.") from exc
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def mutagen_directory(cache: Path) -> Path:
|
|
118
|
+
return cache / "tools" / "mutagen" / MUTAGEN_VERSION
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def cached_mutagen(cache: Path) -> Path | None:
|
|
122
|
+
directory = mutagen_directory(cache)
|
|
123
|
+
executable = directory / "mutagen"
|
|
124
|
+
return executable if executable.is_file() and (directory / "mutagen-agents.tar.gz").is_file() else None
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def download_prompt(cache: Path) -> str:
|
|
128
|
+
return f"Mutagen {MUTAGEN_VERSION} not found. Download it (verified) into {mutagen_directory(cache)}?"
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def install_mutagen(
|
|
132
|
+
cache: Path,
|
|
133
|
+
*,
|
|
134
|
+
system: str | None = None,
|
|
135
|
+
machine: str | None = None,
|
|
136
|
+
fetch: Callable[[str, float], bytes] = download,
|
|
137
|
+
) -> Path:
|
|
138
|
+
"""Download, verify and unpack the pinned Mutagen release; its agent bundle must sit beside the binary."""
|
|
139
|
+
os_name, arch_name = _platform(system, machine)
|
|
140
|
+
checksum = MUTAGEN_SHA256.get((os_name, arch_name))
|
|
141
|
+
if checksum is None:
|
|
142
|
+
raise PreconditionError(f"Unsupported Mutagen platform: {os_name}/{arch_name}", "Install Mutagen 0.18.x on PATH.")
|
|
143
|
+
directory = mutagen_directory(cache)
|
|
144
|
+
name = f"mutagen_{os_name}_{arch_name}_v{MUTAGEN_VERSION}.tar.gz"
|
|
145
|
+
console.step(f"Downloading Mutagen {MUTAGEN_VERSION} (about 100 MB)")
|
|
146
|
+
content = fetch(f"https://github.com/mutagen-io/mutagen/releases/download/v{MUTAGEN_VERSION}/{name}", 300.0)
|
|
147
|
+
if hashlib.sha256(content).hexdigest() != checksum:
|
|
148
|
+
raise PreconditionError("Mutagen checksum mismatch", "Retry offbox setup, or install Mutagen 0.18.x on PATH.")
|
|
149
|
+
executable = directory / "mutagen"
|
|
150
|
+
try:
|
|
151
|
+
members: dict[str, bytes] = {}
|
|
152
|
+
with tarfile.open(fileobj=io.BytesIO(content), mode="r:gz") as package:
|
|
153
|
+
for member_name in _MUTAGEN_MEMBERS:
|
|
154
|
+
member = package.getmember(member_name)
|
|
155
|
+
extracted = package.extractfile(member) if member.isfile() else None
|
|
156
|
+
if extracted is None:
|
|
157
|
+
raise PreconditionError(f"Invalid Mutagen archive member: {member_name}", "Install Mutagen 0.18.x on PATH.")
|
|
158
|
+
members[member_name] = extracted.read()
|
|
159
|
+
directory.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
160
|
+
_atomic_write(directory / "mutagen-agents.tar.gz", members["mutagen-agents.tar.gz"], 0o644)
|
|
161
|
+
_atomic_write(executable, members["mutagen"], 0o755)
|
|
162
|
+
except (OSError, tarfile.TarError, KeyError) as exc:
|
|
163
|
+
raise PreconditionError(
|
|
164
|
+
"Cannot install verified Mutagen", f"Clear {directory} and retry, or install Mutagen 0.18.x on PATH."
|
|
165
|
+
) from exc
|
|
166
|
+
console.success(f"Installed Mutagen {MUTAGEN_VERSION} at {executable}")
|
|
167
|
+
return executable
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def install_hint(name: str, system: str | None = None, release: Mapping[str, str] | None = None) -> str:
|
|
171
|
+
os_name = (system or platform.system()).lower()
|
|
172
|
+
if os_name == "darwin":
|
|
173
|
+
return "Run xcode-select --install or brew install git." if name == "git" else "Run brew install openssh."
|
|
174
|
+
if release is None:
|
|
175
|
+
try:
|
|
176
|
+
release = platform.freedesktop_os_release()
|
|
177
|
+
except OSError:
|
|
178
|
+
release = {}
|
|
179
|
+
family = {release.get("ID", ""), *release.get("ID_LIKE", "").split()}
|
|
180
|
+
if family & {"debian", "ubuntu"}:
|
|
181
|
+
return "Run sudo apt install git openssh-client."
|
|
182
|
+
if family & {"fedora", "rhel", "centos"}:
|
|
183
|
+
return "Run sudo dnf install git openssh-clients."
|
|
184
|
+
return "Install git >=2.30 and OpenSSH >=8.0 with your package manager."
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
class ToolDiscovery:
|
|
188
|
+
def __init__(self, runner: Runner, cache: Path, lookup: Callable[[str], str | None] = shutil.which) -> None:
|
|
189
|
+
self.runner = runner
|
|
190
|
+
self.cache = cache
|
|
191
|
+
self.lookup = lookup
|
|
192
|
+
|
|
193
|
+
@measured("Local tool checks")
|
|
194
|
+
def check(self, name: str) -> Tool:
|
|
195
|
+
executable = self.lookup(name)
|
|
196
|
+
if name in ("ssh", "git"):
|
|
197
|
+
fix = f"{'OpenSSH >=8.0' if name == 'ssh' else 'git >=2.30'} is required. {install_hint(name)}"
|
|
198
|
+
elif name == "mutagen":
|
|
199
|
+
fix = MUTAGEN_FIX
|
|
200
|
+
elif name == "tofu":
|
|
201
|
+
fix = "Install OpenTofu >=1.8 or Terraform >=1.6."
|
|
202
|
+
else:
|
|
203
|
+
fix = f"Install {name}."
|
|
204
|
+
if name == "mutagen" and executable is None:
|
|
205
|
+
cached = cached_mutagen(self.cache)
|
|
206
|
+
if cached is None:
|
|
207
|
+
raise MutagenMissing("Missing tool: mutagen", fix)
|
|
208
|
+
executable = str(cached)
|
|
209
|
+
if name == "tofu" and executable is None:
|
|
210
|
+
executable = self.lookup("terraform")
|
|
211
|
+
if executable is not None:
|
|
212
|
+
name = "terraform"
|
|
213
|
+
else:
|
|
214
|
+
executable = str(install_tofu(self.cache))
|
|
215
|
+
if executable is None:
|
|
216
|
+
raise PreconditionError(f"Missing tool: {name}", fix)
|
|
217
|
+
result = self.runner.run([executable, "-V" if name == "ssh" else "--version"], check=False, timeout=10)
|
|
218
|
+
match = _VERSION.search(result.stdout + "\n" + result.stderr)
|
|
219
|
+
if result.returncode != 0 or match is None:
|
|
220
|
+
raise PreconditionError(f"Cannot determine {name} version", fix)
|
|
221
|
+
found = (int(match[1]), int(match[2]))
|
|
222
|
+
minimum = {"ssh": (8, 0), "git": (2, 30), "mutagen": MUTAGEN_MIN, "tofu": (1, 8), "terraform": (1, 6)}[name]
|
|
223
|
+
if found < minimum or (name == "mutagen" and found != MUTAGEN_MIN):
|
|
224
|
+
if name == "mutagen":
|
|
225
|
+
fix = f"Install Mutagen 0.18.x, or run mutagen daemon stop, remove {executable} from PATH and run offbox setup."
|
|
226
|
+
raise PreconditionError(f"Unsupported {name} version: {match.group(0)}", fix)
|
|
227
|
+
return Tool(name=name, path=Path(executable), version=".".join(part for part in match.groups() if part is not None))
|