nullforge 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nullforge/__init__.py +1 -0
- nullforge/foundry/README.md +0 -0
- nullforge/foundry/__init__.py +7 -0
- nullforge/foundry/full_cast.py +22 -0
- nullforge/inventories/README.md +0 -0
- nullforge/inventories/example.py +87 -0
- nullforge/models/__init__.py +1 -0
- nullforge/models/containers.py +60 -0
- nullforge/models/dns.py +222 -0
- nullforge/models/monitoring/__init__.py +29 -0
- nullforge/models/monitoring/base.py +19 -0
- nullforge/models/monitoring/nezha.py +13 -0
- nullforge/models/netsec.py +43 -0
- nullforge/models/profiles.py +33 -0
- nullforge/models/system.py +15 -0
- nullforge/models/users.py +9 -0
- nullforge/models/warp.py +65 -0
- nullforge/models/zerotrust.py +9 -0
- nullforge/molds/__init__.py +38 -0
- nullforge/molds/base_mold.py +59 -0
- nullforge/molds/containers.py +26 -0
- nullforge/molds/defaults.py +10 -0
- nullforge/molds/dns.py +70 -0
- nullforge/molds/features.py +75 -0
- nullforge/molds/haproxy.py +23 -0
- nullforge/molds/monitoring/__init__.py +45 -0
- nullforge/molds/monitoring/nezha.py +118 -0
- nullforge/molds/netsec.py +237 -0
- nullforge/molds/profiles.py +30 -0
- nullforge/molds/system.py +140 -0
- nullforge/molds/telemt.py +140 -0
- nullforge/molds/tor.py +28 -0
- nullforge/molds/user.py +77 -0
- nullforge/molds/utils.py +115 -0
- nullforge/molds/warp.py +54 -0
- nullforge/molds/xray.py +16 -0
- nullforge/molds/zerotrust.py +59 -0
- nullforge/runes/__init__.py +46 -0
- nullforge/runes/base.py +327 -0
- nullforge/runes/containers.py +212 -0
- nullforge/runes/dns.py +256 -0
- nullforge/runes/haproxy.py +135 -0
- nullforge/runes/monitoring.py +21 -0
- nullforge/runes/netsec.py +547 -0
- nullforge/runes/prepare.py +30 -0
- nullforge/runes/profiles.py +583 -0
- nullforge/runes/telemt.py +252 -0
- nullforge/runes/tor.py +57 -0
- nullforge/runes/users.py +159 -0
- nullforge/runes/warp.py +273 -0
- nullforge/runes/xray.py +81 -0
- nullforge/runes/zerotrust.py +91 -0
- nullforge/smithy/__init__.py +1 -0
- nullforge/smithy/admin.py +40 -0
- nullforge/smithy/arch.py +36 -0
- nullforge/smithy/blocky.py +37 -0
- nullforge/smithy/cloudflare.py +32 -0
- nullforge/smithy/github.py +256 -0
- nullforge/smithy/http.py +85 -0
- nullforge/smithy/install.py +149 -0
- nullforge/smithy/monitoring/__init__.py +1 -0
- nullforge/smithy/monitoring/nezha/__init__.py +0 -0
- nullforge/smithy/monitoring/nezha/agent.py +54 -0
- nullforge/smithy/monitoring/nezha/dashboard.py +137 -0
- nullforge/smithy/monitoring/nezha/deploy.py +81 -0
- nullforge/smithy/network.py +95 -0
- nullforge/smithy/packages.py +176 -0
- nullforge/smithy/service.py +37 -0
- nullforge/smithy/sni.py +358 -0
- nullforge/smithy/swap.py +132 -0
- nullforge/smithy/system.py +86 -0
- nullforge/smithy/versions.py +282 -0
- nullforge/templates/__init__.py +32 -0
- nullforge/templates/cloudflared/tunnel.yml.j2 +6 -0
- nullforge/templates/dns/blocky.yaml.j2 +11 -0
- nullforge/templates/dns/dns.yaml.j2 +6 -0
- nullforge/templates/dns/resolv.conf.j2 +1 -0
- nullforge/templates/dns/resolved.conf.j2 +13 -0
- nullforge/templates/etc/default/zramswap.j2 +13 -0
- nullforge/templates/nvim/nvim_patch.lua.j2 +5 -0
- nullforge/templates/profiles/direnv.toml +2 -0
- nullforge/templates/profiles/starship.toml +186 -0
- nullforge/templates/profiles/tmux.conf +32 -0
- nullforge/templates/profiles/zshrc.j2 +46 -0
- nullforge/templates/scripts/telemt-synfix.sh +89 -0
- nullforge/templates/scripts/teleproxy-warp.sh +127 -0
- nullforge/templates/scripts/warp-v6-policy.sh +118 -0
- nullforge/templates/scripts/zt-tunnel-warp.sh +28 -0
- nullforge/templates/systemd/blocky.service.j2 +19 -0
- nullforge/templates/systemd/cloudflare-tunnel.service.j2 +27 -0
- nullforge/templates/systemd/cloudflare-warp.service.j2 +21 -0
- nullforge/templates/systemd/dns-internal.service.j2 +15 -0
- nullforge/templates/systemd/telemt.service.j2 +36 -0
- nullforge/templates/telemt/telemt.toml.j2 +49 -0
- nullforge/templates/tor/torrc.j2 +5 -0
- nullforge-0.1.0.dist-info/METADATA +39 -0
- nullforge-0.1.0.dist-info/RECORD +99 -0
- nullforge-0.1.0.dist-info/WHEEL +4 -0
- nullforge-0.1.0.dist-info/licenses/LICENSE +7 -0
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
"""GitHub release metadata helpers for NullForge.
|
|
2
|
+
|
|
3
|
+
These run on the *control node* at plan time (like `fetch_github_keys`), not on the
|
|
4
|
+
target. They resolve checksums and detect the right asset for a release so that the
|
|
5
|
+
target-side installers in `smithy/install.py` can verify downloads and, optionally,
|
|
6
|
+
pick assets without a hand-written URL per tool.
|
|
7
|
+
|
|
8
|
+
Everything degrades gracefully: if the API is unreachable or rate-limited, resolvers
|
|
9
|
+
return ``None`` and callers fall back to their existing (unverified) behaviour. Set
|
|
10
|
+
``GITHUB_TOKEN`` (or ``GH_TOKEN``) in the control-node environment to raise the API
|
|
11
|
+
rate limit from 60 to 5000 requests/hour.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import logging
|
|
16
|
+
import os
|
|
17
|
+
import re
|
|
18
|
+
from contextlib import suppress
|
|
19
|
+
|
|
20
|
+
from nullforge.smithy.arch import arch_id
|
|
21
|
+
from nullforge.smithy.http import fetch_text
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
LOG = logging.getLogger("pyinfra")
|
|
25
|
+
|
|
26
|
+
GITHUB_API = "https://api.github.com"
|
|
27
|
+
"""REST API base; queried from the control node only."""
|
|
28
|
+
|
|
29
|
+
_DOWNLOAD_URL_RE = re.compile(
|
|
30
|
+
r"^https://github\.com/(?P<repo>[^/]+/[^/]+)/releases/"
|
|
31
|
+
r"(?:download/(?P<tag>[^/]+)|(?P<latest>latest)/download)/(?P<asset>[^/]+)$"
|
|
32
|
+
)
|
|
33
|
+
"""Matches the release-asset URLs produced by `Versions._release_url`."""
|
|
34
|
+
|
|
35
|
+
_CHECKSUM_HINTS = ("checksum", "sha256", "sha256sums", "sha256sum")
|
|
36
|
+
"""Substrings that flag an assets-wide checksums manifest."""
|
|
37
|
+
|
|
38
|
+
_SHA256_RE = re.compile(r"\b[0-9a-f]{64}\b")
|
|
39
|
+
|
|
40
|
+
_RELEASE_CACHE: dict[tuple[str, str], dict | None] = {}
|
|
41
|
+
"""Per-run cache so a single deploy hits the API at most once per (repo, tag)."""
|
|
42
|
+
|
|
43
|
+
# Arch/OS token synonyms, mirroring how release assets are commonly named.
|
|
44
|
+
_ARCH_TOKENS: dict[str, tuple[str, ...]] = {
|
|
45
|
+
"x86_64": ("x86_64", "amd64", "x64", "64bit"),
|
|
46
|
+
"arm64": ("arm64", "aarch64"),
|
|
47
|
+
}
|
|
48
|
+
_ALL_ARCH_TOKENS = {token for tokens in _ARCH_TOKENS.values() for token in tokens}
|
|
49
|
+
_OS_TOKENS: dict[str, tuple[str, ...]] = {
|
|
50
|
+
"linux": ("linux",),
|
|
51
|
+
"darwin": ("darwin", "macos", "apple"),
|
|
52
|
+
"windows": ("windows", "win"),
|
|
53
|
+
}
|
|
54
|
+
_ALL_OS_TOKENS = {token for tokens in _OS_TOKENS.values() for token in tokens}
|
|
55
|
+
|
|
56
|
+
# Assets that are never the binary we want.
|
|
57
|
+
_NON_BINARY_SUFFIXES = (".sha256", ".sha256sum", ".sig", ".asc", ".pem", ".pub", ".txt", ".sbom", ".json")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _token() -> str | None:
|
|
61
|
+
return os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _headers() -> dict[str, str]:
|
|
65
|
+
headers = {
|
|
66
|
+
"Accept": "application/vnd.github+json",
|
|
67
|
+
"X-GitHub-Api-Version": "2022-11-28",
|
|
68
|
+
}
|
|
69
|
+
token = _token()
|
|
70
|
+
if token:
|
|
71
|
+
headers["Authorization"] = f"Bearer {token}"
|
|
72
|
+
return headers
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def fetch_release(repo: str, tag: str) -> dict | None:
|
|
76
|
+
"""Fetch release metadata for ``owner/repo`` at ``tag`` (``"latest"`` supported).
|
|
77
|
+
|
|
78
|
+
Returns the parsed JSON object, or ``None`` on any failure (unreachable API,
|
|
79
|
+
rate limit, unknown tag). Results are cached for the duration of the run.
|
|
80
|
+
"""
|
|
81
|
+
|
|
82
|
+
cache_key = (repo, tag)
|
|
83
|
+
if cache_key in _RELEASE_CACHE:
|
|
84
|
+
return _RELEASE_CACHE[cache_key]
|
|
85
|
+
|
|
86
|
+
path = "releases/latest" if tag == "latest" else f"releases/tags/{tag}"
|
|
87
|
+
url = f"{GITHUB_API}/repos/{repo}/{path}"
|
|
88
|
+
release: dict | None = None
|
|
89
|
+
try:
|
|
90
|
+
release = json.loads(fetch_text(url, headers=_headers()))
|
|
91
|
+
except Exception as exc: # noqa: BLE001 - network/JSON errors all degrade to None
|
|
92
|
+
LOG.warning(f"GitHub release lookup failed for {repo}@{tag}: {exc}")
|
|
93
|
+
|
|
94
|
+
_RELEASE_CACHE[cache_key] = release
|
|
95
|
+
return release
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _find_asset(release: dict, asset_name: str) -> dict | None:
|
|
99
|
+
for asset in release.get("assets", []):
|
|
100
|
+
if asset.get("name") == asset_name:
|
|
101
|
+
return asset
|
|
102
|
+
return None
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _digest_sha256(asset: dict) -> str | None:
|
|
106
|
+
"""Read the API-provided ``digest`` field (``"sha256:<hex>"``) if present."""
|
|
107
|
+
|
|
108
|
+
digest = asset.get("digest") or ""
|
|
109
|
+
prefix = "sha256:"
|
|
110
|
+
if digest.startswith(prefix):
|
|
111
|
+
candidate = digest[len(prefix) :].strip().lower()
|
|
112
|
+
if _SHA256_RE.fullmatch(candidate):
|
|
113
|
+
return candidate
|
|
114
|
+
return None
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _sha256_from_manifest(release: dict, asset_name: str) -> str | None:
|
|
118
|
+
"""Locate a checksums asset in the release and parse the line for ``asset_name``."""
|
|
119
|
+
|
|
120
|
+
for asset in release.get("assets", []):
|
|
121
|
+
name = (asset.get("name") or "").lower()
|
|
122
|
+
is_per_asset = name == f"{asset_name.lower()}.sha256"
|
|
123
|
+
is_manifest = any(hint in name for hint in _CHECKSUM_HINTS)
|
|
124
|
+
if not (is_per_asset or is_manifest):
|
|
125
|
+
continue
|
|
126
|
+
|
|
127
|
+
url = asset.get("browser_download_url")
|
|
128
|
+
if not url:
|
|
129
|
+
continue
|
|
130
|
+
with suppress(Exception):
|
|
131
|
+
body = fetch_text(url)
|
|
132
|
+
found = _parse_checksums(body, asset_name)
|
|
133
|
+
if found:
|
|
134
|
+
return found
|
|
135
|
+
return None
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _parse_checksums(body: str, asset_name: str) -> str | None:
|
|
139
|
+
"""Extract the sha256 for ``asset_name`` from a checksums file body.
|
|
140
|
+
|
|
141
|
+
Supports both multi-asset manifests (``<hex> <name>`` / ``<hex> *<name>``) and
|
|
142
|
+
single-asset files that contain only the hash (optionally with a filename).
|
|
143
|
+
"""
|
|
144
|
+
|
|
145
|
+
target = asset_name.rsplit("/", 1)[-1]
|
|
146
|
+
lone: str | None = None
|
|
147
|
+
for raw in body.splitlines():
|
|
148
|
+
line = raw.strip()
|
|
149
|
+
if not line:
|
|
150
|
+
continue
|
|
151
|
+
match = _SHA256_RE.search(line)
|
|
152
|
+
if not match:
|
|
153
|
+
continue
|
|
154
|
+
digest = match.group(0).lower()
|
|
155
|
+
rest = (line[: match.start()] + line[match.end() :]).strip().lstrip("*").strip()
|
|
156
|
+
if not rest:
|
|
157
|
+
lone = digest # bare "<hex>" — remember as a single-asset fallback
|
|
158
|
+
continue
|
|
159
|
+
if rest.rsplit("/", 1)[-1].lstrip("*") == target:
|
|
160
|
+
return digest
|
|
161
|
+
return lone
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def asset_sha256(release: dict, asset_name: str) -> str | None:
|
|
165
|
+
"""Best-effort sha256 for ``asset_name``: API digest first, then a checksums file."""
|
|
166
|
+
|
|
167
|
+
asset = _find_asset(release, asset_name)
|
|
168
|
+
if asset is not None:
|
|
169
|
+
digest = _digest_sha256(asset)
|
|
170
|
+
if digest:
|
|
171
|
+
return digest
|
|
172
|
+
return _sha256_from_manifest(release, asset_name)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def parse_download_url(url: str) -> tuple[str, str, str] | None:
|
|
176
|
+
"""Split a GitHub release-asset URL into ``(repo, tag, asset_name)``.
|
|
177
|
+
|
|
178
|
+
Returns ``None`` for anything that is not a matching GitHub release URL (including
|
|
179
|
+
non-string input), so callers degrade to "no checksum" rather than raising.
|
|
180
|
+
"""
|
|
181
|
+
|
|
182
|
+
if not isinstance(url, str):
|
|
183
|
+
return None
|
|
184
|
+
match = _DOWNLOAD_URL_RE.match(url)
|
|
185
|
+
if not match:
|
|
186
|
+
return None
|
|
187
|
+
tag = "latest" if match.group("latest") else match.group("tag")
|
|
188
|
+
return match.group("repo"), tag, match.group("asset")
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def sha256_for_download_url(url: str) -> str | None:
|
|
192
|
+
"""Resolve the sha256 for a release-asset download URL, or ``None`` if unavailable.
|
|
193
|
+
|
|
194
|
+
This is the ergonomic entry point: any URL from `Versions` (e.g. ``telemt_tar()``)
|
|
195
|
+
can be verified without knowing its repo/tag/asset separately.
|
|
196
|
+
"""
|
|
197
|
+
|
|
198
|
+
parsed = parse_download_url(url)
|
|
199
|
+
if parsed is None:
|
|
200
|
+
return None
|
|
201
|
+
repo, tag, asset_name = parsed
|
|
202
|
+
release = fetch_release(repo, tag)
|
|
203
|
+
if release is None:
|
|
204
|
+
return None
|
|
205
|
+
return asset_sha256(release, asset_name)
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def select_asset(
|
|
209
|
+
release: dict,
|
|
210
|
+
*,
|
|
211
|
+
arch: str | None = None,
|
|
212
|
+
os_name: str = "linux",
|
|
213
|
+
include: tuple[str, ...] = (),
|
|
214
|
+
exclude: tuple[str, ...] = (),
|
|
215
|
+
) -> dict | None:
|
|
216
|
+
"""Pick the release asset best matching the current arch/OS (bin-style matcher).
|
|
217
|
+
|
|
218
|
+
Scores each asset by OS/arch token match and ``include`` hints, rejecting assets
|
|
219
|
+
that clearly target a different OS or arch. Returns the highest-scoring asset dict,
|
|
220
|
+
or ``None`` when nothing matches. Useful for tools whose asset names are awkward to
|
|
221
|
+
template, or for resolving a concrete asset from a ``latest`` release.
|
|
222
|
+
"""
|
|
223
|
+
|
|
224
|
+
want_arch = arch or arch_id()
|
|
225
|
+
arch_tokens = _ARCH_TOKENS.get(want_arch, (want_arch,))
|
|
226
|
+
os_tokens = _OS_TOKENS.get(os_name, (os_name,))
|
|
227
|
+
|
|
228
|
+
best: dict | None = None
|
|
229
|
+
best_score = 0
|
|
230
|
+
for asset in release.get("assets", []):
|
|
231
|
+
name = (asset.get("name") or "").lower()
|
|
232
|
+
if not name or name.endswith(_NON_BINARY_SUFFIXES):
|
|
233
|
+
continue
|
|
234
|
+
if any(bad in name for bad in exclude):
|
|
235
|
+
continue
|
|
236
|
+
|
|
237
|
+
# Reject assets that name a different OS/arch than the one we want.
|
|
238
|
+
if any(tok in name for tok in _ALL_OS_TOKENS) and not any(tok in name for tok in os_tokens):
|
|
239
|
+
continue
|
|
240
|
+
if any(tok in name for tok in _ALL_ARCH_TOKENS) and not any(tok in name for tok in arch_tokens):
|
|
241
|
+
continue
|
|
242
|
+
|
|
243
|
+
score = 1
|
|
244
|
+
if any(tok in name for tok in os_tokens):
|
|
245
|
+
score += 2
|
|
246
|
+
if any(tok in name for tok in arch_tokens):
|
|
247
|
+
score += 2
|
|
248
|
+
score += sum(1 for hint in include if hint.lower() in name)
|
|
249
|
+
# Prefer gnu over musl when both are published, all else equal.
|
|
250
|
+
if "gnu" in name:
|
|
251
|
+
score += 1
|
|
252
|
+
|
|
253
|
+
if score > best_score:
|
|
254
|
+
best, best_score = asset, score
|
|
255
|
+
|
|
256
|
+
return best
|
nullforge/smithy/http.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""HTTP utilities for NullForge."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import urllib.request
|
|
5
|
+
|
|
6
|
+
from pyinfra.context import host
|
|
7
|
+
from pyinfra.facts.server import Command
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
LOG = logging.getLogger("pyinfra")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
CURL_FLAGS = (
|
|
14
|
+
"--compressed",
|
|
15
|
+
"--fail",
|
|
16
|
+
"--retry-connrefused",
|
|
17
|
+
"--tlsv1.2",
|
|
18
|
+
)
|
|
19
|
+
"""Curl options that take no value."""
|
|
20
|
+
|
|
21
|
+
CURL_ARGS = {
|
|
22
|
+
"--retry": "3",
|
|
23
|
+
"--connect-timeout": "10",
|
|
24
|
+
"--max-time": "120",
|
|
25
|
+
"--proto": "=https",
|
|
26
|
+
}
|
|
27
|
+
"""Robust curl options for reliable downloads."""
|
|
28
|
+
|
|
29
|
+
CURL_ARGS_STR = " ".join((*CURL_FLAGS, *(f"{k} {v}" for k, v in CURL_ARGS.items())))
|
|
30
|
+
"""String representation of curl arguments."""
|
|
31
|
+
|
|
32
|
+
GITHUB_KEYS_URL = "https://github.com/{user}.keys"
|
|
33
|
+
"""Public endpoint serving user's authorized SSH public keys."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def reachable_address(hostname: str, port: int = 443) -> str | None:
|
|
37
|
+
"""First address of `hostname` this host can actually talk HTTPS to, if any."""
|
|
38
|
+
|
|
39
|
+
cache_key = f"_nullforge_reachable_{hostname}"
|
|
40
|
+
if hasattr(host.data, cache_key):
|
|
41
|
+
return getattr(host.data, cache_key)
|
|
42
|
+
|
|
43
|
+
address = None
|
|
44
|
+
listed = host.get_fact(Command, f"getent ahostsv4 {hostname} | awk '{{print $1}}' | sort -u || true")
|
|
45
|
+
for candidate in str(listed or "").split():
|
|
46
|
+
# HTTPS answer proves path works; status code is irrelevant
|
|
47
|
+
probe = f"curl -sS -o /dev/null --connect-timeout 5 --resolve {hostname}:{port}:{candidate} https://{hostname}/"
|
|
48
|
+
if str(host.get_fact(Command, f"{probe} >/dev/null 2>&1 && echo reachable || true") or "").strip():
|
|
49
|
+
address = candidate
|
|
50
|
+
break
|
|
51
|
+
|
|
52
|
+
setattr(host.data, cache_key, address)
|
|
53
|
+
return address
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def pin_host_args(hostname: str, port: int = 443) -> dict[str, str]:
|
|
57
|
+
address = reachable_address(hostname, port)
|
|
58
|
+
return {"--resolve": f"{hostname}:{port}:{address}"} if address else {}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def fetch_text(url: str, *, timeout: int = 10, headers: dict[str, str] | None = None) -> str:
|
|
62
|
+
"""Fetch text from HTTPS URL on control node."""
|
|
63
|
+
|
|
64
|
+
if not url.startswith("https://"):
|
|
65
|
+
raise ValueError(f"Refusing non-HTTPS URL: {url}")
|
|
66
|
+
|
|
67
|
+
headers = headers or {}
|
|
68
|
+
request = urllib.request.Request(url, headers=headers) # noqa: S310 - https-only guard above
|
|
69
|
+
with urllib.request.urlopen(request, timeout=timeout) as resp: # noqa: S310 - https-only guard above
|
|
70
|
+
return resp.read().decode("utf-8")
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def fetch_github_keys(username: str | None) -> list[str]:
|
|
74
|
+
"""Fetch public SSH keys for GitHub user."""
|
|
75
|
+
|
|
76
|
+
if not username:
|
|
77
|
+
return []
|
|
78
|
+
|
|
79
|
+
try:
|
|
80
|
+
contents = fetch_text(GITHUB_KEYS_URL.format(user=username))
|
|
81
|
+
except Exception as e:
|
|
82
|
+
LOG.warning(f"Failed to fetch GitHub SSH keys for '{username}': {e}")
|
|
83
|
+
return []
|
|
84
|
+
|
|
85
|
+
return contents.splitlines()
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""Release-binary installation helper for NullForge.
|
|
2
|
+
|
|
3
|
+
`install_release_binary` "centralises download -> verify -> extract -> locate -> install"
|
|
4
|
+
dance that every pinned-tool rune used to hand-roll.
|
|
5
|
+
|
|
6
|
+
- auto-detect archive format from URL (tar.gz/xz/bz2, plain tar, zip, gzip, or raw/AppImage binary)
|
|
7
|
+
- verifies download against sha256, resolving it from GitHub release metadata when not supplied
|
|
8
|
+
- extracts and locates executable inside the archive, and installs it to `dest` with requested mode
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import shlex
|
|
12
|
+
from pathlib import PurePosixPath
|
|
13
|
+
|
|
14
|
+
from pyinfra import logger
|
|
15
|
+
from pyinfra.api.operation import OperationMeta
|
|
16
|
+
from pyinfra.operations import files, server
|
|
17
|
+
|
|
18
|
+
from nullforge.smithy.github import sha256_for_download_url
|
|
19
|
+
from nullforge.smithy.versions import Versions
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
STAGING_DIR = "/var/lib/nullforge/staging"
|
|
23
|
+
"""Scratch space for archives and their extracted contents."""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
# Archive kind -> the tar/unzip/gunzip commands that unpack it into `workdir`.
|
|
27
|
+
_TAR_FLAGS = {
|
|
28
|
+
"tar.gz": "-xzf",
|
|
29
|
+
"tar.xz": "-xJf",
|
|
30
|
+
"tar.bz2": "-xjf",
|
|
31
|
+
"tar": "-xf",
|
|
32
|
+
}
|
|
33
|
+
# Ordered longest-suffix-first so `.tar.gz` wins over `.gz`.
|
|
34
|
+
_SUFFIX_KINDS: tuple[tuple[str, str], ...] = (
|
|
35
|
+
(".tar.gz", "tar.gz"),
|
|
36
|
+
(".tgz", "tar.gz"),
|
|
37
|
+
(".tar.xz", "tar.xz"),
|
|
38
|
+
(".txz", "tar.xz"),
|
|
39
|
+
(".tar.bz2", "tar.bz2"),
|
|
40
|
+
(".tbz2", "tar.bz2"),
|
|
41
|
+
(".tar", "tar"),
|
|
42
|
+
(".zip", "zip"),
|
|
43
|
+
(".appimage", "raw"),
|
|
44
|
+
(".gz", "gzip"),
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def detect_archive(url_or_name: str) -> str:
|
|
49
|
+
"""Classify URL/filename as `tar.gz`/`tar.xz`/`tar.bz2`/`tar`/`zip`/`gzip`/`raw`."""
|
|
50
|
+
|
|
51
|
+
name = url_or_name.rsplit("/", 1)[-1].lower()
|
|
52
|
+
for suffix, kind in _SUFFIX_KINDS:
|
|
53
|
+
if name.endswith(suffix):
|
|
54
|
+
return kind
|
|
55
|
+
return "raw"
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _extract_commands(kind: str, archive: str, workdir: str, binary_name: str) -> list[str]:
|
|
59
|
+
if kind in _TAR_FLAGS:
|
|
60
|
+
return [f"tar {_TAR_FLAGS[kind]} {shlex.quote(archive)} -C {shlex.quote(workdir)}"]
|
|
61
|
+
if kind == "zip":
|
|
62
|
+
return [f"unzip -o {shlex.quote(archive)} -d {shlex.quote(workdir)}"]
|
|
63
|
+
if kind == "gzip":
|
|
64
|
+
target = f"{workdir}/{binary_name}"
|
|
65
|
+
return [f"gunzip -c {shlex.quote(archive)} > {shlex.quote(target)}"]
|
|
66
|
+
raise ValueError(f"{kind} is not an extractable archive kind")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def install_release_binary(
|
|
70
|
+
*,
|
|
71
|
+
name: str,
|
|
72
|
+
url: str,
|
|
73
|
+
dest: str,
|
|
74
|
+
binary_name: str | None = None,
|
|
75
|
+
sha256: str | None = None,
|
|
76
|
+
verify: bool = True,
|
|
77
|
+
archive: str | None = None,
|
|
78
|
+
mode: str = "0755",
|
|
79
|
+
sudo: bool = True,
|
|
80
|
+
retries: int = 3,
|
|
81
|
+
retry_delay: int = 10,
|
|
82
|
+
) -> OperationMeta:
|
|
83
|
+
"""Download, verify, extract and install release binary; return install operation.
|
|
84
|
+
|
|
85
|
+
`url` is a release-asset URL, usually straight from `Versions` builder,
|
|
86
|
+
and `dest` is absolute path executable ends up at; `name` is what pyinfra prints for install op.
|
|
87
|
+
|
|
88
|
+
Integrity is best-effort by design: `sha256` pins checksum, and leaving it unset has `verify`
|
|
89
|
+
resolve one from `url`'s GitHub release metadata instead.
|
|
90
|
+
A repo that publishes no checksums therefore installs unverified.
|
|
91
|
+
|
|
92
|
+
Archive kind is detected from `url` unless `archive` forces one, and executable is picked out
|
|
93
|
+
of it by `binary_name`, which defaults to `dest` basename.
|
|
94
|
+
"""
|
|
95
|
+
|
|
96
|
+
binary_name = binary_name or PurePosixPath(dest).name
|
|
97
|
+
kind = archive or detect_archive(url)
|
|
98
|
+
checksum = sha256 if sha256 is not None else (sha256_for_download_url(url) if verify else None)
|
|
99
|
+
if verify and checksum is None:
|
|
100
|
+
logger.warning(f"No sha256 could be resolved for {url}; installing {name} without integrity verification")
|
|
101
|
+
curl_args = Versions.release_curl_args()
|
|
102
|
+
|
|
103
|
+
if kind == "raw":
|
|
104
|
+
return files.download(
|
|
105
|
+
name=name,
|
|
106
|
+
src=url,
|
|
107
|
+
dest=dest,
|
|
108
|
+
mode=mode,
|
|
109
|
+
force=True,
|
|
110
|
+
sha256sum=checksum,
|
|
111
|
+
extra_curl_args=curl_args,
|
|
112
|
+
_sudo=sudo,
|
|
113
|
+
_retries=retries,
|
|
114
|
+
_retry_delay=retry_delay,
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
download_path = f"{STAGING_DIR}/{PurePosixPath(url).name}"
|
|
118
|
+
workdir = f"{STAGING_DIR}/{binary_name}"
|
|
119
|
+
|
|
120
|
+
files.directory(
|
|
121
|
+
name=f"Ensure staging directory for {name}",
|
|
122
|
+
path=STAGING_DIR,
|
|
123
|
+
mode="0700",
|
|
124
|
+
_sudo=sudo,
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
files.download(
|
|
128
|
+
name=f"Download {name}",
|
|
129
|
+
src=url,
|
|
130
|
+
dest=download_path,
|
|
131
|
+
force=True,
|
|
132
|
+
sha256sum=checksum,
|
|
133
|
+
extra_curl_args=curl_args,
|
|
134
|
+
_sudo=sudo,
|
|
135
|
+
_retries=retries,
|
|
136
|
+
_retry_delay=retry_delay,
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
located = f'"$(find {shlex.quote(workdir)} -type f -name {shlex.quote(binary_name)} | head -n1)"'
|
|
140
|
+
return server.shell(
|
|
141
|
+
name=name,
|
|
142
|
+
commands=[
|
|
143
|
+
f"rm -rf {shlex.quote(workdir)} && mkdir -p {shlex.quote(workdir)}",
|
|
144
|
+
*_extract_commands(kind, download_path, workdir, binary_name),
|
|
145
|
+
f"install -D -m {mode} {located} {shlex.quote(dest)}",
|
|
146
|
+
f"rm -rf {shlex.quote(workdir)} {shlex.quote(download_path)}",
|
|
147
|
+
],
|
|
148
|
+
_sudo=sudo,
|
|
149
|
+
)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Monitoring backend provisioning helpers for NullForge."""
|
|
File without changes
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""Nezha monitoring agent install builders."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import shlex
|
|
6
|
+
import uuid
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def agent_uuid_for_hostname(hostname: str, namespace: uuid.UUID | str | None = None) -> str:
|
|
10
|
+
if not isinstance(hostname, (str, bytes)):
|
|
11
|
+
hostname = str(hostname) if hostname is not None else ""
|
|
12
|
+
if namespace is None:
|
|
13
|
+
ns = uuid.NAMESPACE_DNS
|
|
14
|
+
elif isinstance(namespace, uuid.UUID):
|
|
15
|
+
ns = namespace
|
|
16
|
+
else:
|
|
17
|
+
ns = uuid.UUID(namespace)
|
|
18
|
+
return str(uuid.uuid5(ns, hostname))
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def build_nezha_install_command(
|
|
22
|
+
*,
|
|
23
|
+
install_url: str,
|
|
24
|
+
curl_args: str,
|
|
25
|
+
server: str,
|
|
26
|
+
tls: bool,
|
|
27
|
+
client_secret: str,
|
|
28
|
+
agent_uuid: str,
|
|
29
|
+
disable_auto_update: bool = False,
|
|
30
|
+
disable_command_execute: bool = False,
|
|
31
|
+
) -> str:
|
|
32
|
+
"""Build shell one-liner that downloads and runs official Nezha agent installer."""
|
|
33
|
+
|
|
34
|
+
env = {
|
|
35
|
+
"NZ_SERVER": server,
|
|
36
|
+
"NZ_TLS": "true" if tls else "false",
|
|
37
|
+
"NZ_CLIENT_SECRET": client_secret,
|
|
38
|
+
"NZ_UUID": agent_uuid,
|
|
39
|
+
"NZ_DISABLE_AUTO_UPDATE": "true" if disable_auto_update else "false",
|
|
40
|
+
"NZ_DISABLE_COMMAND_EXECUTE": "true" if disable_command_execute else "false",
|
|
41
|
+
}
|
|
42
|
+
env_str = " ".join(f"{key}={shlex.quote(value)}" for key, value in env.items())
|
|
43
|
+
script = '"$script"'
|
|
44
|
+
|
|
45
|
+
# if github download fails, retry the same install via mirror
|
|
46
|
+
run_install = f"env {env_str} {script} || env CN=true {env_str} {script}"
|
|
47
|
+
|
|
48
|
+
return (
|
|
49
|
+
f"script=$(mktemp) "
|
|
50
|
+
f"&& curl -L {curl_args} {shlex.quote(install_url)} -o {script} "
|
|
51
|
+
f"&& chmod +x {script} "
|
|
52
|
+
f"&& {{ {run_install}; }}; "
|
|
53
|
+
f"rc=$?; rm -f {script}; exit $rc"
|
|
54
|
+
)
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"""Nezha dashboard REST client."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import logging
|
|
7
|
+
import urllib.error
|
|
8
|
+
import urllib.request
|
|
9
|
+
from time import sleep
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
LOG = logging.getLogger("pyinfra")
|
|
14
|
+
|
|
15
|
+
REGISTER_POLL_ATTEMPTS = 20
|
|
16
|
+
"""How many times to poll dashboard for freshly-registered agent."""
|
|
17
|
+
|
|
18
|
+
REGISTER_POLL_DELAY = 3
|
|
19
|
+
"""Seconds between registration polls."""
|
|
20
|
+
|
|
21
|
+
HTTP_TIMEOUT = 10
|
|
22
|
+
"""Request timeout, in seconds, for dashboard API calls."""
|
|
23
|
+
|
|
24
|
+
USER_AGENT = "nullforge-nezha-client/1.0"
|
|
25
|
+
"""Sent on every dashboard request."""
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _api_request(
|
|
29
|
+
url: str,
|
|
30
|
+
token: str,
|
|
31
|
+
*,
|
|
32
|
+
method: str = "GET",
|
|
33
|
+
payload: dict[str, object] | None = None,
|
|
34
|
+
) -> dict[str, Any]:
|
|
35
|
+
"""Issue authenticated dashboard API request and return parsed envelope."""
|
|
36
|
+
|
|
37
|
+
if not url.startswith(("https://", "http://")):
|
|
38
|
+
raise ValueError(f"Refusing dashboard URL with unsupported scheme: {url}")
|
|
39
|
+
if url.startswith("http://"):
|
|
40
|
+
LOG.warning(f"Dashboard URL {url} is not HTTPS - the API token is sent in cleartext")
|
|
41
|
+
|
|
42
|
+
data = json.dumps(payload).encode("utf-8") if payload is not None else None
|
|
43
|
+
request = urllib.request.Request(url, data=data, method=method) # noqa: S310 - scheme checked above
|
|
44
|
+
request.add_header("Authorization", f"Bearer {token}")
|
|
45
|
+
request.add_header("User-Agent", USER_AGENT)
|
|
46
|
+
if data is not None:
|
|
47
|
+
request.add_header("Content-Type", "application/json")
|
|
48
|
+
|
|
49
|
+
try:
|
|
50
|
+
with urllib.request.urlopen(request, timeout=HTTP_TIMEOUT) as response: # noqa: S310 - scheme checked above
|
|
51
|
+
body = response.read().decode("utf-8")
|
|
52
|
+
except urllib.error.HTTPError as e:
|
|
53
|
+
detail = e.read().decode("utf-8", "replace").strip()
|
|
54
|
+
raise RuntimeError(
|
|
55
|
+
f"Dashboard API {method} {url} failed ({e.code} {e.reason}): {detail or '<empty body>'}"
|
|
56
|
+
) from e
|
|
57
|
+
|
|
58
|
+
parsed: Any = json.loads(body) if body else {}
|
|
59
|
+
if not isinstance(parsed, dict):
|
|
60
|
+
raise ValueError(f"Unexpected dashboard response shape from {url}: {type(parsed)!r}")
|
|
61
|
+
if parsed.get("success") is False:
|
|
62
|
+
raise RuntimeError(f"Dashboard API error from {url}: {parsed.get('error', 'unknown error')}")
|
|
63
|
+
return parsed
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _find_server(list_url: str, token: str, agent_uuid: str) -> tuple[int | None, str | None]:
|
|
67
|
+
envelope = _api_request(list_url, token)
|
|
68
|
+
servers = envelope.get("data")
|
|
69
|
+
if not isinstance(servers, list):
|
|
70
|
+
return None, None
|
|
71
|
+
|
|
72
|
+
for entry in servers:
|
|
73
|
+
if isinstance(entry, dict) and entry.get("uuid") == agent_uuid:
|
|
74
|
+
server_id = entry.get("id")
|
|
75
|
+
name = entry.get("name")
|
|
76
|
+
if isinstance(server_id, int):
|
|
77
|
+
return server_id, (name if isinstance(name, str) else None)
|
|
78
|
+
return None, None
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def server_name_matches(
|
|
82
|
+
*,
|
|
83
|
+
dashboard_url: str,
|
|
84
|
+
api_token: str,
|
|
85
|
+
agent_uuid: str,
|
|
86
|
+
desired_name: str,
|
|
87
|
+
) -> bool:
|
|
88
|
+
"""Whether the dashboard already lists `agent_uuid` under `desired_name`."""
|
|
89
|
+
|
|
90
|
+
try:
|
|
91
|
+
server_id, current_name = _find_server(f"{dashboard_url}/api/v1/server", api_token, agent_uuid)
|
|
92
|
+
except (RuntimeError, ValueError, OSError, json.JSONDecodeError) as e:
|
|
93
|
+
LOG.warning(f"Could not read Nezha dashboard entry for {agent_uuid}: {e}")
|
|
94
|
+
return False
|
|
95
|
+
|
|
96
|
+
return server_id is not None and current_name == desired_name
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def rename_nezha_server(
|
|
100
|
+
*,
|
|
101
|
+
dashboard_url: str,
|
|
102
|
+
api_token: str,
|
|
103
|
+
agent_uuid: str,
|
|
104
|
+
desired_name: str,
|
|
105
|
+
) -> None:
|
|
106
|
+
"""Rename auto-registered dashboard entry for `agent_uuid` to `desired_name`."""
|
|
107
|
+
|
|
108
|
+
list_url = f"{dashboard_url}/api/v1/server"
|
|
109
|
+
|
|
110
|
+
server_id: int | None = None
|
|
111
|
+
current_name: str | None = None
|
|
112
|
+
for attempt in range(REGISTER_POLL_ATTEMPTS):
|
|
113
|
+
# _api_request raises RuntimeError (with the dashboard's error body) on HTTP
|
|
114
|
+
# errors such as 403; a hard failure propagates instead of pointlessly polling.
|
|
115
|
+
server_id, current_name = _find_server(list_url, api_token, agent_uuid)
|
|
116
|
+
if server_id is not None:
|
|
117
|
+
break
|
|
118
|
+
if attempt < REGISTER_POLL_ATTEMPTS - 1:
|
|
119
|
+
sleep(REGISTER_POLL_DELAY)
|
|
120
|
+
|
|
121
|
+
if server_id is None:
|
|
122
|
+
LOG.warning(
|
|
123
|
+
f"Nezha agent {agent_uuid} did not register within {REGISTER_POLL_ATTEMPTS * REGISTER_POLL_DELAY}s; "
|
|
124
|
+
f"skipping rename to {desired_name}",
|
|
125
|
+
)
|
|
126
|
+
return
|
|
127
|
+
|
|
128
|
+
if current_name == desired_name:
|
|
129
|
+
return
|
|
130
|
+
|
|
131
|
+
_api_request(
|
|
132
|
+
f"{dashboard_url}/api/v1/server/{server_id}",
|
|
133
|
+
api_token,
|
|
134
|
+
method="PATCH",
|
|
135
|
+
payload={"name": desired_name},
|
|
136
|
+
)
|
|
137
|
+
LOG.info(f"Renamed Nezha server {server_id} to {desired_name}")
|