portacode 1.3.32__py3-none-any.whl → 1.4.15.dev3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- portacode/_version.py +2 -2
- portacode/cli.py +158 -14
- portacode/connection/client.py +127 -8
- portacode/connection/handlers/WEBSOCKET_PROTOCOL.md +600 -4
- portacode/connection/handlers/__init__.py +30 -1
- portacode/connection/handlers/diff_handlers.py +603 -0
- portacode/connection/handlers/file_handlers.py +674 -17
- portacode/connection/handlers/project_aware_file_handlers.py +11 -0
- portacode/connection/handlers/project_state/file_system_watcher.py +31 -61
- portacode/connection/handlers/project_state/git_manager.py +139 -572
- portacode/connection/handlers/project_state/handlers.py +28 -14
- portacode/connection/handlers/project_state/manager.py +226 -101
- portacode/connection/handlers/proxmox_infra.py +2082 -0
- portacode/connection/handlers/session.py +465 -84
- portacode/connection/handlers/system_handlers.py +311 -9
- portacode/connection/handlers/tab_factory.py +1 -47
- portacode/connection/handlers/test_proxmox_infra.py +13 -0
- portacode/connection/handlers/update_handler.py +61 -0
- portacode/connection/terminal.py +64 -10
- portacode/keypair.py +63 -1
- portacode/link_capture/__init__.py +38 -0
- portacode/link_capture/__pycache__/__init__.cpython-311.pyc +0 -0
- portacode/link_capture/bin/__pycache__/link_capture_wrapper.cpython-311.pyc +0 -0
- portacode/link_capture/bin/elinks +3 -0
- portacode/link_capture/bin/gio-open +3 -0
- portacode/link_capture/bin/gnome-open +3 -0
- portacode/link_capture/bin/gvfs-open +3 -0
- portacode/link_capture/bin/kde-open +3 -0
- portacode/link_capture/bin/kfmclient +3 -0
- portacode/link_capture/bin/link_capture_exec.sh +11 -0
- portacode/link_capture/bin/link_capture_wrapper.py +75 -0
- portacode/link_capture/bin/links +3 -0
- portacode/link_capture/bin/links2 +3 -0
- portacode/link_capture/bin/lynx +3 -0
- portacode/link_capture/bin/mate-open +3 -0
- portacode/link_capture/bin/netsurf +3 -0
- portacode/link_capture/bin/sensible-browser +3 -0
- portacode/link_capture/bin/w3m +3 -0
- portacode/link_capture/bin/x-www-browser +3 -0
- portacode/link_capture/bin/xdg-open +3 -0
- portacode/pairing.py +103 -0
- portacode/static/js/utils/ntp-clock.js +170 -79
- portacode/utils/diff_apply.py +456 -0
- portacode/utils/diff_renderer.py +371 -0
- portacode/utils/ntp_clock.py +45 -131
- {portacode-1.3.32.dist-info → portacode-1.4.15.dev3.dist-info}/METADATA +71 -3
- portacode-1.4.15.dev3.dist-info/RECORD +98 -0
- {portacode-1.3.32.dist-info → portacode-1.4.15.dev3.dist-info}/WHEEL +1 -1
- test_modules/test_device_online.py +1 -1
- test_modules/test_login_flow.py +8 -4
- test_modules/test_play_store_screenshots.py +294 -0
- testing_framework/.env.example +4 -1
- testing_framework/core/playwright_manager.py +63 -9
- portacode-1.3.32.dist-info/RECORD +0 -70
- {portacode-1.3.32.dist-info → portacode-1.4.15.dev3.dist-info}/entry_points.txt +0 -0
- {portacode-1.3.32.dist-info → portacode-1.4.15.dev3.dist-info}/licenses/LICENSE +0 -0
- {portacode-1.3.32.dist-info → portacode-1.4.15.dev3.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,2082 @@
|
|
|
1
|
+
"""Proxmox infrastructure configuration handler."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import json
|
|
7
|
+
import logging
|
|
8
|
+
import os
|
|
9
|
+
import secrets
|
|
10
|
+
import shlex
|
|
11
|
+
import re
|
|
12
|
+
import select
|
|
13
|
+
import shutil
|
|
14
|
+
import stat
|
|
15
|
+
import subprocess
|
|
16
|
+
import sys
|
|
17
|
+
import tempfile
|
|
18
|
+
import time
|
|
19
|
+
import threading
|
|
20
|
+
import urllib.request
|
|
21
|
+
from datetime import datetime
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple
|
|
24
|
+
|
|
25
|
+
import platformdirs
|
|
26
|
+
|
|
27
|
+
from .base import SyncHandler
|
|
28
|
+
|
|
29
|
+
logger = logging.getLogger(__name__)
|
|
30
|
+
|
|
31
|
+
CONFIG_DIR = Path(platformdirs.user_config_dir("portacode"))
|
|
32
|
+
CONFIG_PATH = CONFIG_DIR / "proxmox_infra.json"
|
|
33
|
+
REPO_ROOT = Path(__file__).resolve().parents[3]
|
|
34
|
+
NET_SETUP_SCRIPT = REPO_ROOT / "proxmox_management" / "net_setup.py"
|
|
35
|
+
CONTAINERS_DIR = CONFIG_DIR / "containers"
|
|
36
|
+
MANAGED_MARKER = "portacode-managed:true"
|
|
37
|
+
|
|
38
|
+
DEFAULT_HOST = "localhost"
|
|
39
|
+
DEFAULT_NODE_NAME = os.uname().nodename.split(".", 1)[0]
|
|
40
|
+
DEFAULT_BRIDGE = "vmbr1"
|
|
41
|
+
SUBNET_CIDR = "10.10.0.1/24"
|
|
42
|
+
BRIDGE_IP = SUBNET_CIDR.split("/", 1)[0]
|
|
43
|
+
DHCP_START = "10.10.0.100"
|
|
44
|
+
DHCP_END = "10.10.0.200"
|
|
45
|
+
DNS_SERVER = "1.1.1.1"
|
|
46
|
+
CLOUDFLARE_DEB_URL = "https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64.deb"
|
|
47
|
+
IFACES_PATH = Path("/etc/network/interfaces")
|
|
48
|
+
SYSCTL_PATH = Path("/etc/sysctl.d/99-portacode-forward.conf")
|
|
49
|
+
UNIT_DIR = Path("/etc/systemd/system")
|
|
50
|
+
_MANAGED_CONTAINERS_CACHE_TTL_S = 30.0
|
|
51
|
+
_MANAGED_CONTAINERS_CACHE: Dict[str, Any] = {"timestamp": 0.0, "summary": None}
|
|
52
|
+
_MANAGED_CONTAINERS_CACHE_LOCK = threading.Lock()
|
|
53
|
+
_CLOUDFLARE_TUNNEL_PROCESSES: Dict[str, subprocess.Popen] = {}
|
|
54
|
+
_CLOUDFLARE_TUNNELS_LOCK = threading.Lock()
|
|
55
|
+
|
|
56
|
+
ProgressCallback = Callable[[int, int, Dict[str, Any], str, Optional[Dict[str, Any]]], None]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _emit_progress_event(
|
|
60
|
+
handler: SyncHandler,
|
|
61
|
+
*,
|
|
62
|
+
step_index: int,
|
|
63
|
+
total_steps: int,
|
|
64
|
+
step_name: str,
|
|
65
|
+
step_label: str,
|
|
66
|
+
status: str,
|
|
67
|
+
message: str,
|
|
68
|
+
phase: str,
|
|
69
|
+
request_id: Optional[str],
|
|
70
|
+
details: Optional[Dict[str, Any]] = None,
|
|
71
|
+
) -> None:
|
|
72
|
+
loop = handler.context.get("event_loop")
|
|
73
|
+
if not loop or loop.is_closed():
|
|
74
|
+
logger.debug(
|
|
75
|
+
"progress event skipped (no event loop) step=%s status=%s",
|
|
76
|
+
step_name,
|
|
77
|
+
status,
|
|
78
|
+
)
|
|
79
|
+
return
|
|
80
|
+
|
|
81
|
+
payload: Dict[str, Any] = {
|
|
82
|
+
"event": "proxmox_container_progress",
|
|
83
|
+
"step_name": step_name,
|
|
84
|
+
"step_label": step_label,
|
|
85
|
+
"status": status,
|
|
86
|
+
"phase": phase,
|
|
87
|
+
"step_index": step_index,
|
|
88
|
+
"total_steps": total_steps,
|
|
89
|
+
"message": message,
|
|
90
|
+
}
|
|
91
|
+
if request_id:
|
|
92
|
+
payload["request_id"] = request_id
|
|
93
|
+
if details:
|
|
94
|
+
payload["details"] = details
|
|
95
|
+
|
|
96
|
+
future = asyncio.run_coroutine_threadsafe(handler.send_response(payload), loop)
|
|
97
|
+
future.add_done_callback(
|
|
98
|
+
lambda fut: logger.warning(
|
|
99
|
+
"Failed to emit progress event for %s: %s", step_name, fut.exception()
|
|
100
|
+
)
|
|
101
|
+
if fut.exception()
|
|
102
|
+
else None
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _call_subprocess(cmd: List[str], **kwargs: Any) -> subprocess.CompletedProcess[str]:
|
|
107
|
+
env = os.environ.copy()
|
|
108
|
+
env.setdefault("DEBIAN_FRONTEND", "noninteractive")
|
|
109
|
+
return subprocess.run(cmd, env=env, text=True, capture_output=True, **kwargs)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _ensure_proxmoxer() -> Any:
|
|
113
|
+
try:
|
|
114
|
+
from proxmoxer import ProxmoxAPI # noqa: F401
|
|
115
|
+
except ModuleNotFoundError as exc:
|
|
116
|
+
python = sys.executable
|
|
117
|
+
logger.info("Proxmoxer missing; installing via pip")
|
|
118
|
+
try:
|
|
119
|
+
_call_subprocess([python, "-m", "pip", "install", "proxmoxer"], check=True)
|
|
120
|
+
except subprocess.CalledProcessError as pip_exc:
|
|
121
|
+
msg = pip_exc.stderr or pip_exc.stdout or str(pip_exc)
|
|
122
|
+
raise RuntimeError(f"Failed to install proxmoxer: {msg}") from pip_exc
|
|
123
|
+
from proxmoxer import ProxmoxAPI # noqa: F401
|
|
124
|
+
from proxmoxer import ProxmoxAPI
|
|
125
|
+
return ProxmoxAPI
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _parse_token(token_identifier: str) -> Tuple[str, str]:
|
|
129
|
+
identifier = token_identifier.strip()
|
|
130
|
+
if "!" not in identifier or "@" not in identifier:
|
|
131
|
+
raise ValueError("Expected API token in the form user@realm!tokenid")
|
|
132
|
+
user_part, token_name = identifier.split("!", 1)
|
|
133
|
+
user = user_part.strip()
|
|
134
|
+
token_name = token_name.strip()
|
|
135
|
+
if "@" not in user:
|
|
136
|
+
raise ValueError("API token missing user realm (user@realm)")
|
|
137
|
+
if not token_name:
|
|
138
|
+
raise ValueError("Token identifier missing token name")
|
|
139
|
+
return user, token_name
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _save_config(data: Dict[str, Any]) -> None:
|
|
143
|
+
CONFIG_DIR.mkdir(parents=True, exist_ok=True)
|
|
144
|
+
tmp_path = CONFIG_PATH.with_suffix(".tmp")
|
|
145
|
+
tmp_path.write_text(json.dumps(data, indent=2), encoding="utf-8")
|
|
146
|
+
os.replace(tmp_path, CONFIG_PATH)
|
|
147
|
+
os.chmod(CONFIG_PATH, stat.S_IRUSR | stat.S_IWUSR)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _load_config() -> Dict[str, Any]:
|
|
151
|
+
if not CONFIG_PATH.exists():
|
|
152
|
+
return {}
|
|
153
|
+
try:
|
|
154
|
+
return json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
|
|
155
|
+
except json.JSONDecodeError as exc:
|
|
156
|
+
logger.warning("Failed to parse Proxmox infra config: %s", exc)
|
|
157
|
+
return {}
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _pick_node(client: Any) -> str:
|
|
161
|
+
nodes = client.nodes().get()
|
|
162
|
+
for node in nodes:
|
|
163
|
+
if node.get("node") == DEFAULT_NODE_NAME:
|
|
164
|
+
return DEFAULT_NODE_NAME
|
|
165
|
+
return nodes[0].get("node") if nodes else DEFAULT_NODE_NAME
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _list_templates(client: Any, node: str, storages: Iterable[Dict[str, Any]]) -> List[str]:
|
|
169
|
+
templates: List[str] = []
|
|
170
|
+
for storage in storages:
|
|
171
|
+
storage_name = storage.get("storage")
|
|
172
|
+
if not storage_name:
|
|
173
|
+
continue
|
|
174
|
+
try:
|
|
175
|
+
items = client.nodes(node).storage(storage_name).content.get()
|
|
176
|
+
except Exception:
|
|
177
|
+
continue
|
|
178
|
+
for item in items:
|
|
179
|
+
if item.get("content") == "vztmpl" and item.get("volid"):
|
|
180
|
+
templates.append(item["volid"])
|
|
181
|
+
return templates
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _pick_storage(storages: Iterable[Dict[str, Any]]) -> str:
|
|
185
|
+
candidates = [s for s in storages if "rootdir" in s.get("content", "") and s.get("avail", 0) > 0]
|
|
186
|
+
if not candidates:
|
|
187
|
+
candidates = [s for s in storages if "rootdir" in s.get("content", "")]
|
|
188
|
+
if not candidates:
|
|
189
|
+
return ""
|
|
190
|
+
candidates.sort(key=lambda entry: entry.get("avail", 0), reverse=True)
|
|
191
|
+
return candidates[0].get("storage", "")
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _write_bridge_config(bridge: str) -> None:
|
|
195
|
+
begin = f"# Portacode INFRA BEGIN {bridge}"
|
|
196
|
+
end = f"# Portacode INFRA END {bridge}"
|
|
197
|
+
current = IFACES_PATH.read_text(encoding="utf-8") if IFACES_PATH.exists() else ""
|
|
198
|
+
if begin in current:
|
|
199
|
+
return
|
|
200
|
+
block = f"""
|
|
201
|
+
{begin}
|
|
202
|
+
auto {bridge}
|
|
203
|
+
iface {bridge} inet static
|
|
204
|
+
address {SUBNET_CIDR}
|
|
205
|
+
bridge-ports none
|
|
206
|
+
bridge-stp off
|
|
207
|
+
bridge-fd 0
|
|
208
|
+
{end}
|
|
209
|
+
|
|
210
|
+
"""
|
|
211
|
+
mode = "a" if IFACES_PATH.exists() else "w"
|
|
212
|
+
with open(IFACES_PATH, mode, encoding="utf-8") as fh:
|
|
213
|
+
if current and not current.endswith("\n"):
|
|
214
|
+
fh.write("\n")
|
|
215
|
+
fh.write(block)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _ensure_sysctl() -> None:
|
|
219
|
+
SYSCTL_PATH.write_text("net.ipv4.ip_forward=1\n", encoding="utf-8")
|
|
220
|
+
_call_subprocess(["/sbin/sysctl", "-w", "net.ipv4.ip_forward=1"], check=True)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _write_units(bridge: str) -> None:
|
|
224
|
+
nat_name = f"portacode-{bridge}-nat.service"
|
|
225
|
+
dns_name = f"portacode-{bridge}-dnsmasq.service"
|
|
226
|
+
nat = UNIT_DIR / nat_name
|
|
227
|
+
dns = UNIT_DIR / dns_name
|
|
228
|
+
nat.write_text(f"""[Unit]
|
|
229
|
+
Description=Portacode NAT for {bridge}
|
|
230
|
+
After=network-online.target
|
|
231
|
+
Wants=network-online.target
|
|
232
|
+
|
|
233
|
+
[Service]
|
|
234
|
+
Type=oneshot
|
|
235
|
+
RemainAfterExit=yes
|
|
236
|
+
ExecStart=/usr/sbin/iptables -t nat -A POSTROUTING -s {BRIDGE_IP}/24 -o vmbr0 -j MASQUERADE
|
|
237
|
+
ExecStart=/usr/sbin/iptables -A FORWARD -i {bridge} -o vmbr0 -j ACCEPT
|
|
238
|
+
ExecStart=/usr/sbin/iptables -A FORWARD -i vmbr0 -o {bridge} -m state --state RELATED,ESTABLISHED -j ACCEPT
|
|
239
|
+
ExecStop=/usr/sbin/iptables -t nat -D POSTROUTING -s {BRIDGE_IP}/24 -o vmbr0 -j MASQUERADE
|
|
240
|
+
ExecStop=/usr/sbin/iptables -D FORWARD -i {bridge} -o vmbr0 -j ACCEPT
|
|
241
|
+
ExecStop=/usr/sbin/iptables -D FORWARD -i vmbr0 -o {bridge} -m state --state RELATED,ESTABLISHED -j ACCEPT
|
|
242
|
+
|
|
243
|
+
[Install]
|
|
244
|
+
WantedBy=multi-user.target
|
|
245
|
+
""", encoding="utf-8")
|
|
246
|
+
dns.write_text(f"""[Unit]
|
|
247
|
+
Description=Portacode dnsmasq for {bridge}
|
|
248
|
+
After=network-online.target
|
|
249
|
+
Wants=network-online.target
|
|
250
|
+
|
|
251
|
+
[Service]
|
|
252
|
+
Type=simple
|
|
253
|
+
ExecStart=/usr/sbin/dnsmasq --keep-in-foreground --interface={bridge} --bind-interfaces --listen-address={BRIDGE_IP} \
|
|
254
|
+
--port=0 --dhcp-range={DHCP_START},{DHCP_END},12h \
|
|
255
|
+
--dhcp-option=option:router,{BRIDGE_IP} \
|
|
256
|
+
--dhcp-option=option:dns-server,{DNS_SERVER} \
|
|
257
|
+
--conf-file=/dev/null --pid-file=/run/portacode_dnsmasq.pid --dhcp-leasefile=/var/lib/misc/portacode_dnsmasq.leases
|
|
258
|
+
Restart=always
|
|
259
|
+
|
|
260
|
+
[Install]
|
|
261
|
+
WantedBy=multi-user.target
|
|
262
|
+
""", encoding="utf-8")
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _ensure_bridge(bridge: str = DEFAULT_BRIDGE) -> Dict[str, Any]:
|
|
266
|
+
if os.geteuid() != 0:
|
|
267
|
+
raise PermissionError("Bridge setup requires root privileges")
|
|
268
|
+
if not shutil.which("dnsmasq"):
|
|
269
|
+
apt = shutil.which("apt-get")
|
|
270
|
+
if not apt:
|
|
271
|
+
raise RuntimeError("dnsmasq is missing and apt-get unavailable to install it")
|
|
272
|
+
update = _call_subprocess([apt, "update"], check=False)
|
|
273
|
+
if update.returncode not in (0, 100):
|
|
274
|
+
msg = update.stderr or update.stdout or f"exit status {update.returncode}"
|
|
275
|
+
raise RuntimeError(f"apt-get update failed: {msg}")
|
|
276
|
+
_call_subprocess([apt, "install", "-y", "dnsmasq"], check=True)
|
|
277
|
+
_write_bridge_config(bridge)
|
|
278
|
+
_ensure_sysctl()
|
|
279
|
+
_write_units(bridge)
|
|
280
|
+
_call_subprocess(["/bin/systemctl", "daemon-reload"], check=True)
|
|
281
|
+
nat_service = f"portacode-{bridge}-nat.service"
|
|
282
|
+
dns_service = f"portacode-{bridge}-dnsmasq.service"
|
|
283
|
+
_call_subprocess(["/bin/systemctl", "enable", "--now", nat_service, dns_service], check=True)
|
|
284
|
+
_call_subprocess(["/sbin/ifup", bridge], check=False)
|
|
285
|
+
return {"applied": True, "bridge": bridge, "message": f"Bridge {bridge} configured"}
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _ensure_cloudflared_installed() -> None:
|
|
289
|
+
if shutil.which("cloudflared"):
|
|
290
|
+
return
|
|
291
|
+
apt = shutil.which("apt-get")
|
|
292
|
+
if not apt:
|
|
293
|
+
raise RuntimeError("cloudflared is missing and apt-get is unavailable to install it")
|
|
294
|
+
download_dir = Path(tempfile.mkdtemp())
|
|
295
|
+
deb_path = download_dir / "cloudflared.deb"
|
|
296
|
+
try:
|
|
297
|
+
urllib.request.urlretrieve(CLOUDFLARE_DEB_URL, deb_path)
|
|
298
|
+
try:
|
|
299
|
+
_call_subprocess(["dpkg", "-i", str(deb_path)], check=True)
|
|
300
|
+
except subprocess.CalledProcessError:
|
|
301
|
+
_call_subprocess([apt, "install", "-f", "-y"], check=True)
|
|
302
|
+
_call_subprocess(["dpkg", "-i", str(deb_path)], check=True)
|
|
303
|
+
finally:
|
|
304
|
+
shutil.rmtree(download_dir, ignore_errors=True)
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def _verify_connectivity(timeout: float = 5.0) -> bool:
|
|
308
|
+
try:
|
|
309
|
+
_call_subprocess(["/bin/ping", "-c", "2", "1.1.1.1"], check=True, timeout=timeout)
|
|
310
|
+
return True
|
|
311
|
+
except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
|
|
312
|
+
return False
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _revert_bridge() -> None:
|
|
316
|
+
try:
|
|
317
|
+
if NET_SETUP_SCRIPT.exists():
|
|
318
|
+
_call_subprocess([sys.executable, str(NET_SETUP_SCRIPT), "revert"], check=True)
|
|
319
|
+
except Exception as exc:
|
|
320
|
+
logger.warning("Proxmox bridge revert failed: %s", exc)
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def _ensure_containers_dir() -> None:
|
|
324
|
+
CONTAINERS_DIR.mkdir(parents=True, exist_ok=True)
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def _invalidate_managed_containers_cache() -> None:
|
|
328
|
+
with _MANAGED_CONTAINERS_CACHE_LOCK:
|
|
329
|
+
_MANAGED_CONTAINERS_CACHE["timestamp"] = 0.0
|
|
330
|
+
_MANAGED_CONTAINERS_CACHE["summary"] = None
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def _load_managed_container_records() -> List[Dict[str, Any]]:
|
|
334
|
+
_ensure_containers_dir()
|
|
335
|
+
records: List[Dict[str, Any]] = []
|
|
336
|
+
for path in sorted(CONTAINERS_DIR.glob("ct-*.json")):
|
|
337
|
+
try:
|
|
338
|
+
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
339
|
+
except Exception as exc: # pragma: no cover - best effort logging
|
|
340
|
+
logger.debug("Unable to read container record %s: %s", path, exc)
|
|
341
|
+
continue
|
|
342
|
+
records.append(payload)
|
|
343
|
+
return records
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def _build_managed_containers_summary(records: List[Dict[str, Any]]) -> Dict[str, Any]:
|
|
347
|
+
total_ram = 0
|
|
348
|
+
total_disk = 0
|
|
349
|
+
total_cpu_share = 0.0
|
|
350
|
+
containers: List[Dict[str, Any]] = []
|
|
351
|
+
|
|
352
|
+
def _as_int(value: Any) -> int:
|
|
353
|
+
try:
|
|
354
|
+
return int(value)
|
|
355
|
+
except (TypeError, ValueError):
|
|
356
|
+
return 0
|
|
357
|
+
|
|
358
|
+
def _as_float(value: Any) -> float:
|
|
359
|
+
try:
|
|
360
|
+
return float(value)
|
|
361
|
+
except (TypeError, ValueError):
|
|
362
|
+
return 0.0
|
|
363
|
+
|
|
364
|
+
for record in sorted(records, key=lambda entry: _as_int(entry.get("vmid"))):
|
|
365
|
+
ram_mib = _as_int(record.get("ram_mib"))
|
|
366
|
+
disk_gib = _as_int(record.get("disk_gib"))
|
|
367
|
+
cpu_share = _as_float(record.get("cpus"))
|
|
368
|
+
total_ram += ram_mib
|
|
369
|
+
total_disk += disk_gib
|
|
370
|
+
total_cpu_share += cpu_share
|
|
371
|
+
status = (record.get("status") or "unknown").lower()
|
|
372
|
+
containers.append(
|
|
373
|
+
{
|
|
374
|
+
"vmid": str(_as_int(record.get("vmid"))) if record.get("vmid") is not None else None,
|
|
375
|
+
"hostname": record.get("hostname"),
|
|
376
|
+
"template": record.get("template"),
|
|
377
|
+
"storage": record.get("storage"),
|
|
378
|
+
"disk_gib": disk_gib,
|
|
379
|
+
"ram_mib": ram_mib,
|
|
380
|
+
"cpu_share": cpu_share,
|
|
381
|
+
"created_at": record.get("created_at"),
|
|
382
|
+
"status": status,
|
|
383
|
+
"tunnel": record.get("tunnel"),
|
|
384
|
+
}
|
|
385
|
+
)
|
|
386
|
+
|
|
387
|
+
return {
|
|
388
|
+
"updated_at": datetime.utcnow().isoformat() + "Z",
|
|
389
|
+
"count": len(containers),
|
|
390
|
+
"total_ram_mib": total_ram,
|
|
391
|
+
"total_disk_gib": total_disk,
|
|
392
|
+
"total_cpu_share": round(total_cpu_share, 2),
|
|
393
|
+
"containers": containers,
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def _get_managed_containers_summary(force: bool = False) -> Dict[str, Any]:
|
|
398
|
+
def _refresh_container_statuses(records: List[Dict[str, Any]], config: Dict[str, Any] | None) -> None:
|
|
399
|
+
if not records or not config:
|
|
400
|
+
return
|
|
401
|
+
try:
|
|
402
|
+
proxmox = _connect_proxmox(config)
|
|
403
|
+
node = _get_node_from_config(config)
|
|
404
|
+
statuses = {
|
|
405
|
+
str(ct.get("vmid")): (ct.get("status") or "unknown").lower()
|
|
406
|
+
for ct in proxmox.nodes(node).lxc.get()
|
|
407
|
+
}
|
|
408
|
+
except Exception as exc: # pragma: no cover - best effort
|
|
409
|
+
logger.debug("Failed to refresh container statuses: %s", exc)
|
|
410
|
+
return
|
|
411
|
+
for record in records:
|
|
412
|
+
vmid = record.get("vmid")
|
|
413
|
+
if vmid is None:
|
|
414
|
+
continue
|
|
415
|
+
try:
|
|
416
|
+
vmid_key = str(int(vmid))
|
|
417
|
+
except (ValueError, TypeError):
|
|
418
|
+
continue
|
|
419
|
+
status = statuses.get(vmid_key)
|
|
420
|
+
if status:
|
|
421
|
+
record["status"] = status
|
|
422
|
+
|
|
423
|
+
now = time.monotonic()
|
|
424
|
+
with _MANAGED_CONTAINERS_CACHE_LOCK:
|
|
425
|
+
cache_ts = _MANAGED_CONTAINERS_CACHE["timestamp"]
|
|
426
|
+
cached = _MANAGED_CONTAINERS_CACHE["summary"]
|
|
427
|
+
if not force and cached and now - cache_ts < _MANAGED_CONTAINERS_CACHE_TTL_S:
|
|
428
|
+
return cached
|
|
429
|
+
config = _load_config()
|
|
430
|
+
records = _load_managed_container_records()
|
|
431
|
+
_refresh_container_statuses(records, config)
|
|
432
|
+
summary = _build_managed_containers_summary(records)
|
|
433
|
+
with _MANAGED_CONTAINERS_CACHE_LOCK:
|
|
434
|
+
_MANAGED_CONTAINERS_CACHE["timestamp"] = now
|
|
435
|
+
_MANAGED_CONTAINERS_CACHE["summary"] = summary
|
|
436
|
+
return summary
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def _format_rootfs(storage: str, disk_gib: int, storage_type: str) -> str:
|
|
440
|
+
if storage_type in ("lvm", "lvmthin"):
|
|
441
|
+
return f"{storage}:{disk_gib}"
|
|
442
|
+
return f"{storage}:{disk_gib}G"
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def _get_provisioning_user_info(message: Dict[str, Any]) -> Tuple[str, str, str]:
|
|
446
|
+
user = (message.get("username") or "svcuser").strip() if message else "svcuser"
|
|
447
|
+
user = user or "svcuser"
|
|
448
|
+
password = message.get("password")
|
|
449
|
+
if not password:
|
|
450
|
+
password = secrets.token_urlsafe(10)
|
|
451
|
+
ssh_key = (message.get("ssh_key") or "").strip() if message else ""
|
|
452
|
+
return user, password, ssh_key
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
def _friendly_step_label(step_name: str) -> str:
|
|
456
|
+
if not step_name:
|
|
457
|
+
return "Step"
|
|
458
|
+
normalized = step_name.replace("_", " ").strip()
|
|
459
|
+
return normalized.capitalize()
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def _build_bootstrap_steps(
|
|
463
|
+
user: str,
|
|
464
|
+
password: str,
|
|
465
|
+
ssh_key: str,
|
|
466
|
+
include_portacode_connect: bool = True,
|
|
467
|
+
) -> List[Dict[str, Any]]:
|
|
468
|
+
steps = [
|
|
469
|
+
{
|
|
470
|
+
"name": "apt_update",
|
|
471
|
+
"cmd": "apt-get update -y",
|
|
472
|
+
"retries": 4,
|
|
473
|
+
"retry_delay_s": 5,
|
|
474
|
+
"retry_on": [
|
|
475
|
+
"Temporary failure resolving",
|
|
476
|
+
"Could not resolve",
|
|
477
|
+
"Failed to fetch",
|
|
478
|
+
],
|
|
479
|
+
},
|
|
480
|
+
{
|
|
481
|
+
"name": "install_deps",
|
|
482
|
+
"cmd": "apt-get install -y python3 python3-pip sudo --fix-missing",
|
|
483
|
+
"retries": 5,
|
|
484
|
+
"retry_delay_s": 5,
|
|
485
|
+
"retry_on": [
|
|
486
|
+
"lock-frontend",
|
|
487
|
+
"Unable to acquire the dpkg frontend lock",
|
|
488
|
+
"Temporary failure resolving",
|
|
489
|
+
"Could not resolve",
|
|
490
|
+
"Failed to fetch",
|
|
491
|
+
],
|
|
492
|
+
},
|
|
493
|
+
{"name": "user_exists", "cmd": f"id -u {user} >/dev/null 2>&1 || adduser --disabled-password --gecos '' {user}", "retries": 0},
|
|
494
|
+
{"name": "add_sudo", "cmd": f"usermod -aG sudo {user}", "retries": 0},
|
|
495
|
+
]
|
|
496
|
+
if password:
|
|
497
|
+
steps.append({"name": "set_password", "cmd": f"echo '{user}:{password}' | chpasswd", "retries": 0})
|
|
498
|
+
if ssh_key:
|
|
499
|
+
steps.append({
|
|
500
|
+
"name": "add_ssh_key",
|
|
501
|
+
"cmd": f"install -d -m 700 /home/{user}/.ssh && echo '{ssh_key}' >> /home/{user}/.ssh/authorized_keys && chown -R {user}:{user} /home/{user}/.ssh",
|
|
502
|
+
"retries": 0,
|
|
503
|
+
})
|
|
504
|
+
steps.extend(
|
|
505
|
+
[
|
|
506
|
+
{"name": "pip_upgrade", "cmd": "python3 -m pip install --upgrade pip", "retries": 0},
|
|
507
|
+
{"name": "install_portacode", "cmd": "python3 -m pip install --upgrade portacode", "retries": 0},
|
|
508
|
+
]
|
|
509
|
+
)
|
|
510
|
+
if include_portacode_connect:
|
|
511
|
+
steps.append({"name": "portacode_connect", "type": "portacode_connect", "timeout_s": 30})
|
|
512
|
+
return steps
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
def _get_storage_type(storages: Iterable[Dict[str, Any]], storage_name: str) -> str:
|
|
516
|
+
for entry in storages:
|
|
517
|
+
if entry.get("storage") == storage_name:
|
|
518
|
+
return entry.get("type", "")
|
|
519
|
+
return ""
|
|
520
|
+
|
|
521
|
+
|
|
522
|
+
def _validate_positive_number(value: Any, default: float) -> float:
|
|
523
|
+
try:
|
|
524
|
+
candidate = float(value)
|
|
525
|
+
if candidate > 0:
|
|
526
|
+
return candidate
|
|
527
|
+
except Exception:
|
|
528
|
+
pass
|
|
529
|
+
return float(default)
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
def _wait_for_task(proxmox: Any, node: str, upid: str) -> Tuple[Dict[str, Any], float]:
|
|
533
|
+
start = time.time()
|
|
534
|
+
while True:
|
|
535
|
+
status = proxmox.nodes(node).tasks(upid).status.get()
|
|
536
|
+
if status.get("status") == "stopped":
|
|
537
|
+
return status, time.time() - start
|
|
538
|
+
time.sleep(1)
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
def _list_running_managed(proxmox: Any, node: str) -> List[Tuple[str, Dict[str, Any]]]:
|
|
542
|
+
entries = []
|
|
543
|
+
for ct in proxmox.nodes(node).lxc.get():
|
|
544
|
+
if ct.get("status") != "running":
|
|
545
|
+
continue
|
|
546
|
+
vmid = str(ct.get("vmid"))
|
|
547
|
+
cfg = proxmox.nodes(node).lxc(vmid).config.get()
|
|
548
|
+
if cfg and MANAGED_MARKER in (cfg.get("description") or ""):
|
|
549
|
+
entries.append((vmid, cfg))
|
|
550
|
+
return entries
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
def _start_container(proxmox: Any, node: str, vmid: int) -> Tuple[Dict[str, Any], float]:
|
|
554
|
+
status = proxmox.nodes(node).lxc(vmid).status.current.get()
|
|
555
|
+
if status.get("status") == "running":
|
|
556
|
+
uptime = status.get("uptime", 0)
|
|
557
|
+
logger.info("Container %s already running (%ss)", vmid, uptime)
|
|
558
|
+
return status, 0.0
|
|
559
|
+
|
|
560
|
+
node_status = proxmox.nodes(node).status.get()
|
|
561
|
+
mem_total_mb = int(node_status.get("memory", {}).get("total", 0) // (1024**2))
|
|
562
|
+
cores_total = int(node_status.get("cpuinfo", {}).get("cores", 0))
|
|
563
|
+
|
|
564
|
+
running = _list_running_managed(proxmox, node)
|
|
565
|
+
used_mem_mb = sum(int(cfg.get("memory", 0)) for _, cfg in running)
|
|
566
|
+
used_cores = sum(int(cfg.get("cores", 0)) for _, cfg in running)
|
|
567
|
+
|
|
568
|
+
target_cfg = proxmox.nodes(node).lxc(vmid).config.get()
|
|
569
|
+
target_mem_mb = int(target_cfg.get("memory", 0))
|
|
570
|
+
target_cores = int(target_cfg.get("cores", 0))
|
|
571
|
+
|
|
572
|
+
if mem_total_mb and used_mem_mb + target_mem_mb > mem_total_mb:
|
|
573
|
+
raise RuntimeError("Not enough RAM to start this container safely.")
|
|
574
|
+
if cores_total and used_cores + target_cores > cores_total:
|
|
575
|
+
raise RuntimeError("Not enough CPU cores to start this container safely.")
|
|
576
|
+
|
|
577
|
+
upid = proxmox.nodes(node).lxc(vmid).status.start.post()
|
|
578
|
+
return _wait_for_task(proxmox, node, upid)
|
|
579
|
+
|
|
580
|
+
|
|
581
|
+
def _stop_container(proxmox: Any, node: str, vmid: int) -> Tuple[Dict[str, Any], float]:
|
|
582
|
+
status = proxmox.nodes(node).lxc(vmid).status.current.get()
|
|
583
|
+
if status.get("status") != "running":
|
|
584
|
+
return status, 0.0
|
|
585
|
+
upid = proxmox.nodes(node).lxc(vmid).status.stop.post()
|
|
586
|
+
return _wait_for_task(proxmox, node, upid)
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
def _delete_container(proxmox: Any, node: str, vmid: int) -> Tuple[Dict[str, Any], float]:
|
|
590
|
+
upid = proxmox.nodes(node).lxc(vmid).delete()
|
|
591
|
+
return _wait_for_task(proxmox, node, upid)
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
def _write_container_record(vmid: int, payload: Dict[str, Any]) -> None:
|
|
595
|
+
_ensure_containers_dir()
|
|
596
|
+
path = CONTAINERS_DIR / f"ct-{vmid}.json"
|
|
597
|
+
path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
|
598
|
+
_invalidate_managed_containers_cache()
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
def _read_container_record(vmid: int) -> Dict[str, Any]:
|
|
602
|
+
path = CONTAINERS_DIR / f"ct-{vmid}.json"
|
|
603
|
+
if not path.exists():
|
|
604
|
+
raise FileNotFoundError(f"Container record {path} missing")
|
|
605
|
+
return json.loads(path.read_text(encoding="utf-8"))
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
def _update_container_record(vmid: int, updates: Dict[str, Any]) -> None:
|
|
609
|
+
record = _read_container_record(vmid)
|
|
610
|
+
record.update(updates)
|
|
611
|
+
_write_container_record(vmid, record)
|
|
612
|
+
|
|
613
|
+
|
|
614
|
+
def _remove_container_record(vmid: int) -> None:
|
|
615
|
+
path = CONTAINERS_DIR / f"ct-{vmid}.json"
|
|
616
|
+
if path.exists():
|
|
617
|
+
path.unlink()
|
|
618
|
+
_invalidate_managed_containers_cache()
|
|
619
|
+
|
|
620
|
+
|
|
621
|
+
def _update_container_tunnel(vmid: int, tunnel: Optional[Dict[str, Any]]) -> None:
|
|
622
|
+
record = _read_container_record(vmid)
|
|
623
|
+
if tunnel:
|
|
624
|
+
record["tunnel"] = tunnel
|
|
625
|
+
else:
|
|
626
|
+
record.pop("tunnel", None)
|
|
627
|
+
_write_container_record(vmid, record)
|
|
628
|
+
|
|
629
|
+
|
|
630
|
+
def _ensure_cloudflare_token(config: Dict[str, Any]) -> str:
|
|
631
|
+
cloudflare = config.get("cloudflare") or {}
|
|
632
|
+
token = cloudflare.get("api_token")
|
|
633
|
+
if not token:
|
|
634
|
+
raise RuntimeError("Cloudflare API token is not configured.")
|
|
635
|
+
return token
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
def _launch_container_tunnel(proxmox: Any, node: str, vmid: int, tunnel: Dict[str, Any]) -> Dict[str, Any]:
|
|
639
|
+
port = int(tunnel.get("container_port") or 0)
|
|
640
|
+
if not port:
|
|
641
|
+
raise ValueError("container_port is required to create a tunnel.")
|
|
642
|
+
requested_hostname = tunnel.get("url") or None
|
|
643
|
+
protocol = (tunnel.get("protocol") or "http").lower()
|
|
644
|
+
ip_address = _resolve_container_ip(proxmox, node, vmid)
|
|
645
|
+
target_url = f"{protocol}://{ip_address}:{port}"
|
|
646
|
+
name = tunnel.get("name") or _build_tunnel_name(vmid, port)
|
|
647
|
+
_stop_cloudflare_process(name)
|
|
648
|
+
proc, assigned_url = _start_cloudflare_process(name, target_url, hostname=requested_hostname)
|
|
649
|
+
if not assigned_url:
|
|
650
|
+
raise RuntimeError("Failed to determine Cloudflare hostname for the tunnel.")
|
|
651
|
+
updated = {
|
|
652
|
+
"name": name,
|
|
653
|
+
"container_port": port,
|
|
654
|
+
"url": assigned_url,
|
|
655
|
+
"protocol": protocol,
|
|
656
|
+
"status": "running",
|
|
657
|
+
"pid": proc.pid,
|
|
658
|
+
"target_ip": ip_address,
|
|
659
|
+
"target_url": target_url,
|
|
660
|
+
"last_updated": datetime.utcnow().isoformat() + "Z",
|
|
661
|
+
}
|
|
662
|
+
_update_container_tunnel(vmid, updated)
|
|
663
|
+
return updated
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
def _stop_container_tunnel(vmid: int) -> None:
|
|
667
|
+
try:
|
|
668
|
+
record = _read_container_record(vmid)
|
|
669
|
+
except FileNotFoundError:
|
|
670
|
+
return
|
|
671
|
+
tunnel = record.get("tunnel")
|
|
672
|
+
if not tunnel:
|
|
673
|
+
return
|
|
674
|
+
name = tunnel.get("name") or _build_tunnel_name(vmid, int(tunnel.get("container_port") or 0))
|
|
675
|
+
stopped = _stop_cloudflare_process(name)
|
|
676
|
+
if not stopped and tunnel.get("status") == "stopped":
|
|
677
|
+
return
|
|
678
|
+
tunnel_update = {
|
|
679
|
+
**tunnel,
|
|
680
|
+
"status": "stopped",
|
|
681
|
+
"pid": None,
|
|
682
|
+
"last_updated": datetime.utcnow().isoformat() + "Z",
|
|
683
|
+
}
|
|
684
|
+
_update_container_tunnel(vmid, tunnel_update)
|
|
685
|
+
|
|
686
|
+
|
|
687
|
+
def _remove_container_tunnel_state(vmid: int) -> None:
|
|
688
|
+
_stop_container_tunnel(vmid)
|
|
689
|
+
_update_container_tunnel(vmid, None)
|
|
690
|
+
|
|
691
|
+
|
|
692
|
+
def _ensure_container_tunnel_running(proxmox: Any, node: str, vmid: int) -> None:
|
|
693
|
+
try:
|
|
694
|
+
record = _read_container_record(vmid)
|
|
695
|
+
except FileNotFoundError:
|
|
696
|
+
return
|
|
697
|
+
tunnel = record.get("tunnel")
|
|
698
|
+
if not tunnel:
|
|
699
|
+
return
|
|
700
|
+
_ensure_cloudflare_token(_load_config())
|
|
701
|
+
_launch_container_tunnel(proxmox, node, vmid, tunnel)
|
|
702
|
+
|
|
703
|
+
|
|
704
|
+
def _build_container_payload(message: Dict[str, Any], config: Dict[str, Any]) -> Dict[str, Any]:
|
|
705
|
+
templates = config.get("templates") or []
|
|
706
|
+
default_template = templates[0] if templates else ""
|
|
707
|
+
template = message.get("template") or default_template
|
|
708
|
+
if not template:
|
|
709
|
+
raise ValueError("Container template is required.")
|
|
710
|
+
|
|
711
|
+
bridge = config.get("network", {}).get("bridge", DEFAULT_BRIDGE)
|
|
712
|
+
hostname = (message.get("hostname") or "").strip()
|
|
713
|
+
disk_gib = int(max(round(_validate_positive_number(message.get("disk_gib") or message.get("disk"), 3)), 1))
|
|
714
|
+
ram_mib = int(max(round(_validate_positive_number(message.get("ram_mib") or message.get("ram"), 2048)), 1))
|
|
715
|
+
cpus = _validate_positive_number(message.get("cpus"), 0.2)
|
|
716
|
+
storage = message.get("storage") or config.get("default_storage") or ""
|
|
717
|
+
if not storage:
|
|
718
|
+
raise ValueError("Storage pool could not be determined.")
|
|
719
|
+
|
|
720
|
+
user, password, ssh_key = _get_provisioning_user_info(message)
|
|
721
|
+
|
|
722
|
+
payload = {
|
|
723
|
+
"template": template,
|
|
724
|
+
"storage": storage,
|
|
725
|
+
"disk_gib": disk_gib,
|
|
726
|
+
"ram_mib": ram_mib,
|
|
727
|
+
"cpus": cpus,
|
|
728
|
+
"hostname": hostname,
|
|
729
|
+
"net0": f"name=eth0,bridge={bridge},ip=dhcp",
|
|
730
|
+
"unprivileged": 1,
|
|
731
|
+
"swap_mb": 0,
|
|
732
|
+
"username": user,
|
|
733
|
+
"password": password,
|
|
734
|
+
"ssh_public_key": ssh_key,
|
|
735
|
+
"description": MANAGED_MARKER,
|
|
736
|
+
}
|
|
737
|
+
return payload
|
|
738
|
+
|
|
739
|
+
|
|
740
|
+
def _ensure_infra_configured() -> Dict[str, Any]:
|
|
741
|
+
config = _load_config()
|
|
742
|
+
if not config or not config.get("token_value"):
|
|
743
|
+
raise RuntimeError("Proxmox infrastructure is not configured.")
|
|
744
|
+
return config
|
|
745
|
+
|
|
746
|
+
|
|
747
|
+
def _get_node_from_config(config: Dict[str, Any]) -> str:
|
|
748
|
+
return config.get("node") or DEFAULT_NODE_NAME
|
|
749
|
+
|
|
750
|
+
|
|
751
|
+
def _parse_ctid(message: Dict[str, Any]) -> int:
|
|
752
|
+
for key in ("ctid", "vmid"):
|
|
753
|
+
value = message.get(key)
|
|
754
|
+
if value is not None:
|
|
755
|
+
try:
|
|
756
|
+
return int(str(value).strip())
|
|
757
|
+
except ValueError:
|
|
758
|
+
raise ValueError(f"{key} must be an integer") from None
|
|
759
|
+
raise ValueError("ctid is required")
|
|
760
|
+
|
|
761
|
+
|
|
762
|
+
def _ensure_container_managed(
|
|
763
|
+
proxmox: Any, node: str, vmid: int
|
|
764
|
+
) -> Tuple[Dict[str, Any], Dict[str, Any]]:
|
|
765
|
+
record = _read_container_record(vmid)
|
|
766
|
+
ct_cfg = proxmox.nodes(node).lxc(str(vmid)).config.get()
|
|
767
|
+
if not ct_cfg or MANAGED_MARKER not in (ct_cfg.get("description") or ""):
|
|
768
|
+
raise RuntimeError(f"Container {vmid} is not managed by Portacode.")
|
|
769
|
+
return record, ct_cfg
|
|
770
|
+
|
|
771
|
+
|
|
772
|
+
def _connect_proxmox(config: Dict[str, Any]) -> Any:
|
|
773
|
+
ProxmoxAPI = _ensure_proxmoxer()
|
|
774
|
+
return ProxmoxAPI(
|
|
775
|
+
config.get("host", DEFAULT_HOST),
|
|
776
|
+
user=config.get("user"),
|
|
777
|
+
token_name=config.get("token_name"),
|
|
778
|
+
token_value=config.get("token_value"),
|
|
779
|
+
verify_ssl=config.get("verify_ssl", False),
|
|
780
|
+
timeout=60,
|
|
781
|
+
)
|
|
782
|
+
|
|
783
|
+
|
|
784
|
+
def _run_pct(vmid: int, cmd: str, input_text: Optional[str] = None) -> Dict[str, Any]:
|
|
785
|
+
full = ["pct", "exec", str(vmid), "--", "bash", "-lc", cmd]
|
|
786
|
+
start = time.time()
|
|
787
|
+
proc = subprocess.run(full, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, input=input_text)
|
|
788
|
+
return {
|
|
789
|
+
"cmd": cmd,
|
|
790
|
+
"returncode": proc.returncode,
|
|
791
|
+
"stdout": proc.stdout.strip(),
|
|
792
|
+
"stderr": proc.stderr.strip(),
|
|
793
|
+
"elapsed_s": round(time.time() - start, 2),
|
|
794
|
+
}
|
|
795
|
+
|
|
796
|
+
|
|
797
|
+
def _run_pct_check(vmid: int, cmd: str) -> Dict[str, Any]:
|
|
798
|
+
res = _run_pct(vmid, cmd)
|
|
799
|
+
if res["returncode"] != 0:
|
|
800
|
+
raise RuntimeError(res.get("stderr") or res.get("stdout") or "command failed")
|
|
801
|
+
return res
|
|
802
|
+
|
|
803
|
+
|
|
804
|
+
def _run_pct_exec(vmid: int, command: Sequence[str]) -> subprocess.CompletedProcess[str]:
|
|
805
|
+
return _call_subprocess(["pct", "exec", str(vmid), "--", *command])
|
|
806
|
+
|
|
807
|
+
|
|
808
|
+
def _run_pct_exec_check(vmid: int, command: Sequence[str]) -> subprocess.CompletedProcess[str]:
|
|
809
|
+
res = _run_pct_exec(vmid, command)
|
|
810
|
+
if res.returncode != 0:
|
|
811
|
+
raise RuntimeError(res.stderr or res.stdout or f"pct exec {' '.join(command)} failed")
|
|
812
|
+
return res
|
|
813
|
+
|
|
814
|
+
|
|
815
|
+
def _run_pct_push(vmid: int, src: str, dest: str) -> subprocess.CompletedProcess[str]:
|
|
816
|
+
return _call_subprocess(["pct", "push", str(vmid), src, dest])
|
|
817
|
+
|
|
818
|
+
|
|
819
|
+
def _build_tunnel_name(vmid: int, port: int) -> str:
|
|
820
|
+
return f"portacode-ct{vmid}-{port}"
|
|
821
|
+
|
|
822
|
+
|
|
823
|
+
def _get_cloudflared_binary() -> str:
|
|
824
|
+
binary = shutil.which("cloudflared")
|
|
825
|
+
if not binary:
|
|
826
|
+
raise RuntimeError(
|
|
827
|
+
"cloudflared is required for Cloudflare tunnels but was not found on PATH. "
|
|
828
|
+
"Install cloudflared and run 'cloudflared tunnel login' before creating tunnels."
|
|
829
|
+
)
|
|
830
|
+
return binary
|
|
831
|
+
|
|
832
|
+
|
|
833
|
+
def _drain_stream(stream: Optional[Any]) -> None:
|
|
834
|
+
if stream is None:
|
|
835
|
+
return
|
|
836
|
+
try:
|
|
837
|
+
for _ in iter(stream.readline, ""):
|
|
838
|
+
continue
|
|
839
|
+
except Exception:
|
|
840
|
+
pass
|
|
841
|
+
finally:
|
|
842
|
+
try:
|
|
843
|
+
stream.close()
|
|
844
|
+
except Exception:
|
|
845
|
+
pass
|
|
846
|
+
|
|
847
|
+
|
|
848
|
+
def _await_quick_tunnel_url(proc: subprocess.Popen, timeout: float = 15.0) -> Optional[str]:
|
|
849
|
+
if not proc.stdout:
|
|
850
|
+
return None
|
|
851
|
+
cf_re = re.compile(r"https://[A-Za-z0-9\-.]+\.cfargotunnel\.com")
|
|
852
|
+
deadline = time.time() + timeout
|
|
853
|
+
while time.time() < deadline:
|
|
854
|
+
ready, _, _ = select.select([proc.stdout], [], [], 1)
|
|
855
|
+
if not ready:
|
|
856
|
+
continue
|
|
857
|
+
line = proc.stdout.readline()
|
|
858
|
+
if not line:
|
|
859
|
+
continue
|
|
860
|
+
match = cf_re.search(line)
|
|
861
|
+
if match:
|
|
862
|
+
return match.group(0)
|
|
863
|
+
return None
|
|
864
|
+
|
|
865
|
+
|
|
866
|
+
def _start_cloudflare_process(name: str, target_url: str, hostname: Optional[str] = None) -> Tuple[subprocess.Popen, Optional[str]]:
|
|
867
|
+
binary = _get_cloudflared_binary()
|
|
868
|
+
cmd = [
|
|
869
|
+
binary,
|
|
870
|
+
"tunnel",
|
|
871
|
+
"--url",
|
|
872
|
+
target_url,
|
|
873
|
+
"--no-autoupdate",
|
|
874
|
+
]
|
|
875
|
+
if hostname:
|
|
876
|
+
cmd.extend(["--hostname", hostname])
|
|
877
|
+
stdout_target = subprocess.DEVNULL
|
|
878
|
+
else:
|
|
879
|
+
stdout_target = subprocess.PIPE
|
|
880
|
+
proc = subprocess.Popen(
|
|
881
|
+
cmd,
|
|
882
|
+
stdout=stdout_target,
|
|
883
|
+
stderr=subprocess.PIPE,
|
|
884
|
+
text=True,
|
|
885
|
+
)
|
|
886
|
+
with _CLOUDFLARE_TUNNELS_LOCK:
|
|
887
|
+
_CLOUDFLARE_TUNNEL_PROCESSES[name] = proc
|
|
888
|
+
assigned_url = hostname
|
|
889
|
+
if not hostname:
|
|
890
|
+
assigned_url = _await_quick_tunnel_url(proc)
|
|
891
|
+
threading.Thread(target=_drain_stream, args=(proc.stdout,), daemon=True).start()
|
|
892
|
+
threading.Thread(target=_drain_stream, args=(proc.stderr,), daemon=True).start()
|
|
893
|
+
return proc, assigned_url
|
|
894
|
+
|
|
895
|
+
|
|
896
|
+
def _stop_cloudflare_process(name: str) -> bool:
|
|
897
|
+
with _CLOUDFLARE_TUNNELS_LOCK:
|
|
898
|
+
proc = _CLOUDFLARE_TUNNEL_PROCESSES.pop(name, None)
|
|
899
|
+
if not proc:
|
|
900
|
+
return False
|
|
901
|
+
try:
|
|
902
|
+
proc.terminate()
|
|
903
|
+
proc.wait(timeout=5)
|
|
904
|
+
except subprocess.TimeoutExpired:
|
|
905
|
+
proc.kill()
|
|
906
|
+
proc.wait()
|
|
907
|
+
return True
|
|
908
|
+
|
|
909
|
+
|
|
910
|
+
def _resolve_container_ip(proxmox: Any, node: str, vmid: int) -> str:
|
|
911
|
+
status = proxmox.nodes(node).lxc(vmid).status.current.get()
|
|
912
|
+
if status:
|
|
913
|
+
ip_field = status.get("ip")
|
|
914
|
+
if isinstance(ip_field, list):
|
|
915
|
+
for entry in ip_field:
|
|
916
|
+
if isinstance(entry, str) and "." in entry:
|
|
917
|
+
return entry.split("/")[0]
|
|
918
|
+
elif isinstance(ip_field, str) and "." in ip_field:
|
|
919
|
+
return ip_field.split("/")[0]
|
|
920
|
+
res = _run_pct_exec(vmid, ["ip", "-4", "-o", "addr", "show", "eth0"])
|
|
921
|
+
line = res.stdout.splitlines()[0] if res.stdout else ""
|
|
922
|
+
parts = line.split()
|
|
923
|
+
if len(parts) >= 4:
|
|
924
|
+
addr = parts[3]
|
|
925
|
+
return addr.split("/")[0]
|
|
926
|
+
raise RuntimeError("Unable to determine container IP address")
|
|
927
|
+
|
|
928
|
+
|
|
929
|
+
def _push_bytes_to_container(
|
|
930
|
+
vmid: int, user: str, path: str, data: bytes, mode: int = 0o600
|
|
931
|
+
) -> None:
|
|
932
|
+
logger.debug("Preparing to push %d bytes to container vmid=%s path=%s for user=%s", len(data), vmid, path, user)
|
|
933
|
+
tmp_path: Optional[str] = None
|
|
934
|
+
try:
|
|
935
|
+
parent = Path(path).parent
|
|
936
|
+
parent_str = parent.as_posix()
|
|
937
|
+
if parent_str not in {"", ".", "/"}:
|
|
938
|
+
_run_pct_exec_check(vmid, ["mkdir", "-p", parent_str])
|
|
939
|
+
_run_pct_exec_check(vmid, ["chown", "-R", f"{user}:{user}", parent_str])
|
|
940
|
+
|
|
941
|
+
with tempfile.NamedTemporaryFile(delete=False) as tmp:
|
|
942
|
+
tmp.write(data)
|
|
943
|
+
tmp.flush()
|
|
944
|
+
os.fsync(tmp.fileno())
|
|
945
|
+
tmp_path = tmp.name
|
|
946
|
+
|
|
947
|
+
push_res = _run_pct_push(vmid, tmp_path, path)
|
|
948
|
+
if push_res.returncode != 0:
|
|
949
|
+
raise RuntimeError(push_res.stderr or push_res.stdout or f"pct push returned {push_res.returncode}")
|
|
950
|
+
|
|
951
|
+
_run_pct_exec_check(vmid, ["chown", f"{user}:{user}", path])
|
|
952
|
+
_run_pct_exec_check(vmid, ["chmod", format(mode, "o"), path])
|
|
953
|
+
logger.debug("Successfully pushed %d bytes to vmid=%s path=%s", len(data), vmid, path)
|
|
954
|
+
except Exception as exc:
|
|
955
|
+
logger.error("Failed to write to container vmid=%s path=%s for user=%s: %s", vmid, path, user, exc)
|
|
956
|
+
raise
|
|
957
|
+
finally:
|
|
958
|
+
if tmp_path:
|
|
959
|
+
try:
|
|
960
|
+
os.remove(tmp_path)
|
|
961
|
+
except OSError as cleanup_exc:
|
|
962
|
+
logger.warning("Failed to remove temporary file %s: %s", tmp_path, cleanup_exc)
|
|
963
|
+
|
|
964
|
+
|
|
965
|
+
def _resolve_portacode_key_dir(vmid: int, user: str) -> str:
|
|
966
|
+
data_dir_cmd = f"su - {user} -c 'echo -n ${{XDG_DATA_HOME:-$HOME/.local/share}}'"
|
|
967
|
+
data_home = _run_pct_check(vmid, data_dir_cmd)["stdout"].strip()
|
|
968
|
+
portacode_dir = f"{data_home}/portacode"
|
|
969
|
+
_run_pct_exec_check(vmid, ["mkdir", "-p", portacode_dir])
|
|
970
|
+
_run_pct_exec_check(vmid, ["chown", "-R", f"{user}:{user}", portacode_dir])
|
|
971
|
+
return f"{portacode_dir}/keys"
|
|
972
|
+
|
|
973
|
+
|
|
974
|
+
def _deploy_device_keypair(vmid: int, user: str, private_key: str, public_key: str) -> None:
|
|
975
|
+
key_dir = _resolve_portacode_key_dir(vmid, user)
|
|
976
|
+
priv_path = f"{key_dir}/id_portacode"
|
|
977
|
+
pub_path = f"{key_dir}/id_portacode.pub"
|
|
978
|
+
_push_bytes_to_container(vmid, user, priv_path, private_key.encode(), mode=0o600)
|
|
979
|
+
_push_bytes_to_container(vmid, user, pub_path, public_key.encode(), mode=0o644)
|
|
980
|
+
|
|
981
|
+
|
|
982
|
+
def _portacode_connect_and_read_key(vmid: int, user: str, timeout_s: int = 10) -> Dict[str, Any]:
|
|
983
|
+
cmd = ["pct", "exec", str(vmid), "--", "bash", "-lc", f"su - {user} -c 'portacode connect'"]
|
|
984
|
+
proc = subprocess.Popen(cmd, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
985
|
+
start = time.time()
|
|
986
|
+
|
|
987
|
+
data_dir_cmd = f"su - {user} -c 'echo -n ${{XDG_DATA_HOME:-$HOME/.local/share}}'"
|
|
988
|
+
data_dir = _run_pct_check(vmid, data_dir_cmd)["stdout"].strip()
|
|
989
|
+
key_dir = f"{data_dir}/portacode/keys"
|
|
990
|
+
pub_path = f"{key_dir}/id_portacode.pub"
|
|
991
|
+
priv_path = f"{key_dir}/id_portacode"
|
|
992
|
+
|
|
993
|
+
def file_size(path: str) -> Optional[int]:
|
|
994
|
+
stat_cmd = f"su - {user} -c 'test -s {path} && stat -c %s {path}'"
|
|
995
|
+
res = _run_pct(vmid, stat_cmd)
|
|
996
|
+
if res["returncode"] != 0:
|
|
997
|
+
return None
|
|
998
|
+
try:
|
|
999
|
+
return int(res["stdout"].strip())
|
|
1000
|
+
except ValueError:
|
|
1001
|
+
return None
|
|
1002
|
+
|
|
1003
|
+
last_pub = last_priv = None
|
|
1004
|
+
stable = 0
|
|
1005
|
+
history: List[Dict[str, Any]] = []
|
|
1006
|
+
|
|
1007
|
+
process_exited = False
|
|
1008
|
+
exit_out = exit_err = ""
|
|
1009
|
+
while time.time() - start < timeout_s:
|
|
1010
|
+
if proc.poll() is not None:
|
|
1011
|
+
process_exited = True
|
|
1012
|
+
exit_out, exit_err = proc.communicate(timeout=1)
|
|
1013
|
+
history.append(
|
|
1014
|
+
{
|
|
1015
|
+
"timestamp_s": round(time.time() - start, 2),
|
|
1016
|
+
"status": "process_exited",
|
|
1017
|
+
"returncode": proc.returncode,
|
|
1018
|
+
}
|
|
1019
|
+
)
|
|
1020
|
+
break
|
|
1021
|
+
pub_size = file_size(pub_path)
|
|
1022
|
+
priv_size = file_size(priv_path)
|
|
1023
|
+
if pub_size and priv_size:
|
|
1024
|
+
if pub_size == last_pub and priv_size == last_priv:
|
|
1025
|
+
stable += 1
|
|
1026
|
+
else:
|
|
1027
|
+
stable = 0
|
|
1028
|
+
last_pub, last_priv = pub_size, priv_size
|
|
1029
|
+
if stable >= 1:
|
|
1030
|
+
history.append(
|
|
1031
|
+
{
|
|
1032
|
+
"timestamp_s": round(time.time() - start, 2),
|
|
1033
|
+
"pub_size": pub_size,
|
|
1034
|
+
"priv_size": priv_size,
|
|
1035
|
+
"stable": stable,
|
|
1036
|
+
}
|
|
1037
|
+
)
|
|
1038
|
+
break
|
|
1039
|
+
history.append(
|
|
1040
|
+
{
|
|
1041
|
+
"timestamp_s": round(time.time() - start, 2),
|
|
1042
|
+
"pub_size": pub_size,
|
|
1043
|
+
"priv_size": priv_size,
|
|
1044
|
+
"stable": stable,
|
|
1045
|
+
}
|
|
1046
|
+
)
|
|
1047
|
+
time.sleep(1)
|
|
1048
|
+
|
|
1049
|
+
final_pub = file_size(pub_path)
|
|
1050
|
+
final_priv = file_size(priv_path)
|
|
1051
|
+
if final_pub and final_priv:
|
|
1052
|
+
key_res = _run_pct(vmid, f"su - {user} -c 'cat {pub_path}'")
|
|
1053
|
+
if not process_exited:
|
|
1054
|
+
proc.terminate()
|
|
1055
|
+
try:
|
|
1056
|
+
proc.wait(timeout=3)
|
|
1057
|
+
except subprocess.TimeoutExpired:
|
|
1058
|
+
proc.kill()
|
|
1059
|
+
return {
|
|
1060
|
+
"ok": True,
|
|
1061
|
+
"public_key": key_res["stdout"].strip(),
|
|
1062
|
+
"history": history,
|
|
1063
|
+
}
|
|
1064
|
+
|
|
1065
|
+
if not process_exited:
|
|
1066
|
+
proc.terminate()
|
|
1067
|
+
try:
|
|
1068
|
+
proc.wait(timeout=3)
|
|
1069
|
+
except subprocess.TimeoutExpired:
|
|
1070
|
+
proc.kill()
|
|
1071
|
+
exit_out, exit_err = proc.communicate(timeout=1)
|
|
1072
|
+
history.append(
|
|
1073
|
+
{
|
|
1074
|
+
"timestamp_s": round(time.time() - start, 2),
|
|
1075
|
+
"status": "timeout_waiting_for_keys",
|
|
1076
|
+
}
|
|
1077
|
+
)
|
|
1078
|
+
return {
|
|
1079
|
+
"ok": False,
|
|
1080
|
+
"error": "timed out waiting for portacode key files",
|
|
1081
|
+
"stdout": (exit_out or "").strip(),
|
|
1082
|
+
"stderr": (exit_err or "").strip(),
|
|
1083
|
+
"history": history,
|
|
1084
|
+
}
|
|
1085
|
+
|
|
1086
|
+
proc.terminate()
|
|
1087
|
+
try:
|
|
1088
|
+
proc.wait(timeout=3)
|
|
1089
|
+
except subprocess.TimeoutExpired:
|
|
1090
|
+
proc.kill()
|
|
1091
|
+
|
|
1092
|
+
key_res = _run_pct(vmid, f"su - {user} -c 'cat {pub_path}'")
|
|
1093
|
+
return {
|
|
1094
|
+
"ok": True,
|
|
1095
|
+
"public_key": key_res["stdout"].strip(),
|
|
1096
|
+
"history": history,
|
|
1097
|
+
}
|
|
1098
|
+
|
|
1099
|
+
|
|
1100
|
+
def _summarize_error(res: Dict[str, Any]) -> str:
|
|
1101
|
+
text = f"{res.get('stdout','')}\n{res.get('stderr','')}"
|
|
1102
|
+
if "No space left on device" in text:
|
|
1103
|
+
return "Disk full inside container; increase rootfs or clean apt cache."
|
|
1104
|
+
if "Unable to acquire the dpkg frontend lock" in text or "lock-frontend" in text:
|
|
1105
|
+
return "Another apt/dpkg process is running; retry after it finishes."
|
|
1106
|
+
if "Temporary failure resolving" in text or "Could not resolve" in text:
|
|
1107
|
+
return "DNS/network resolution failed inside container."
|
|
1108
|
+
if "Failed to fetch" in text:
|
|
1109
|
+
return "Package repo fetch failed; check network and apt sources."
|
|
1110
|
+
return "Command failed; see stdout/stderr for details."
|
|
1111
|
+
|
|
1112
|
+
|
|
1113
|
+
def _run_setup_steps(
|
|
1114
|
+
vmid: int,
|
|
1115
|
+
steps: List[Dict[str, Any]],
|
|
1116
|
+
user: str,
|
|
1117
|
+
progress_callback: Optional[ProgressCallback] = None,
|
|
1118
|
+
start_index: int = 1,
|
|
1119
|
+
total_steps: Optional[int] = None,
|
|
1120
|
+
) -> Tuple[List[Dict[str, Any]], bool]:
|
|
1121
|
+
results: List[Dict[str, Any]] = []
|
|
1122
|
+
computed_total = total_steps if total_steps is not None else start_index + len(steps) - 1
|
|
1123
|
+
for offset, step in enumerate(steps):
|
|
1124
|
+
step_index = start_index + offset
|
|
1125
|
+
if progress_callback:
|
|
1126
|
+
progress_callback(step_index, computed_total, step, "in_progress", None)
|
|
1127
|
+
|
|
1128
|
+
if step.get("type") == "portacode_connect":
|
|
1129
|
+
res = _portacode_connect_and_read_key(vmid, user, timeout_s=step.get("timeout_s", 10))
|
|
1130
|
+
res["name"] = step["name"]
|
|
1131
|
+
results.append(res)
|
|
1132
|
+
if not res.get("ok"):
|
|
1133
|
+
if progress_callback:
|
|
1134
|
+
progress_callback(step_index, computed_total, step, "failed", res)
|
|
1135
|
+
return results, False
|
|
1136
|
+
if progress_callback:
|
|
1137
|
+
progress_callback(step_index, computed_total, step, "completed", res)
|
|
1138
|
+
continue
|
|
1139
|
+
|
|
1140
|
+
attempts = 0
|
|
1141
|
+
retry_on = step.get("retry_on", [])
|
|
1142
|
+
max_attempts = step.get("retries", 0) + 1
|
|
1143
|
+
while True:
|
|
1144
|
+
attempts += 1
|
|
1145
|
+
res = _run_pct(vmid, step["cmd"])
|
|
1146
|
+
res["name"] = step["name"]
|
|
1147
|
+
res["attempt"] = attempts
|
|
1148
|
+
if res["returncode"] != 0:
|
|
1149
|
+
res["error_summary"] = _summarize_error(res)
|
|
1150
|
+
results.append(res)
|
|
1151
|
+
if res["returncode"] == 0:
|
|
1152
|
+
if progress_callback:
|
|
1153
|
+
progress_callback(step_index, computed_total, step, "completed", res)
|
|
1154
|
+
break
|
|
1155
|
+
|
|
1156
|
+
will_retry = False
|
|
1157
|
+
if attempts < max_attempts and retry_on:
|
|
1158
|
+
stderr_stdout = (res.get("stderr", "") + res.get("stdout", ""))
|
|
1159
|
+
if any(tok in stderr_stdout for tok in retry_on):
|
|
1160
|
+
will_retry = True
|
|
1161
|
+
|
|
1162
|
+
if progress_callback:
|
|
1163
|
+
status = "retrying" if will_retry else "failed"
|
|
1164
|
+
progress_callback(step_index, computed_total, step, status, res)
|
|
1165
|
+
|
|
1166
|
+
if will_retry:
|
|
1167
|
+
time.sleep(step.get("retry_delay_s", 3))
|
|
1168
|
+
continue
|
|
1169
|
+
|
|
1170
|
+
return results, False
|
|
1171
|
+
return results, True
|
|
1172
|
+
|
|
1173
|
+
|
|
1174
|
+
def _bootstrap_portacode(
|
|
1175
|
+
vmid: int,
|
|
1176
|
+
user: str,
|
|
1177
|
+
password: str,
|
|
1178
|
+
ssh_key: str,
|
|
1179
|
+
steps: Optional[List[Dict[str, Any]]] = None,
|
|
1180
|
+
progress_callback: Optional[ProgressCallback] = None,
|
|
1181
|
+
start_index: int = 1,
|
|
1182
|
+
total_steps: Optional[int] = None,
|
|
1183
|
+
default_public_key: Optional[str] = None,
|
|
1184
|
+
) -> Tuple[str, List[Dict[str, Any]]]:
|
|
1185
|
+
actual_steps = steps if steps is not None else _build_bootstrap_steps(user, password, ssh_key)
|
|
1186
|
+
results, ok = _run_setup_steps(
|
|
1187
|
+
vmid,
|
|
1188
|
+
actual_steps,
|
|
1189
|
+
user,
|
|
1190
|
+
progress_callback=progress_callback,
|
|
1191
|
+
start_index=start_index,
|
|
1192
|
+
total_steps=total_steps,
|
|
1193
|
+
)
|
|
1194
|
+
if not ok:
|
|
1195
|
+
details = results[-1] if results else {}
|
|
1196
|
+
summary = details.get("error_summary") or details.get("stderr") or details.get("stdout") or details.get("name")
|
|
1197
|
+
history = details.get("history")
|
|
1198
|
+
history_snippet = ""
|
|
1199
|
+
if isinstance(history, list) and history:
|
|
1200
|
+
history_snippet = f" history={history[-3:]}"
|
|
1201
|
+
command = details.get("cmd")
|
|
1202
|
+
command_text = ""
|
|
1203
|
+
if command:
|
|
1204
|
+
if isinstance(command, (list, tuple)):
|
|
1205
|
+
command_text = shlex.join(str(entry) for entry in command)
|
|
1206
|
+
else:
|
|
1207
|
+
command_text = str(command)
|
|
1208
|
+
command_suffix = f" command={command_text}" if command_text else ""
|
|
1209
|
+
if summary:
|
|
1210
|
+
logger.warning(
|
|
1211
|
+
"Portacode bootstrap failure summary=%s%s%s",
|
|
1212
|
+
summary,
|
|
1213
|
+
f" history_len={len(history)}" if history else "",
|
|
1214
|
+
f" command={command_text}" if command_text else "",
|
|
1215
|
+
)
|
|
1216
|
+
raise RuntimeError(
|
|
1217
|
+
f"Portacode bootstrap steps failed: {summary}{history_snippet}{command_suffix}"
|
|
1218
|
+
)
|
|
1219
|
+
raise RuntimeError("Portacode bootstrap steps failed.")
|
|
1220
|
+
key_step = next((entry for entry in results if entry.get("name") == "portacode_connect"), None)
|
|
1221
|
+
public_key = key_step.get("public_key") if key_step else default_public_key
|
|
1222
|
+
if not public_key:
|
|
1223
|
+
raise RuntimeError("Portacode connect did not return a public key.")
|
|
1224
|
+
return public_key, results
|
|
1225
|
+
|
|
1226
|
+
|
|
1227
|
+
def _build_cloudflare_snapshot(cloudflare_config: Dict[str, Any] | None) -> Dict[str, Any]:
|
|
1228
|
+
if not cloudflare_config:
|
|
1229
|
+
return {"configured": False}
|
|
1230
|
+
return {"configured": bool(cloudflare_config.get("api_token"))}
|
|
1231
|
+
|
|
1232
|
+
|
|
1233
|
+
def build_snapshot(config: Dict[str, Any]) -> Dict[str, Any]:
|
|
1234
|
+
network = config.get("network", {})
|
|
1235
|
+
base_network = {
|
|
1236
|
+
"applied": network.get("applied", False),
|
|
1237
|
+
"message": network.get("message"),
|
|
1238
|
+
"bridge": network.get("bridge", DEFAULT_BRIDGE),
|
|
1239
|
+
}
|
|
1240
|
+
cloudflare_snapshot = _build_cloudflare_snapshot(config.get("cloudflare"))
|
|
1241
|
+
if not config:
|
|
1242
|
+
return {
|
|
1243
|
+
"configured": False,
|
|
1244
|
+
"network": base_network,
|
|
1245
|
+
"cloudflare": cloudflare_snapshot,
|
|
1246
|
+
}
|
|
1247
|
+
return {
|
|
1248
|
+
"configured": True,
|
|
1249
|
+
"host": config.get("host"),
|
|
1250
|
+
"node": config.get("node"),
|
|
1251
|
+
"user": config.get("user"),
|
|
1252
|
+
"token_name": config.get("token_name"),
|
|
1253
|
+
"default_storage": config.get("default_storage"),
|
|
1254
|
+
"templates": config.get("templates") or [],
|
|
1255
|
+
"last_verified": config.get("last_verified"),
|
|
1256
|
+
"network": base_network,
|
|
1257
|
+
"cloudflare": cloudflare_snapshot,
|
|
1258
|
+
}
|
|
1259
|
+
|
|
1260
|
+
|
|
1261
|
+
def _resolve_proxmox_credentials(
|
|
1262
|
+
token_identifier: Optional[str],
|
|
1263
|
+
token_value: Optional[str],
|
|
1264
|
+
existing: Dict[str, Any],
|
|
1265
|
+
) -> Tuple[str, str, str]:
|
|
1266
|
+
if token_identifier:
|
|
1267
|
+
if not token_value:
|
|
1268
|
+
raise ValueError("token_value is required when providing a new token_identifier")
|
|
1269
|
+
user, token_name = _parse_token(token_identifier)
|
|
1270
|
+
return user, token_name, token_value
|
|
1271
|
+
if existing and existing.get("user") and existing.get("token_name") and existing.get("token_value"):
|
|
1272
|
+
return existing["user"], existing["token_name"], existing["token_value"]
|
|
1273
|
+
raise ValueError("Proxmox token identifier and value are required when no existing configuration is available")
|
|
1274
|
+
|
|
1275
|
+
|
|
1276
|
+
def _build_cloudflare_config(existing: Dict[str, Any], api_token: Optional[str]) -> Dict[str, Any]:
|
|
1277
|
+
cloudflare: Dict[str, Any] = dict(existing.get("cloudflare", {}) or {})
|
|
1278
|
+
if api_token:
|
|
1279
|
+
cloudflare["api_token"] = api_token
|
|
1280
|
+
if cloudflare.get("api_token"):
|
|
1281
|
+
cloudflare["configured"] = True
|
|
1282
|
+
elif "configured" in cloudflare:
|
|
1283
|
+
cloudflare.pop("configured", None)
|
|
1284
|
+
return cloudflare
|
|
1285
|
+
|
|
1286
|
+
|
|
1287
|
+
def configure_infrastructure(
|
|
1288
|
+
token_identifier: Optional[str] = None,
|
|
1289
|
+
token_value: Optional[str] = None,
|
|
1290
|
+
verify_ssl: Optional[bool] = None,
|
|
1291
|
+
cloudflare_api_token: Optional[str] = None,
|
|
1292
|
+
) -> Dict[str, Any]:
|
|
1293
|
+
ProxmoxAPI = _ensure_proxmoxer()
|
|
1294
|
+
existing = _load_config()
|
|
1295
|
+
user, token_name, resolved_token_value = _resolve_proxmox_credentials(
|
|
1296
|
+
token_identifier, token_value, existing
|
|
1297
|
+
)
|
|
1298
|
+
actual_verify_ssl = verify_ssl if verify_ssl is not None else existing.get("verify_ssl", False)
|
|
1299
|
+
client = ProxmoxAPI(
|
|
1300
|
+
DEFAULT_HOST,
|
|
1301
|
+
user=user,
|
|
1302
|
+
token_name=token_name,
|
|
1303
|
+
token_value=resolved_token_value,
|
|
1304
|
+
verify_ssl=actual_verify_ssl,
|
|
1305
|
+
timeout=30,
|
|
1306
|
+
)
|
|
1307
|
+
node = _pick_node(client)
|
|
1308
|
+
status = client.nodes(node).status.get()
|
|
1309
|
+
storages = client.nodes(node).storage.get()
|
|
1310
|
+
default_storage = _pick_storage(storages)
|
|
1311
|
+
templates = _list_templates(client, node, storages)
|
|
1312
|
+
network = dict(existing.get("network", {}) or {})
|
|
1313
|
+
_ensure_cloudflared_installed()
|
|
1314
|
+
if not network.get("applied"):
|
|
1315
|
+
try:
|
|
1316
|
+
network = _ensure_bridge()
|
|
1317
|
+
# Wait for network convergence before validating connectivity
|
|
1318
|
+
time.sleep(2)
|
|
1319
|
+
if not _verify_connectivity():
|
|
1320
|
+
raise RuntimeError("Connectivity check failed; bridge reverted")
|
|
1321
|
+
network["health"] = "healthy"
|
|
1322
|
+
except Exception as exc:
|
|
1323
|
+
logger.warning("Bridge setup failed; reverting previous changes: %s", exc)
|
|
1324
|
+
_revert_bridge()
|
|
1325
|
+
raise
|
|
1326
|
+
config = {
|
|
1327
|
+
"host": DEFAULT_HOST,
|
|
1328
|
+
"node": node,
|
|
1329
|
+
"user": user,
|
|
1330
|
+
"token_name": token_name,
|
|
1331
|
+
"token_value": resolved_token_value,
|
|
1332
|
+
"verify_ssl": actual_verify_ssl,
|
|
1333
|
+
"default_storage": default_storage,
|
|
1334
|
+
"templates": templates,
|
|
1335
|
+
"last_verified": datetime.utcnow().isoformat() + "Z",
|
|
1336
|
+
"network": network,
|
|
1337
|
+
"node_status": status,
|
|
1338
|
+
}
|
|
1339
|
+
cloudflare = _build_cloudflare_config(existing, cloudflare_api_token)
|
|
1340
|
+
if cloudflare:
|
|
1341
|
+
config["cloudflare"] = cloudflare
|
|
1342
|
+
_save_config(config)
|
|
1343
|
+
snapshot = build_snapshot(config)
|
|
1344
|
+
snapshot["node_status"] = status
|
|
1345
|
+
snapshot["managed_containers"] = _get_managed_containers_summary(force=True)
|
|
1346
|
+
return snapshot
|
|
1347
|
+
|
|
1348
|
+
|
|
1349
|
+
def get_infra_snapshot() -> Dict[str, Any]:
|
|
1350
|
+
config = _load_config()
|
|
1351
|
+
snapshot = build_snapshot(config)
|
|
1352
|
+
if config.get("node_status"):
|
|
1353
|
+
snapshot["node_status"] = config["node_status"]
|
|
1354
|
+
snapshot["managed_containers"] = _get_managed_containers_summary()
|
|
1355
|
+
return snapshot
|
|
1356
|
+
|
|
1357
|
+
|
|
1358
|
+
def revert_infrastructure() -> Dict[str, Any]:
|
|
1359
|
+
_revert_bridge()
|
|
1360
|
+
if CONFIG_PATH.exists():
|
|
1361
|
+
CONFIG_PATH.unlink()
|
|
1362
|
+
snapshot = build_snapshot({})
|
|
1363
|
+
snapshot["network"] = snapshot.get("network", {})
|
|
1364
|
+
snapshot["network"]["applied"] = False
|
|
1365
|
+
snapshot["network"]["message"] = "Reverted to previous network state"
|
|
1366
|
+
snapshot["network"]["bridge"] = DEFAULT_BRIDGE
|
|
1367
|
+
snapshot["managed_containers"] = _get_managed_containers_summary(force=True)
|
|
1368
|
+
return snapshot
|
|
1369
|
+
|
|
1370
|
+
|
|
1371
|
+
def _allocate_vmid(proxmox: Any) -> int:
|
|
1372
|
+
return int(proxmox.cluster.nextid.get())
|
|
1373
|
+
|
|
1374
|
+
|
|
1375
|
+
def _instantiate_container(proxmox: Any, node: str, payload: Dict[str, Any]) -> Tuple[int, float]:
|
|
1376
|
+
from proxmoxer.core import ResourceException
|
|
1377
|
+
|
|
1378
|
+
storage_type = _get_storage_type(proxmox.nodes(node).storage.get(), payload["storage"])
|
|
1379
|
+
rootfs = _format_rootfs(payload["storage"], payload["disk_gib"], storage_type)
|
|
1380
|
+
vmid = _allocate_vmid(proxmox)
|
|
1381
|
+
if not payload.get("hostname"):
|
|
1382
|
+
payload["hostname"] = f"ct{vmid}"
|
|
1383
|
+
try:
|
|
1384
|
+
upid = proxmox.nodes(node).lxc.create(
|
|
1385
|
+
vmid=vmid,
|
|
1386
|
+
hostname=payload["hostname"],
|
|
1387
|
+
ostemplate=payload["template"],
|
|
1388
|
+
rootfs=rootfs,
|
|
1389
|
+
memory=int(payload["ram_mib"]),
|
|
1390
|
+
swap=int(payload.get("swap_mb", 0)),
|
|
1391
|
+
cores=max(int(payload.get("cores", 1)), 1),
|
|
1392
|
+
cpuunits=int(payload.get("cpuunits", 256)),
|
|
1393
|
+
net0=payload["net0"],
|
|
1394
|
+
unprivileged=int(payload.get("unprivileged", 1)),
|
|
1395
|
+
description=payload.get("description", MANAGED_MARKER),
|
|
1396
|
+
password=payload.get("password") or None,
|
|
1397
|
+
ssh_public_keys=payload.get("ssh_public_key") or None,
|
|
1398
|
+
)
|
|
1399
|
+
status, elapsed = _wait_for_task(proxmox, node, upid)
|
|
1400
|
+
return vmid, elapsed
|
|
1401
|
+
except ResourceException as exc:
|
|
1402
|
+
raise RuntimeError(f"Failed to create container: {exc}") from exc
|
|
1403
|
+
|
|
1404
|
+
|
|
1405
|
+
class CreateProxmoxContainerHandler(SyncHandler):
|
|
1406
|
+
"""Provision a new managed LXC container via the Proxmox API."""
|
|
1407
|
+
|
|
1408
|
+
@property
|
|
1409
|
+
def command_name(self) -> str:
|
|
1410
|
+
return "create_proxmox_container"
|
|
1411
|
+
|
|
1412
|
+
def execute(self, message: Dict[str, Any]) -> Dict[str, Any]:
|
|
1413
|
+
logger.info("create_proxmox_container command received")
|
|
1414
|
+
request_id = message.get("request_id")
|
|
1415
|
+
device_id = message.get("device_id")
|
|
1416
|
+
device_public_key = (message.get("device_public_key") or "").strip()
|
|
1417
|
+
device_private_key = (message.get("device_private_key") or "").strip()
|
|
1418
|
+
has_device_keypair = bool(device_public_key and device_private_key)
|
|
1419
|
+
bootstrap_user, bootstrap_password, bootstrap_ssh_key = _get_provisioning_user_info(message)
|
|
1420
|
+
bootstrap_steps = _build_bootstrap_steps(
|
|
1421
|
+
bootstrap_user,
|
|
1422
|
+
bootstrap_password,
|
|
1423
|
+
bootstrap_ssh_key,
|
|
1424
|
+
include_portacode_connect=not has_device_keypair,
|
|
1425
|
+
)
|
|
1426
|
+
total_steps = 3 + len(bootstrap_steps) + 2
|
|
1427
|
+
current_step_index = 1
|
|
1428
|
+
|
|
1429
|
+
def _run_lifecycle_step(
|
|
1430
|
+
step_name: str,
|
|
1431
|
+
step_label: str,
|
|
1432
|
+
start_message: str,
|
|
1433
|
+
success_message: str,
|
|
1434
|
+
action,
|
|
1435
|
+
):
|
|
1436
|
+
nonlocal current_step_index
|
|
1437
|
+
step_index = current_step_index
|
|
1438
|
+
_emit_progress_event(self,
|
|
1439
|
+
step_index=step_index,
|
|
1440
|
+
total_steps=total_steps,
|
|
1441
|
+
step_name=step_name,
|
|
1442
|
+
step_label=step_label,
|
|
1443
|
+
status="in_progress",
|
|
1444
|
+
message=start_message,
|
|
1445
|
+
phase="lifecycle",
|
|
1446
|
+
request_id=request_id,
|
|
1447
|
+
)
|
|
1448
|
+
try:
|
|
1449
|
+
result = action()
|
|
1450
|
+
except Exception as exc:
|
|
1451
|
+
_emit_progress_event(
|
|
1452
|
+
self,
|
|
1453
|
+
step_index=step_index,
|
|
1454
|
+
total_steps=total_steps,
|
|
1455
|
+
step_name=step_name,
|
|
1456
|
+
step_label=step_label,
|
|
1457
|
+
status="failed",
|
|
1458
|
+
message=f"{step_label} failed: {exc}",
|
|
1459
|
+
phase="lifecycle",
|
|
1460
|
+
request_id=request_id,
|
|
1461
|
+
details={"error": str(exc)},
|
|
1462
|
+
)
|
|
1463
|
+
raise
|
|
1464
|
+
_emit_progress_event(
|
|
1465
|
+
self,
|
|
1466
|
+
step_index=step_index,
|
|
1467
|
+
total_steps=total_steps,
|
|
1468
|
+
step_name=step_name,
|
|
1469
|
+
step_label=step_label,
|
|
1470
|
+
status="completed",
|
|
1471
|
+
message=success_message,
|
|
1472
|
+
phase="lifecycle",
|
|
1473
|
+
request_id=request_id,
|
|
1474
|
+
)
|
|
1475
|
+
current_step_index += 1
|
|
1476
|
+
return result
|
|
1477
|
+
|
|
1478
|
+
def _validate_environment():
|
|
1479
|
+
if os.geteuid() != 0:
|
|
1480
|
+
raise PermissionError("Container creation requires root privileges.")
|
|
1481
|
+
config = _load_config()
|
|
1482
|
+
if not config or not config.get("token_value"):
|
|
1483
|
+
raise ValueError("Proxmox infrastructure is not configured.")
|
|
1484
|
+
if not config.get("network", {}).get("applied"):
|
|
1485
|
+
raise RuntimeError("Proxmox bridge setup must be applied before creating containers.")
|
|
1486
|
+
return config
|
|
1487
|
+
|
|
1488
|
+
config = _run_lifecycle_step(
|
|
1489
|
+
"validate_environment",
|
|
1490
|
+
"Validating infrastructure",
|
|
1491
|
+
"Checking token, permissions, and bridge setup…",
|
|
1492
|
+
"Infrastructure validated.",
|
|
1493
|
+
_validate_environment,
|
|
1494
|
+
)
|
|
1495
|
+
|
|
1496
|
+
def _create_container():
|
|
1497
|
+
proxmox = _connect_proxmox(config)
|
|
1498
|
+
node = config.get("node") or DEFAULT_NODE_NAME
|
|
1499
|
+
payload = _build_container_payload(message, config)
|
|
1500
|
+
payload["cpuunits"] = max(int(payload["cpus"] * 1024), 10)
|
|
1501
|
+
payload["memory"] = int(payload["ram_mib"])
|
|
1502
|
+
payload["node"] = node
|
|
1503
|
+
logger.debug(
|
|
1504
|
+
"Provisioning container node=%s template=%s ram=%s cpu=%s storage=%s",
|
|
1505
|
+
node,
|
|
1506
|
+
payload["template"],
|
|
1507
|
+
payload["ram_mib"],
|
|
1508
|
+
payload["cpus"],
|
|
1509
|
+
payload["storage"],
|
|
1510
|
+
)
|
|
1511
|
+
vmid, _ = _instantiate_container(proxmox, node, payload)
|
|
1512
|
+
payload["vmid"] = vmid
|
|
1513
|
+
payload["created_at"] = datetime.utcnow().isoformat() + "Z"
|
|
1514
|
+
payload["status"] = "creating"
|
|
1515
|
+
_write_container_record(vmid, payload)
|
|
1516
|
+
return proxmox, node, vmid, payload
|
|
1517
|
+
|
|
1518
|
+
proxmox, node, vmid, payload = _run_lifecycle_step(
|
|
1519
|
+
"create_container",
|
|
1520
|
+
"Creating container",
|
|
1521
|
+
"Provisioning the LXC container…",
|
|
1522
|
+
"Container created.",
|
|
1523
|
+
_create_container,
|
|
1524
|
+
)
|
|
1525
|
+
|
|
1526
|
+
def _start_container_step():
|
|
1527
|
+
_start_container(proxmox, node, vmid)
|
|
1528
|
+
|
|
1529
|
+
_run_lifecycle_step(
|
|
1530
|
+
"start_container",
|
|
1531
|
+
"Starting container",
|
|
1532
|
+
"Booting the container…",
|
|
1533
|
+
"Container startup completed.",
|
|
1534
|
+
_start_container_step,
|
|
1535
|
+
)
|
|
1536
|
+
_update_container_record(vmid, {"status": "running"})
|
|
1537
|
+
|
|
1538
|
+
def _bootstrap_progress_callback(
|
|
1539
|
+
step_index: int,
|
|
1540
|
+
total: int,
|
|
1541
|
+
step: Dict[str, Any],
|
|
1542
|
+
status: str,
|
|
1543
|
+
result: Optional[Dict[str, Any]],
|
|
1544
|
+
):
|
|
1545
|
+
label = step.get("display_name") or _friendly_step_label(step.get("name", "bootstrap"))
|
|
1546
|
+
error_summary = (result or {}).get("error_summary") or (result or {}).get("error")
|
|
1547
|
+
attempt = (result or {}).get("attempt")
|
|
1548
|
+
if status == "in_progress":
|
|
1549
|
+
message_text = f"{label} is running…"
|
|
1550
|
+
elif status == "completed":
|
|
1551
|
+
message_text = f"{label} completed."
|
|
1552
|
+
elif status == "retrying":
|
|
1553
|
+
attempt_desc = f" (attempt {attempt})" if attempt else ""
|
|
1554
|
+
message_text = f"{label} failed{attempt_desc}; retrying…"
|
|
1555
|
+
else:
|
|
1556
|
+
message_text = f"{label} failed"
|
|
1557
|
+
if error_summary:
|
|
1558
|
+
message_text += f": {error_summary}"
|
|
1559
|
+
details: Dict[str, Any] = {}
|
|
1560
|
+
if attempt:
|
|
1561
|
+
details["attempt"] = attempt
|
|
1562
|
+
if error_summary:
|
|
1563
|
+
details["error_summary"] = error_summary
|
|
1564
|
+
_emit_progress_event(
|
|
1565
|
+
self,
|
|
1566
|
+
step_index=step_index,
|
|
1567
|
+
total_steps=total,
|
|
1568
|
+
step_name=step.get("name", "bootstrap"),
|
|
1569
|
+
step_label=label,
|
|
1570
|
+
status=status,
|
|
1571
|
+
message=message_text,
|
|
1572
|
+
phase="bootstrap",
|
|
1573
|
+
request_id=request_id,
|
|
1574
|
+
details=details or None,
|
|
1575
|
+
)
|
|
1576
|
+
|
|
1577
|
+
public_key, steps = _bootstrap_portacode(
|
|
1578
|
+
vmid,
|
|
1579
|
+
payload["username"],
|
|
1580
|
+
payload["password"],
|
|
1581
|
+
payload["ssh_public_key"],
|
|
1582
|
+
steps=bootstrap_steps,
|
|
1583
|
+
progress_callback=_bootstrap_progress_callback,
|
|
1584
|
+
start_index=current_step_index,
|
|
1585
|
+
total_steps=total_steps,
|
|
1586
|
+
default_public_key=device_public_key if has_device_keypair else None,
|
|
1587
|
+
)
|
|
1588
|
+
current_step_index += len(bootstrap_steps)
|
|
1589
|
+
|
|
1590
|
+
service_installed = False
|
|
1591
|
+
if has_device_keypair:
|
|
1592
|
+
logger.info(
|
|
1593
|
+
"deploying dashboard-provided Portacode keypair (device_id=%s) into container %s",
|
|
1594
|
+
device_id,
|
|
1595
|
+
vmid,
|
|
1596
|
+
)
|
|
1597
|
+
_deploy_device_keypair(
|
|
1598
|
+
vmid,
|
|
1599
|
+
payload["username"],
|
|
1600
|
+
device_private_key,
|
|
1601
|
+
device_public_key,
|
|
1602
|
+
)
|
|
1603
|
+
service_installed = True
|
|
1604
|
+
service_start_index = current_step_index
|
|
1605
|
+
|
|
1606
|
+
auth_step_name = "setup_device_authentication"
|
|
1607
|
+
auth_label = "Setting up device authentication"
|
|
1608
|
+
_emit_progress_event(
|
|
1609
|
+
self,
|
|
1610
|
+
step_index=service_start_index,
|
|
1611
|
+
total_steps=total_steps,
|
|
1612
|
+
step_name=auth_step_name,
|
|
1613
|
+
step_label=auth_label,
|
|
1614
|
+
status="in_progress",
|
|
1615
|
+
message="Notifying the server of the new device…",
|
|
1616
|
+
phase="service",
|
|
1617
|
+
request_id=request_id,
|
|
1618
|
+
)
|
|
1619
|
+
_emit_progress_event(
|
|
1620
|
+
self,
|
|
1621
|
+
step_index=service_start_index,
|
|
1622
|
+
total_steps=total_steps,
|
|
1623
|
+
step_name=auth_step_name,
|
|
1624
|
+
step_label=auth_label,
|
|
1625
|
+
status="completed",
|
|
1626
|
+
message="Authentication metadata recorded.",
|
|
1627
|
+
phase="service",
|
|
1628
|
+
request_id=request_id,
|
|
1629
|
+
)
|
|
1630
|
+
|
|
1631
|
+
install_step = service_start_index + 1
|
|
1632
|
+
install_label = "Launching Portacode service"
|
|
1633
|
+
_emit_progress_event(
|
|
1634
|
+
self,
|
|
1635
|
+
step_index=install_step,
|
|
1636
|
+
total_steps=total_steps,
|
|
1637
|
+
step_name="launch_portacode_service",
|
|
1638
|
+
step_label=install_label,
|
|
1639
|
+
status="in_progress",
|
|
1640
|
+
message="Running sudo portacode service install…",
|
|
1641
|
+
phase="service",
|
|
1642
|
+
request_id=request_id,
|
|
1643
|
+
)
|
|
1644
|
+
|
|
1645
|
+
cmd = f"su - {payload['username']} -c 'sudo -S portacode service install'"
|
|
1646
|
+
res = _run_pct(vmid, cmd, input_text=payload["password"] + "\n")
|
|
1647
|
+
|
|
1648
|
+
if res["returncode"] != 0:
|
|
1649
|
+
_emit_progress_event(
|
|
1650
|
+
self,
|
|
1651
|
+
step_index=install_step,
|
|
1652
|
+
total_steps=total_steps,
|
|
1653
|
+
step_name="launch_portacode_service",
|
|
1654
|
+
step_label=install_label,
|
|
1655
|
+
status="failed",
|
|
1656
|
+
message=f"{install_label} failed: {res.get('stderr') or res.get('stdout')}",
|
|
1657
|
+
phase="service",
|
|
1658
|
+
request_id=request_id,
|
|
1659
|
+
details={
|
|
1660
|
+
"stderr": res.get("stderr"),
|
|
1661
|
+
"stdout": res.get("stdout"),
|
|
1662
|
+
},
|
|
1663
|
+
)
|
|
1664
|
+
raise RuntimeError(res.get("stderr") or res.get("stdout") or "Service install failed")
|
|
1665
|
+
|
|
1666
|
+
_emit_progress_event(
|
|
1667
|
+
self,
|
|
1668
|
+
step_index=install_step,
|
|
1669
|
+
total_steps=total_steps,
|
|
1670
|
+
step_name="launch_portacode_service",
|
|
1671
|
+
step_label=install_label,
|
|
1672
|
+
status="completed",
|
|
1673
|
+
message="Portacode service install finished.",
|
|
1674
|
+
phase="service",
|
|
1675
|
+
request_id=request_id,
|
|
1676
|
+
)
|
|
1677
|
+
|
|
1678
|
+
logger.info("create_proxmox_container: portacode service install completed inside ct %s", vmid)
|
|
1679
|
+
|
|
1680
|
+
current_step_index += 2
|
|
1681
|
+
|
|
1682
|
+
return {
|
|
1683
|
+
"event": "proxmox_container_created",
|
|
1684
|
+
"success": True,
|
|
1685
|
+
"message": f"Container {vmid} is ready and Portacode key captured.",
|
|
1686
|
+
"ctid": str(vmid),
|
|
1687
|
+
"public_key": public_key,
|
|
1688
|
+
"container": {
|
|
1689
|
+
"vmid": vmid,
|
|
1690
|
+
"hostname": payload["hostname"],
|
|
1691
|
+
"template": payload["template"],
|
|
1692
|
+
"storage": payload["storage"],
|
|
1693
|
+
"disk_gib": payload["disk_gib"],
|
|
1694
|
+
"ram_mib": payload["ram_mib"],
|
|
1695
|
+
"cpus": payload["cpus"],
|
|
1696
|
+
},
|
|
1697
|
+
"setup_steps": steps,
|
|
1698
|
+
"device_id": device_id,
|
|
1699
|
+
"service_installed": service_installed,
|
|
1700
|
+
}
|
|
1701
|
+
|
|
1702
|
+
|
|
1703
|
+
class StartPortacodeServiceHandler(SyncHandler):
|
|
1704
|
+
"""Start the Portacode service inside a newly created container."""
|
|
1705
|
+
|
|
1706
|
+
@property
|
|
1707
|
+
def command_name(self) -> str:
|
|
1708
|
+
return "start_portacode_service"
|
|
1709
|
+
|
|
1710
|
+
def execute(self, message: Dict[str, Any]) -> Dict[str, Any]:
|
|
1711
|
+
ctid = message.get("ctid")
|
|
1712
|
+
if not ctid:
|
|
1713
|
+
raise ValueError("ctid is required")
|
|
1714
|
+
try:
|
|
1715
|
+
vmid = int(ctid)
|
|
1716
|
+
except ValueError:
|
|
1717
|
+
raise ValueError("ctid must be an integer")
|
|
1718
|
+
|
|
1719
|
+
record = _read_container_record(vmid)
|
|
1720
|
+
user = record.get("username")
|
|
1721
|
+
password = record.get("password")
|
|
1722
|
+
if not user or not password:
|
|
1723
|
+
raise RuntimeError("Container credentials unavailable")
|
|
1724
|
+
|
|
1725
|
+
start_index = int(message.get("step_index", 1))
|
|
1726
|
+
total_steps = int(message.get("total_steps", start_index + 2))
|
|
1727
|
+
request_id = message.get("request_id")
|
|
1728
|
+
|
|
1729
|
+
auth_step_name = "setup_device_authentication"
|
|
1730
|
+
auth_label = "Setting up device authentication"
|
|
1731
|
+
_emit_progress_event(
|
|
1732
|
+
self,
|
|
1733
|
+
step_index=start_index,
|
|
1734
|
+
total_steps=total_steps,
|
|
1735
|
+
step_name=auth_step_name,
|
|
1736
|
+
step_label=auth_label,
|
|
1737
|
+
status="in_progress",
|
|
1738
|
+
message="Notifying the server of the new device…",
|
|
1739
|
+
phase="service",
|
|
1740
|
+
request_id=request_id,
|
|
1741
|
+
)
|
|
1742
|
+
_emit_progress_event(
|
|
1743
|
+
self,
|
|
1744
|
+
step_index=start_index,
|
|
1745
|
+
total_steps=total_steps,
|
|
1746
|
+
step_name=auth_step_name,
|
|
1747
|
+
step_label=auth_label,
|
|
1748
|
+
status="completed",
|
|
1749
|
+
message="Authentication metadata recorded.",
|
|
1750
|
+
phase="service",
|
|
1751
|
+
request_id=request_id,
|
|
1752
|
+
)
|
|
1753
|
+
|
|
1754
|
+
install_step = start_index + 1
|
|
1755
|
+
install_label = "Launching Portacode service"
|
|
1756
|
+
_emit_progress_event(
|
|
1757
|
+
self,
|
|
1758
|
+
step_index=install_step,
|
|
1759
|
+
total_steps=total_steps,
|
|
1760
|
+
step_name="launch_portacode_service",
|
|
1761
|
+
step_label=install_label,
|
|
1762
|
+
status="in_progress",
|
|
1763
|
+
message="Running sudo portacode service install…",
|
|
1764
|
+
phase="service",
|
|
1765
|
+
request_id=request_id,
|
|
1766
|
+
)
|
|
1767
|
+
|
|
1768
|
+
cmd = f"su - {user} -c 'sudo -S portacode service install'"
|
|
1769
|
+
res = _run_pct(vmid, cmd, input_text=password + "\n")
|
|
1770
|
+
|
|
1771
|
+
if res["returncode"] != 0:
|
|
1772
|
+
_emit_progress_event(
|
|
1773
|
+
self,
|
|
1774
|
+
step_index=install_step,
|
|
1775
|
+
total_steps=total_steps,
|
|
1776
|
+
step_name="launch_portacode_service",
|
|
1777
|
+
step_label=install_label,
|
|
1778
|
+
status="failed",
|
|
1779
|
+
message=f"{install_label} failed: {res.get('stderr') or res.get('stdout')}",
|
|
1780
|
+
phase="service",
|
|
1781
|
+
request_id=request_id,
|
|
1782
|
+
details={
|
|
1783
|
+
"stderr": res.get("stderr"),
|
|
1784
|
+
"stdout": res.get("stdout"),
|
|
1785
|
+
},
|
|
1786
|
+
)
|
|
1787
|
+
raise RuntimeError(res.get("stderr") or res.get("stdout") or "Service install failed")
|
|
1788
|
+
|
|
1789
|
+
_emit_progress_event(
|
|
1790
|
+
self,
|
|
1791
|
+
step_index=install_step,
|
|
1792
|
+
total_steps=total_steps,
|
|
1793
|
+
step_name="launch_portacode_service",
|
|
1794
|
+
step_label=install_label,
|
|
1795
|
+
status="completed",
|
|
1796
|
+
message="Portacode service install finished.",
|
|
1797
|
+
phase="service",
|
|
1798
|
+
request_id=request_id,
|
|
1799
|
+
)
|
|
1800
|
+
|
|
1801
|
+
return {
|
|
1802
|
+
"event": "proxmox_service_started",
|
|
1803
|
+
"success": True,
|
|
1804
|
+
"message": "Portacode service install completed",
|
|
1805
|
+
"ctid": str(vmid),
|
|
1806
|
+
}
|
|
1807
|
+
|
|
1808
|
+
|
|
1809
|
+
class StartProxmoxContainerHandler(SyncHandler):
|
|
1810
|
+
"""Start a managed container via the Proxmox API."""
|
|
1811
|
+
|
|
1812
|
+
@property
|
|
1813
|
+
def command_name(self) -> str:
|
|
1814
|
+
return "start_proxmox_container"
|
|
1815
|
+
|
|
1816
|
+
def execute(self, message: Dict[str, Any]) -> Dict[str, Any]:
|
|
1817
|
+
vmid = _parse_ctid(message)
|
|
1818
|
+
config = _ensure_infra_configured()
|
|
1819
|
+
proxmox = _connect_proxmox(config)
|
|
1820
|
+
node = _get_node_from_config(config)
|
|
1821
|
+
_ensure_container_managed(proxmox, node, vmid)
|
|
1822
|
+
|
|
1823
|
+
status, elapsed = _start_container(proxmox, node, vmid)
|
|
1824
|
+
_update_container_record(vmid, {"status": "running"})
|
|
1825
|
+
try:
|
|
1826
|
+
_ensure_container_tunnel_running(proxmox, node, vmid)
|
|
1827
|
+
except Exception as exc:
|
|
1828
|
+
raise RuntimeError(f"Failed to start Cloudflare tunnel for container {vmid}: {exc}") from exc
|
|
1829
|
+
|
|
1830
|
+
infra = get_infra_snapshot()
|
|
1831
|
+
return {
|
|
1832
|
+
"event": "proxmox_container_action",
|
|
1833
|
+
"action": "start",
|
|
1834
|
+
"success": True,
|
|
1835
|
+
"ctid": str(vmid),
|
|
1836
|
+
"message": f"Started container {vmid} in {elapsed:.1f}s.",
|
|
1837
|
+
"details": {"exitstatus": status.get("exitstatus")},
|
|
1838
|
+
"status": status.get("status"),
|
|
1839
|
+
"infra": infra,
|
|
1840
|
+
}
|
|
1841
|
+
|
|
1842
|
+
|
|
1843
|
+
class StopProxmoxContainerHandler(SyncHandler):
|
|
1844
|
+
"""Stop a managed container via the Proxmox API."""
|
|
1845
|
+
|
|
1846
|
+
@property
|
|
1847
|
+
def command_name(self) -> str:
|
|
1848
|
+
return "stop_proxmox_container"
|
|
1849
|
+
|
|
1850
|
+
def execute(self, message: Dict[str, Any]) -> Dict[str, Any]:
|
|
1851
|
+
vmid = _parse_ctid(message)
|
|
1852
|
+
config = _ensure_infra_configured()
|
|
1853
|
+
proxmox = _connect_proxmox(config)
|
|
1854
|
+
node = _get_node_from_config(config)
|
|
1855
|
+
_ensure_container_managed(proxmox, node, vmid)
|
|
1856
|
+
|
|
1857
|
+
status, elapsed = _stop_container(proxmox, node, vmid)
|
|
1858
|
+
_stop_container_tunnel(vmid)
|
|
1859
|
+
final_status = status.get("status") or "stopped"
|
|
1860
|
+
_update_container_record(vmid, {"status": final_status})
|
|
1861
|
+
|
|
1862
|
+
infra = get_infra_snapshot()
|
|
1863
|
+
message_text = (
|
|
1864
|
+
f"Container {vmid} is already stopped."
|
|
1865
|
+
if final_status != "running" and elapsed == 0.0
|
|
1866
|
+
else f"Stopped container {vmid} in {elapsed:.1f}s."
|
|
1867
|
+
)
|
|
1868
|
+
return {
|
|
1869
|
+
"event": "proxmox_container_action",
|
|
1870
|
+
"action": "stop",
|
|
1871
|
+
"success": True,
|
|
1872
|
+
"ctid": str(vmid),
|
|
1873
|
+
"message": message_text,
|
|
1874
|
+
"details": {"exitstatus": status.get("exitstatus")},
|
|
1875
|
+
"status": final_status,
|
|
1876
|
+
"infra": infra,
|
|
1877
|
+
}
|
|
1878
|
+
|
|
1879
|
+
|
|
1880
|
+
class RemoveProxmoxContainerHandler(SyncHandler):
|
|
1881
|
+
"""Delete a managed container via the Proxmox API."""
|
|
1882
|
+
|
|
1883
|
+
@property
|
|
1884
|
+
def command_name(self) -> str:
|
|
1885
|
+
return "remove_proxmox_container"
|
|
1886
|
+
|
|
1887
|
+
def execute(self, message: Dict[str, Any]) -> Dict[str, Any]:
|
|
1888
|
+
vmid = _parse_ctid(message)
|
|
1889
|
+
config = _ensure_infra_configured()
|
|
1890
|
+
proxmox = _connect_proxmox(config)
|
|
1891
|
+
node = _get_node_from_config(config)
|
|
1892
|
+
_ensure_container_managed(proxmox, node, vmid)
|
|
1893
|
+
|
|
1894
|
+
_stop_container_tunnel(vmid)
|
|
1895
|
+
stop_status, stop_elapsed = _stop_container(proxmox, node, vmid)
|
|
1896
|
+
delete_status, delete_elapsed = _delete_container(proxmox, node, vmid)
|
|
1897
|
+
try:
|
|
1898
|
+
_update_container_tunnel(vmid, None)
|
|
1899
|
+
except FileNotFoundError:
|
|
1900
|
+
pass
|
|
1901
|
+
_remove_container_record(vmid)
|
|
1902
|
+
|
|
1903
|
+
infra = get_infra_snapshot()
|
|
1904
|
+
return {
|
|
1905
|
+
"event": "proxmox_container_action",
|
|
1906
|
+
"action": "remove",
|
|
1907
|
+
"success": True,
|
|
1908
|
+
"ctid": str(vmid),
|
|
1909
|
+
"message": f"Deleted container {vmid} in {delete_elapsed:.1f}s.",
|
|
1910
|
+
"details": {
|
|
1911
|
+
"stop_exitstatus": stop_status.get("exitstatus"),
|
|
1912
|
+
"delete_exitstatus": delete_status.get("exitstatus"),
|
|
1913
|
+
},
|
|
1914
|
+
"status": "deleted",
|
|
1915
|
+
"infra": infra,
|
|
1916
|
+
}
|
|
1917
|
+
|
|
1918
|
+
|
|
1919
|
+
class CreateCloudflareTunnelHandler(SyncHandler):
|
|
1920
|
+
"""Create a Cloudflare tunnel for a container."""
|
|
1921
|
+
|
|
1922
|
+
@property
|
|
1923
|
+
def command_name(self) -> str:
|
|
1924
|
+
return "create_cloudflare_tunnel"
|
|
1925
|
+
|
|
1926
|
+
def execute(self, message: Dict[str, Any]) -> Dict[str, Any]:
|
|
1927
|
+
vmid = _parse_ctid(message)
|
|
1928
|
+
container_port = int(message.get("container_port") or 0)
|
|
1929
|
+
if container_port <= 0:
|
|
1930
|
+
raise ValueError("container_port is required and must be greater than zero.")
|
|
1931
|
+
hostname = (message.get("cloudflare_url") or message.get("hostname") or "").strip()
|
|
1932
|
+
hostname = hostname or None
|
|
1933
|
+
protocol = (message.get("protocol") or "http").strip().lower()
|
|
1934
|
+
if protocol not in {"http", "https", "tcp"}:
|
|
1935
|
+
raise ValueError("protocol must be one of http, https, or tcp.")
|
|
1936
|
+
config = _ensure_infra_configured()
|
|
1937
|
+
_ensure_cloudflare_token(config)
|
|
1938
|
+
proxmox = _connect_proxmox(config)
|
|
1939
|
+
node = _get_node_from_config(config)
|
|
1940
|
+
_ensure_container_managed(proxmox, node, vmid)
|
|
1941
|
+
status = proxmox.nodes(node).lxc(vmid).status.current.get().get("status")
|
|
1942
|
+
if status != "running":
|
|
1943
|
+
raise RuntimeError("Container must be running to create a tunnel.")
|
|
1944
|
+
tunnel = {
|
|
1945
|
+
"container_port": container_port,
|
|
1946
|
+
"protocol": protocol,
|
|
1947
|
+
}
|
|
1948
|
+
if hostname:
|
|
1949
|
+
tunnel["url"] = hostname
|
|
1950
|
+
created = _launch_container_tunnel(proxmox, node, vmid, tunnel)
|
|
1951
|
+
infra = get_infra_snapshot()
|
|
1952
|
+
host_url = created.get("url")
|
|
1953
|
+
response_message = f"Created Cloudflare tunnel for container {vmid}."
|
|
1954
|
+
if host_url:
|
|
1955
|
+
response_message = f"{response_message[:-1]} -> {host_url}."
|
|
1956
|
+
response = {
|
|
1957
|
+
"event": "cloudflare_tunnel_created",
|
|
1958
|
+
"ctid": str(vmid),
|
|
1959
|
+
"success": True,
|
|
1960
|
+
"message": response_message,
|
|
1961
|
+
"tunnel": created,
|
|
1962
|
+
"infra": infra,
|
|
1963
|
+
}
|
|
1964
|
+
device_id = message.get("device_id")
|
|
1965
|
+
if device_id:
|
|
1966
|
+
response["device_id"] = device_id
|
|
1967
|
+
return response
|
|
1968
|
+
|
|
1969
|
+
|
|
1970
|
+
class UpdateCloudflareTunnelHandler(SyncHandler):
|
|
1971
|
+
"""Update an existing Cloudflare tunnel for a container."""
|
|
1972
|
+
|
|
1973
|
+
@property
|
|
1974
|
+
def command_name(self) -> str:
|
|
1975
|
+
return "update_cloudflare_tunnel"
|
|
1976
|
+
|
|
1977
|
+
def execute(self, message: Dict[str, Any]) -> Dict[str, Any]:
|
|
1978
|
+
vmid = _parse_ctid(message)
|
|
1979
|
+
config = _ensure_infra_configured()
|
|
1980
|
+
_ensure_cloudflare_token(config)
|
|
1981
|
+
proxmox = _connect_proxmox(config)
|
|
1982
|
+
node = _get_node_from_config(config)
|
|
1983
|
+
_ensure_container_managed(proxmox, node, vmid)
|
|
1984
|
+
record = _read_container_record(vmid)
|
|
1985
|
+
tunnel = record.get("tunnel")
|
|
1986
|
+
if not tunnel:
|
|
1987
|
+
raise RuntimeError("No Cloudflare tunnel configured for this container.")
|
|
1988
|
+
container_port = int(message.get("container_port") or tunnel.get("container_port") or 0)
|
|
1989
|
+
if container_port <= 0:
|
|
1990
|
+
raise ValueError("container_port must be greater than zero.")
|
|
1991
|
+
hostname = (message.get("cloudflare_url") or tunnel.get("url") or "").strip()
|
|
1992
|
+
hostname = hostname or None
|
|
1993
|
+
protocol = (message.get("protocol") or tunnel.get("protocol") or "http").strip().lower()
|
|
1994
|
+
if protocol not in {"http", "https", "tcp"}:
|
|
1995
|
+
raise ValueError("protocol must be one of http, https, or tcp.")
|
|
1996
|
+
updated_tunnel = {
|
|
1997
|
+
"container_port": container_port,
|
|
1998
|
+
"protocol": protocol,
|
|
1999
|
+
}
|
|
2000
|
+
if hostname:
|
|
2001
|
+
updated_tunnel["url"] = hostname
|
|
2002
|
+
result = _launch_container_tunnel(proxmox, node, vmid, updated_tunnel)
|
|
2003
|
+
infra = get_infra_snapshot()
|
|
2004
|
+
host_url = result.get("url")
|
|
2005
|
+
response_message = f"Updated Cloudflare tunnel for container {vmid}."
|
|
2006
|
+
if host_url:
|
|
2007
|
+
response_message = f"{response_message[:-1]} -> {host_url}."
|
|
2008
|
+
response = {
|
|
2009
|
+
"event": "cloudflare_tunnel_updated",
|
|
2010
|
+
"ctid": str(vmid),
|
|
2011
|
+
"success": True,
|
|
2012
|
+
"message": response_message,
|
|
2013
|
+
"tunnel": result,
|
|
2014
|
+
"infra": infra,
|
|
2015
|
+
}
|
|
2016
|
+
device_id = message.get("device_id")
|
|
2017
|
+
if device_id:
|
|
2018
|
+
response["device_id"] = device_id
|
|
2019
|
+
return response
|
|
2020
|
+
|
|
2021
|
+
|
|
2022
|
+
class RemoveCloudflareTunnelHandler(SyncHandler):
|
|
2023
|
+
"""Remove any Cloudflare tunnel associated with a container."""
|
|
2024
|
+
|
|
2025
|
+
@property
|
|
2026
|
+
def command_name(self) -> str:
|
|
2027
|
+
return "remove_cloudflare_tunnel"
|
|
2028
|
+
|
|
2029
|
+
def execute(self, message: Dict[str, Any]) -> Dict[str, Any]:
|
|
2030
|
+
vmid = _parse_ctid(message)
|
|
2031
|
+
_remove_container_tunnel_state(vmid)
|
|
2032
|
+
infra = get_infra_snapshot()
|
|
2033
|
+
response = {
|
|
2034
|
+
"event": "cloudflare_tunnel_removed",
|
|
2035
|
+
"ctid": str(vmid),
|
|
2036
|
+
"success": True,
|
|
2037
|
+
"message": f"Removed Cloudflare tunnel state for container {vmid}.",
|
|
2038
|
+
"tunnel": None,
|
|
2039
|
+
"infra": infra,
|
|
2040
|
+
}
|
|
2041
|
+
device_id = message.get("device_id")
|
|
2042
|
+
if device_id:
|
|
2043
|
+
response["device_id"] = device_id
|
|
2044
|
+
return response
|
|
2045
|
+
|
|
2046
|
+
|
|
2047
|
+
class ConfigureProxmoxInfraHandler(SyncHandler):
|
|
2048
|
+
@property
|
|
2049
|
+
def command_name(self) -> str:
|
|
2050
|
+
return "setup_proxmox_infra"
|
|
2051
|
+
|
|
2052
|
+
def execute(self, message: Dict[str, Any]) -> Dict[str, Any]:
|
|
2053
|
+
token_identifier = message.get("token_identifier")
|
|
2054
|
+
token_value = message.get("token_value")
|
|
2055
|
+
verify_ssl = message.get("verify_ssl")
|
|
2056
|
+
snapshot = configure_infrastructure(
|
|
2057
|
+
token_identifier=token_identifier,
|
|
2058
|
+
token_value=token_value,
|
|
2059
|
+
verify_ssl=verify_ssl,
|
|
2060
|
+
cloudflare_api_token=message.get("cloudflare_api_token"),
|
|
2061
|
+
)
|
|
2062
|
+
return {
|
|
2063
|
+
"event": "proxmox_infra_configured",
|
|
2064
|
+
"success": True,
|
|
2065
|
+
"message": "Proxmox infrastructure configured",
|
|
2066
|
+
"infra": snapshot,
|
|
2067
|
+
}
|
|
2068
|
+
|
|
2069
|
+
|
|
2070
|
+
class RevertProxmoxInfraHandler(SyncHandler):
|
|
2071
|
+
@property
|
|
2072
|
+
def command_name(self) -> str:
|
|
2073
|
+
return "revert_proxmox_infra"
|
|
2074
|
+
|
|
2075
|
+
def execute(self, message: Dict[str, Any]) -> Dict[str, Any]:
|
|
2076
|
+
snapshot = revert_infrastructure()
|
|
2077
|
+
return {
|
|
2078
|
+
"event": "proxmox_infra_reverted",
|
|
2079
|
+
"success": True,
|
|
2080
|
+
"message": "Proxmox infrastructure configuration reverted",
|
|
2081
|
+
"infra": snapshot,
|
|
2082
|
+
}
|