agentworks-cli 0.2.1__py3-none-any.whl

agentworks/vms/provisioners/lima.py

@@ -0,0 +1,295 @@
+"""Lima VM provisioner -- local and remote VM Host variants."""
+
+from __future__ import annotations
+
+import json
+import shlex
+import tempfile
+import textwrap
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from agentworks import output
+from agentworks.db import VMStatus
+from agentworks.ssh import ExecTarget, LimaTarget, RemoteLimaTarget, SSHError, SSHTarget, copy_to
+from agentworks.ssh import run as ssh_run
+from agentworks.vms.base import ProvisionResult, VMProvisioner
+from agentworks.vms.bootstrap_script import generate_bootstrap_script, parse_bootstrap_output, vm_hostname
+from agentworks.vms.cloud_init import PROVISIONING_PACKAGES
+
+if TYPE_CHECKING:
+    from agentworks.config import Config
+    from agentworks.db import VMRow
+
+# Lima template for Debian cloud VMs (values substituted at create time).
+# The provision block runs the full bootstrap script (user, packages, swap,
+# SSH key, Tailscale) as a system-level provisioner during limactl start.
+LIMA_TEMPLATE = """\
+# Agentworks Debian VM template for Lima
+arch: default
+images:
+  - location: https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2
+    arch: x86_64
+  - location: https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-arm64.qcow2
+    arch: aarch64
+cpus: {cpus}
+memory: {memory}GiB
+disk: {disk}GiB
+ssh:
+  localPort: 0
+mountType: virtiofs
+provision:
+  - mode: system
+    script: |
+{provision_script}
+"""
+
+
+class LimaProvisioner(VMProvisioner):
+    """Provisions Lima VMs, either locally or on a remote VM Host."""
+
+    def __init__(self, vm_host_ssh: str | None = None) -> None:
+        """Initialize the Lima provisioner.
+
+        Args:
+            vm_host_ssh: SSH host for remote mode. None for local mode.
+        """
+        self._vm_host_ssh = vm_host_ssh
+
+    @property
+    def is_remote(self) -> bool:
+        return self._vm_host_ssh is not None
+
+    def _run_lima(self, command: str, *, check: bool = True) -> str:
+        """Run a limactl command, locally or on the VM Host."""
+        if self.is_remote:
+            assert self._vm_host_ssh is not None
+            target = SSHTarget(host=self._vm_host_ssh, user=None, login_shell=True)
+            result = ssh_run(target, command, check=check)
+            return result.stdout
+        else:
+            import subprocess
+
+            proc = subprocess.run(
+                shlex.split(command),
+                capture_output=True,
+                text=True,
+                encoding="utf-8",
+                errors="replace",
+            )
+            if check and proc.returncode != 0:
+                raise SSHError(f"limactl failed: {proc.stderr.strip()}")
+            return proc.stdout
+
+    def create(
+        self,
+        vm_name: str,
+        config: Config,
+        *,
+        cpus: int = 4,
+        memory: int = 8,
+        disk: int = 50,
+        tailscale_auth_key: str | None = None,
+    ) -> ProvisionResult:
+        if not self.is_remote:
+            import shutil
+
+            if not shutil.which("limactl"):
+                from agentworks.output import VMError
+
+                raise VMError(
+                    "'limactl' not found. Lima is not installed on this machine. "
+                    "For remote Lima VMs, set defaults.vm_host in your config or pass --vm-host."
+                )
+
+        if self.is_remote:
+            output.info(f"Connecting to VM host '{self._vm_host_ssh}'...")
+        output.info(f"Creating Lima VM '{vm_name}' ({'remote' if self.is_remote else 'local'})...")
+        output.detail(f"Resources: {cpus} CPUs, {memory} GiB memory, {disk} GiB disk")
+        if config.vm.swap > 0:
+            output.detail(f"Swap: {config.vm.swap} GiB")
+
+        # Generate the full bootstrap script and embed in the Lima provision block.
+        # This handles user creation, system packages, swap, SSH key, and Tailscale.
+        if tailscale_auth_key:
+            ssh_pub_key = config.operator.ssh_public_key.read_text().strip()
+            provision_script = generate_bootstrap_script(
+                admin_username=config.admin.username,
+                ssh_public_key=ssh_pub_key,
+                provisioning_packages=PROVISIONING_PACKAGES,
+                tailscale_auth_key=tailscale_auth_key,
+                hostname=vm_hostname("lima", vm_name),
+                swap=config.vm.swap,
+            )
+        else:
+            # No Tailscale key -- provision block is a no-op.
+            # Phase A bootstrap will handle everything separately.
+            provision_script = (
+                "#!/bin/bash\necho '##STEP## Provision'\necho '##SUCCESS## no-op (deferred to Phase A)'\n"
+            )
+
+        # Indent the provision script for YAML embedding (6 spaces)
+        indented_script = textwrap.indent(provision_script, "      ")
+        rendered = LIMA_TEMPLATE.format(cpus=cpus, memory=memory, disk=disk, provision_script=indented_script)
+
+        if self.is_remote:
+            self._create_remote(vm_name, rendered)
+        else:
+            self._create_local(vm_name, rendered)
+
+        output.detail(f"Lima VM '{vm_name}' created.")
+
+        # If Tailscale was provisioned via the provision block, extract the IP
+        tailscale_ip = None
+        bootstrap_complete = False
+        if tailscale_auth_key:
+            output.detail("Retrieving Tailscale IP...")
+            try:
+                ip_output = self._run_lima(f"limactl shell {vm_name} sudo tailscale ip -4")
+                tailscale_ip = ip_output.strip()
+                bootstrap_complete = True
+                output.detail(f"Tailscale IP: {tailscale_ip}")
+            except SSHError as e:
+                output.warn(f"could not retrieve Tailscale IP: {e}")
+                output.warn("Tailscale will be set up during Phase A bootstrap.")
+
+        if self.is_remote:
+            assert self._vm_host_ssh is not None
+            return ProvisionResult(
+                admin_exec_target=ExecTarget(
+                    remote_lima=RemoteLimaTarget(vm_name=vm_name, vm_host_ssh=self._vm_host_ssh),
+                ),
+                bootstrap_complete=bootstrap_complete,
+                tailscale_ip=tailscale_ip,
+            )
+        else:
+            return ProvisionResult(
+                admin_exec_target=ExecTarget(lima=LimaTarget(vm_name=vm_name)),
+                bootstrap_complete=bootstrap_complete,
+                tailscale_ip=tailscale_ip,
+            )
+
+    def _create_local(self, vm_name: str, lima_yaml: str) -> None:
+        """Create and start a Lima VM locally."""
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f:
+            f.write(lima_yaml)
+            template_path = f.name
+
+        try:
+            self._run_lima(f"limactl create --name {vm_name} --tty=false {template_path}")
+            self._run_lima(f"limactl start {vm_name}")
+        except SSHError:
+            self._log_provision_errors(vm_name)
+            raise
+        finally:
+            Path(template_path).unlink(missing_ok=True)
+
+    def _create_remote(self, vm_name: str, lima_yaml: str) -> None:
+        """Create and start a Lima VM on a remote VM Host."""
+        assert self._vm_host_ssh is not None
+        target = SSHTarget(host=self._vm_host_ssh, user=None)
+
+        # Write Lima YAML locally and copy to VM Host
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f:
+            f.write(lima_yaml)
+            local_template = f.name
+
+        remote_template = f"/tmp/agentworks-{vm_name}.yaml"
+        copy_to(target, local_template, remote_template)
+        Path(local_template).unlink()
+
+        # Run limactl create + start as a single detached operation
+        from agentworks.remote_exec import run_detached
+        from agentworks.ssh import SSHLogger
+
+        ssh_logger = SSHLogger(vm_name, "vm-provision")
+        host_target = ExecTarget(
+            ssh=SSHTarget(host=self._vm_host_ssh, user=None, login_shell=True),
+            logger=ssh_logger,
+        )
+        lima_cmd = f"limactl create --name {vm_name} --tty=false {remote_template} && limactl start {vm_name}"
+        output.detail("Starting and provisioning VM via Lima (this may take several minutes)...")
+        result = run_detached(
+            host_target,
+            lima_cmd,
+            label=f"Lima ({vm_name})",
+            base_path=f"/tmp/agentworks-lima-{vm_name}",
+            timeout=600,
+            quiet=True,
+        )
+        if result.exit_code != 0:
+            # Parse structured markers from provision script output if present
+            bootstrap = parse_bootstrap_output(result.output, result.exit_code)
+            for step in bootstrap.steps:
+                if step.error:
+                    ssh_logger.log_error(f"Provision step '{step.name}': {step.error}")
+
+            ssh_logger.log_error(f"limactl failed (exit {result.exit_code})")
+            ssh_logger.log_error(result.output)
+            ssh_logger.close()
+            raise SSHError(
+                f"limactl create/start failed (exit {result.exit_code})\n"
+                f"SSH log: {ssh_logger.path}\n"
+                f"Last output:\n{result.output[-1000:]}"
+            )
+        ssh_logger.close()
+
+        # Clean up remote temp file
+        ssh_run(target, f"rm -f {remote_template}", check=False)
+
+    def _log_provision_errors(self, vm_name: str) -> None:
+        """Attempt to surface provision script errors from Lima logs."""
+        try:
+            log_output = self._run_lima(
+                f"limactl shell {vm_name} cat /var/log/cloud-init-output.log 2>/dev/null || true",
+                check=False,
+            )
+            if log_output.strip():
+                bootstrap = parse_bootstrap_output(log_output, 1)
+                for step in bootstrap.steps:
+                    if step.error:
+                        output.warn(f"Provision error ({step.name}): {step.error}")
+        except SSHError:
+            pass
+
+    def start(self, vm: VMRow) -> None:
+        output.info(f"Starting Lima VM '{vm.name}'...")
+        self._run_lima(f"limactl start {vm.name}")
+        output.info(f"Lima VM '{vm.name}' started")
+
+    def stop(self, vm: VMRow) -> None:
+        output.info(f"Stopping Lima VM '{vm.name}'...")
+        self._run_lima(f"limactl stop {vm.name}")
+        output.info(f"Lima VM '{vm.name}' stopped")
+
+    def delete(self, vm: VMRow) -> None:
+        output.info(f"Deleting Lima VM '{vm.name}'...")
+        self._run_lima(f"limactl delete --force {vm.name}", check=False)
+        output.info(f"Lima VM '{vm.name}' deleted")
+
+    def admin_exec_target(self, vm: VMRow, *, config: object | None = None) -> ExecTarget:
+        if self.is_remote:
+            assert self._vm_host_ssh is not None
+            return ExecTarget(
+                remote_lima=RemoteLimaTarget(vm_name=vm.name, vm_host_ssh=self._vm_host_ssh),
+            )
+        return ExecTarget(lima=LimaTarget(vm_name=vm.name))
+
+    def status(self, vm: VMRow) -> VMStatus:
+        try:
+            output = self._run_lima(f"limactl list --json {vm.name}", check=False)
+        except SSHError:
+            return VMStatus.UNKNOWN
+
+        for line in output.strip().splitlines():
+            try:
+                entry = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            raw_status = entry.get("status", "").lower()
+            if raw_status == "running":
+                return VMStatus.RUNNING
+            if raw_status == "stopped":
+                return VMStatus.STOPPED
+            return VMStatus.UNKNOWN
+        return VMStatus.UNKNOWN

agentworks/vms/provisioners/proxmox.py

@@ -0,0 +1,279 @@
+"""Proxmox VE VM provisioner -- creates and manages VMs via the Proxmox REST API."""
+
+from __future__ import annotations
+
+import os
+import time
+import urllib.parse
+from typing import TYPE_CHECKING
+
+from agentworks import output
+from agentworks.db import VMStatus
+from agentworks.ssh import ExecTarget, SSHTarget
+from agentworks.vms.base import ProvisionResult, VMProvisioner
+from agentworks.vms.bootstrap_script import generate_bootstrap_script, vm_hostname
+from agentworks.vms.cloud_init import PROVISIONING_PACKAGES
+from agentworks.vms.provisioners.proxmox_api import ProxmoxAPI, ProxmoxAPIError
+
+if TYPE_CHECKING:
+    from agentworks.config import Config, ProxmoxConfig
+    from agentworks.db import VMRow
+
+
+class ProxmoxProvisioner(VMProvisioner):
+    """Provisions VMs on Proxmox VE via clone + cloud-init + guest agent bootstrap."""
+
+    def __init__(self, proxmox_config: ProxmoxConfig) -> None:
+        self._cfg = proxmox_config
+        token_secret = os.environ.get("PROXMOX_TOKEN_SECRET", "")
+        if not token_secret:
+            raise RuntimeError(
+                "PROXMOX_TOKEN_SECRET environment variable is required"
+            )
+        self._api = ProxmoxAPI(
+            api_url=proxmox_config.api_url,
+            token_id=proxmox_config.token_id,
+            token_secret=token_secret,
+            verify_ssl=proxmox_config.verify_ssl,
+        )
+
+    def create(
+        self,
+        vm_name: str,
+        config: Config,
+        *,
+        cpus: int | None = None,
+        memory: int | None = None,
+        disk: int | None = None,
+        admin_username: str = "agentworks",
+        tailscale_auth_key: str | None = None,
+    ) -> ProvisionResult:
+        node = self._cfg.node
+        template_vmid = self._cfg.template_vmid
+
+        output.info(f"Provisioning Proxmox VM '{vm_name}' on node {node}...")
+
+        # 1. Get next VMID
+        newid = self._api.next_id()
+        output.detail(f"Allocated VMID: {newid}")
+
+        # 2. Clone template into the agentworks pool
+        output.detail(f"Cloning template {template_vmid}...")
+        upid = self._api.clone_vm(
+            node, template_vmid, newid, vm_name,
+            storage=self._cfg.storage,
+            pool=self._cfg.pool,
+        )
+        self._api.wait_for_task(node, upid)
+        output.detail("Clone complete")
+
+        # 3. Configure VM resources
+        vm_config: dict[str, object] = {}
+        if cpus is not None:
+            vm_config["cores"] = cpus
+        if memory is not None:
+            vm_config["memory"] = memory * 1024  # GiB -> MiB
+
+        # Cloud-init: user, SSH key, network
+        ssh_pub_key = config.operator.ssh_public_key.read_text().strip()
+        vm_config["ciuser"] = admin_username
+        vm_config["sshkeys"] = urllib.parse.quote(ssh_pub_key, safe="")
+        vm_config["ipconfig0"] = "ip=dhcp"
+
+        # Boot order, guest agent, and CPU type (host passthrough exposes
+        # AVX/AVX2 which tools like Bun require)
+        vm_config["boot"] = "order=scsi0"
+        vm_config["agent"] = "enabled=1"
+        vm_config["cpu"] = "host"
+
+        output.detail("Configuring VM...")
+        self._api.configure_vm(node, newid, **vm_config)
+
+        # 4. Resize disk if requested
+        if disk is not None:
+            output.detail(f"Resizing disk to {disk}G...")
+            self._api.resize_disk(node, newid, "scsi0", f"{disk}G")
+
+        # 5. Start VM
+        output.detail("Starting VM...")
+        upid = self._api.start_vm(node, newid)
+        self._api.wait_for_task(node, upid)
+
+        # 6. Wait for guest agent and get VM IP
+        output.detail("Waiting for guest agent...")
+        ip = self._wait_for_guest_ip(node, newid)
+        output.detail(f"VM IP: {ip}")
+
+        # 7. Wait for cloud-init to finish (releases apt lock)
+        output.detail("Waiting for cloud-init...")
+        self._wait_for_cloud_init(node, newid)
+
+        # 8. Run bootstrap script via guest agent
+        bootstrap_complete = False
+        tailscale_ip: str | None = None
+        if tailscale_auth_key:
+            output.detail("Running bootstrap via guest agent...")
+            bootstrap = generate_bootstrap_script(
+                admin_username=admin_username,
+                ssh_public_key=ssh_pub_key,
+                provisioning_packages=PROVISIONING_PACKAGES,
+                tailscale_auth_key=tailscale_auth_key,
+                hostname=vm_hostname("proxmox", vm_name),
+                swap=config.vm.swap,
+            )
+            tailscale_ip = self._run_bootstrap_via_agent(node, newid, bootstrap)
+            bootstrap_complete = tailscale_ip is not None
+            if tailscale_ip:
+                output.detail(f"Tailscale IP: {tailscale_ip}")
+
+        host = tailscale_ip or ip
+        target = ExecTarget(
+            ssh=SSHTarget(
+                host=host,
+                user=admin_username,
+                identity_file=config.operator.ssh_private_key,
+            )
+        )
+
+        return ProvisionResult(
+            admin_exec_target=target,
+            proxmox_vmid=str(newid),
+            bootstrap_complete=bootstrap_complete,
+            tailscale_ip=tailscale_ip,
+        )
+
+    def start(self, vm: VMRow) -> None:
+        vmid = self._vmid(vm)
+        upid = self._api.start_vm(self._cfg.node, vmid)
+        self._api.wait_for_task(self._cfg.node, upid)
+
+    def stop(self, vm: VMRow) -> None:
+        vmid = self._vmid(vm)
+        upid = self._api.stop_vm(self._cfg.node, vmid)
+        self._api.wait_for_task(self._cfg.node, upid)
+
+    def delete(self, vm: VMRow) -> None:
+        vmid = self._vmid(vm)
+        node = self._cfg.node
+
+        # Stop if running
+        try:
+            status = self._api.vm_status(node, vmid)
+            if status.get("status") == "running":
+                upid = self._api.stop_vm(node, vmid)
+                self._api.wait_for_task(node, upid)
+        except ProxmoxAPIError:
+            pass  # VM may already be gone
+
+        # Delete VM
+        try:
+            upid = self._api.delete_vm(node, vmid)
+            self._api.wait_for_task(node, upid)
+        except ProxmoxAPIError:
+            pass  # best-effort
+
+    def status(self, vm: VMRow) -> VMStatus:
+        vmid = self._vmid(vm)
+        try:
+            result = self._api.vm_status(self._cfg.node, vmid)
+        except ProxmoxAPIError:
+            return VMStatus.UNKNOWN
+        pve_status = result.get("status", "")
+        if pve_status == "running":
+            return VMStatus.RUNNING
+        if pve_status == "stopped":
+            return VMStatus.STOPPED
+        return VMStatus.UNKNOWN
+
+    def admin_exec_target(self, vm: VMRow, *, config: object | None = None) -> ExecTarget:
+        raise NotImplementedError(
+            "Proxmox provisioning transport not yet implemented. "
+            "Requires QEMU guest agent exec integration. "
+            "See docs/sdd/2026-04-27-exec-target-cleanup/hla.md for details."
+        )
+
+    # -- Helpers ---------------------------------------------------------------
+
+    def _vmid(self, vm: VMRow) -> int:
+        if not vm.proxmox_vmid:
+            raise RuntimeError(f"VM '{vm.name}' has no proxmox_vmid")
+        return int(vm.proxmox_vmid)
+
+    def _wait_for_cloud_init(
+        self, node: str, vmid: int, *, timeout: int = 300
+    ) -> None:
+        """Wait for cloud-init to finish inside the VM."""
+        deadline = time.monotonic() + timeout
+        while time.monotonic() < deadline:
+            try:
+                result = self._api.guest_agent_exec_wait(
+                    node, vmid, "/usr/bin/cloud-init", ["status", "--wait"],
+                    timeout=60,
+                )
+                if result is not None and result.get("exitcode", -1) == 0:
+                    return
+            except ProxmoxAPIError:
+                pass
+            time.sleep(5)
+        # Don't fail -- cloud-init may not be installed or may have already finished
+
+    def _wait_for_guest_ip(
+        self, node: str, vmid: int, *, timeout: int = 120
+    ) -> str:
+        """Poll the guest agent until it reports a non-loopback IPv4 address."""
+        deadline = time.monotonic() + timeout
+        while time.monotonic() < deadline:
+            try:
+                interfaces = self._api.guest_agent_network(node, vmid)
+                for iface in interfaces:
+                    if iface.get("name") == "lo":
+                        continue
+                    for addr in iface.get("ip-addresses", []):
+                        if addr.get("ip-address-type") == "ipv4":
+                            ip = addr["ip-address"]
+                            if not ip.startswith("127."):
+                                return str(ip)
+            except ProxmoxAPIError:
+                pass  # guest agent not ready yet
+            time.sleep(3)
+        raise RuntimeError(
+            f"Timed out waiting for guest agent IP on VMID {vmid}"
+        )
+
+    def _run_bootstrap_via_agent(
+        self, node: str, vmid: int, script: str
+    ) -> str | None:
+        """Write and run the bootstrap script via the guest agent.
+
+        Returns the Tailscale IP if bootstrap succeeds, None otherwise.
+        """
+        from agentworks.vms.bootstrap_script import parse_bootstrap_output
+
+        # Write script to VM via guest agent file-write
+        self._api.guest_agent_file_write(
+            node, vmid, "/tmp/agentworks-bootstrap.sh", script
+        )
+
+        # Run bootstrap (long-running -- installs packages, joins tailscale)
+        # bash is invoked explicitly so the script doesn't need +x
+        result = self._api.guest_agent_exec_wait(
+            node, vmid, "/bin/bash", ["/tmp/agentworks-bootstrap.sh"],
+            timeout=600,
+        )
+
+        if result is None:
+            output.warn("bootstrap timed out")
+            return None
+
+        exit_code = result.get("exitcode", -1)
+        stdout = result.get("out-data", "")
+        parsed = parse_bootstrap_output(stdout, exit_code)
+
+        if parsed.ok:
+            return parsed.tailscale_ip
+
+        stderr = result.get("err-data", "")
+        if stderr:
+            output.warn(f"Bootstrap stderr: {stderr[:500]}")
+
+        return None