py-pve-cloud 0.13.22__tar.gz → 0.14.2rc14__tar.gz

{py_pve_cloud-0.13.22/src/py_pve_cloud.egg-info → py_pve_cloud-0.14.2rc14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: py-pve-cloud
-Version: 0.13.22
+Version: 0.14.2rc14
 Author-email: Tobias Huebner <tobias.huebner@vmzberlin.com>
 License-Expression: GPL-3.0-or-later
 License-File: LICENSE.md

{py_pve_cloud-0.13.22 → py_pve_cloud-0.14.2rc14}/README.md

@@ -18,4 +18,4 @@ PROXY_IP=$(ssh root@$PVE_HOST_IP cat /etc/pve/cloud/cluster_vars.yaml | yq '.pve
 export PG_CONN_STR=postgresql+psycopg2://postgres:$PATRONI_PASS@$PROXY_IP:5000/pve_cloud?sslmode=disable
 ```
 
-To create a new migration the database needs to be on the latest version, run `alembic upgrade head` to upgrade it.
+To create a new migration the database needs to be on the latest version, run `alembic upgrade head` to upgrade it.

py_pve_cloud-0.14.2rc14/src/pve_cloud/_version.py

@@ -0,0 +1 @@
+__version__ = "0.14.2rc14"

py_pve_cloud-0.14.2rc14/src/pve_cloud/cli/pvcli.py

@@ -0,0 +1,187 @@
+import argparse
+import os
+
+import paramiko
+import yaml
+from proxmoxer import ProxmoxAPI
+
+from pve_cloud.cli.pvclu import get_ssh_master_kubeconfig
+from pve_cloud.lib.inventory import *
+
+
+def connect_cluster(args):
+    # try load current dynamic inventory
+    inv_path = os.path.expanduser("~/.pve-cloud-dyn-inv.yaml")
+    if os.path.exists(inv_path):
+        with open(inv_path, "r") as file:
+            dynamic_inventory = yaml.safe_load(file)
+    else:
+        # initialize empty
+        dynamic_inventory = {}
+
+    # connect to the cluster via paramiko and check if cloud files are already there
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+    ssh.connect(args.pve_host, username="root")
+
+    # since we need root we cant use sftp and root via ssh is disabled
+    _, stdout, _ = ssh.exec_command("cat /etc/pve/cloud/cluster_vars.yaml")
+    cluster_vars = yaml.safe_load(stdout.read().decode("utf-8"))
+
+    if not cluster_vars:
+        # cluster has not been yet initialized
+        pve_cloud_domain = input(
+            "Cluster has not yet been fully initialized, assign the cluster a cloud domain and press ENTER:"
+        )
+    else:
+        pve_cloud_domain = cluster_vars["pve_cloud_domain"]
+
+    # init cloud domain if not there
+    if pve_cloud_domain not in dynamic_inventory:
+        dynamic_inventory[pve_cloud_domain] = {}
+
+    # connect to the passed host
+    proxmox = ProxmoxAPI(args.pve_host, user="root", backend="ssh_paramiko")
+
+    # try get the cluster name
+    cluster_name = None
+    status_resp = proxmox.cluster.status.get()
+    for entry in status_resp:
+        if entry["id"] == "cluster":
+            cluster_name = entry["name"]
+            break
+
+    if cluster_name is None:
+        raise Exception("Could not get cluster name")
+
+    if cluster_name in dynamic_inventory[pve_cloud_domain] and not args.force:
+        print(
+            f"cluster {cluster_name} already in dynamic inventory, add --force to overwrite current local inv."
+        )
+        return
+
+    # overwrite on force / create fresh
+    dynamic_inventory[pve_cloud_domain][cluster_name] = {}
+
+    # not present => add and safe the dynamic inventory
+    cluster_hosts = proxmox.nodes.get()
+
+    for node in cluster_hosts:
+        node_name = node["node"]
+
+        if node["status"] == "offline":
+            print(f"skipping offline node {node_name}")
+            continue
+
+        # get the main ip
+        ifaces = proxmox.nodes(node_name).network.get()
+        node_ip_address = None
+        for iface in ifaces:
+            if "gateway" in iface:
+                if node_ip_address is not None:
+                    raise Exception(
+                        f"found multiple ifaces with gateways for node {node_name}"
+                    )
+                node_ip_address = iface.get("address")
+
+        if node_ip_address is None:
+            raise Exception(f"Could not find ip for node {node_name}")
+
+        print(f"adding {node_name}")
+        dynamic_inventory[pve_cloud_domain][cluster_name][node_name] = {
+            "ansible_user": "root",
+            "ansible_host": node_ip_address,
+        }
+
+    print(f"writing dyn inv to {inv_path}")
+    with open(inv_path, "w") as file:
+        yaml.dump(dynamic_inventory, file)
+
+
+def print_kubeconfig(args):
+    if not os.path.exists(args.inventory):
+        print("The specified inventory file does not exist!")
+        return
+
+    with open(args.inventory, "r") as f:
+        inventory = yaml.safe_load(f)
+
+    target_pve = inventory["target_pve"]
+
+    target_cloud_domain = get_cloud_domain(target_pve)
+    pve_inventory = get_pve_inventory(target_cloud_domain)
+
+    # find target cluster in loaded inventory
+    target_cluster = None
+
+    for cluster in pve_inventory:
+        if target_pve.endswith((cluster + "." + target_cloud_domain)):
+            target_cluster = cluster
+            break
+
+    if not target_cluster:
+        print("could not find target cluster in pve inventory!")
+        return
+
+    first_host = list(pve_inventory[target_cluster].keys())[0]
+
+    # connect to the first pve host in the dyn inv, assumes they are all online
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect(
+        pve_inventory[target_cluster][first_host]["ansible_host"], username="root"
+    )
+
+    # since we need root we cant use sftp and root via ssh is disabled
+    _, stdout, _ = ssh.exec_command("cat /etc/pve/cloud/cluster_vars.yaml")
+
+    cluster_vars = yaml.safe_load(stdout.read().decode("utf-8"))
+
+    print(get_ssh_master_kubeconfig(cluster_vars, inventory["stack_name"]))
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="PVE general purpose cli for setting up."
+    )
+
+    base_parser = argparse.ArgumentParser(add_help=False)
+
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    connect_cluster_parser = subparsers.add_parser(
+        "connect-cluster",
+        help="Add an entire pve cluster to this machine for use.",
+        parents=[base_parser],
+    )
+    connect_cluster_parser.add_argument(
+        "--pve-host",
+        type=str,
+        help="PVE Host to connect to and add the entire cluster for the local machine.",
+        required=True,
+    )
+    connect_cluster_parser.add_argument(
+        "--force", action="store_true", help="Will read the cluster if set."
+    )
+    connect_cluster_parser.set_defaults(func=connect_cluster)
+
+    print_kconf_parser = subparsers.add_parser(
+        "print-kubeconfig",
+        help="Print the kubeconfig from a k8s cluster deployed with pve cloud.",
+        parents=[base_parser],
+    )
+    print_kconf_parser.add_argument(
+        "--inventory",
+        type=str,
+        help="PVE cloud kubespray inventory yaml file.",
+        required=True,
+    )
+    print_kconf_parser.set_defaults(func=print_kubeconfig)
+
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

py_pve_cloud-0.14.2rc14/src/pve_cloud/cli/pvclu.py

@@ -0,0 +1,146 @@
+import argparse
+import re
+
+import dns.resolver
+import paramiko
+import yaml
+
+from pve_cloud.lib.inventory import *
+
+
+def get_cluster_vars(pve_host):
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+    ssh.connect(pve_host, username="root")
+
+    # since we need root we cant use sftp and root via ssh is disabled
+    _, stdout, _ = ssh.exec_command("cat /etc/pve/cloud/cluster_vars.yaml")
+
+    cluster_vars = yaml.safe_load(stdout.read().decode("utf-8"))
+
+    return cluster_vars
+
+
+def get_cloud_env(pve_host):
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+    ssh.connect(pve_host, username="root")
+
+    # since we need root we cant use sftp and root via ssh is disabled
+    _, stdout, _ = ssh.exec_command("cat /etc/pve/cloud/cluster_vars.yaml")
+
+    cluster_vars = yaml.safe_load(stdout.read().decode("utf-8"))
+
+    _, stdout, _ = ssh.exec_command("cat /etc/pve/cloud/secrets/patroni.pass")
+
+    patroni_pass = stdout.read().decode("utf-8").strip()
+
+    # fetch bind update key for ingress dns validation
+    _, stdout, _ = ssh.exec_command("sudo cat /etc/pve/cloud/secrets/internal.key")
+    bind_key_file = stdout.read().decode("utf-8")
+
+    bind_internal_key = re.search(r'secret\s+"([^"]+)";', bind_key_file).group(1)
+
+    return cluster_vars, patroni_pass, bind_internal_key
+
+
+def get_online_pve_host_prsr(args):
+    print(
+        f"export PVE_ANSIBLE_HOST='{get_online_pve_host(args.target_pve, suppress_warnings=True)}'"
+    )
+
+
+def get_ssh_master_kubeconfig(cluster_vars, stack_name):
+    resolver = dns.resolver.Resolver()
+    resolver.nameservers = [
+        cluster_vars["bind_master_ip"],
+        cluster_vars["bind_slave_ip"],
+    ]
+
+    ddns_answer = resolver.resolve(
+        f"masters-{stack_name}.{cluster_vars['pve_cloud_domain']}"
+    )
+    ddns_ips = [rdata.to_text() for rdata in ddns_answer]
+
+    if not ddns_ips:
+        raise Exception("No master could be found via DNS!")
+
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+    ssh.connect(ddns_ips[0], username="admin")
+
+    # since we need root we cant use sftp and root via ssh is disabled
+    _, stdout, _ = ssh.exec_command("sudo cat /etc/kubernetes/admin.conf")
+
+    admin_conf = yaml.safe_load(stdout.read().decode("utf-8"))
+    # rewrite variables for external access
+    admin_conf["clusters"][0]["cluster"]["server"] = f"https://{ddns_ips[0]}:6443"
+    admin_conf["clusters"][0]["name"] = stack_name
+
+    admin_conf["contexts"][0]["context"]["cluster"] = stack_name
+    admin_conf["contexts"][0]["name"] = stack_name
+
+    admin_conf["current-context"] = stack_name
+
+    return yaml.safe_dump(admin_conf)
+
+
+def export_pg_conn_str(args):
+    cloud_domain = get_cloud_domain(args.target_pve, suppress_warnings=True)
+    pve_inventory = get_pve_inventory(cloud_domain, suppress_warnings=True)
+
+    # get ansible ip for first host in target cluster
+    ansible_host = None
+    for cluster in pve_inventory:
+        if args.target_pve.startswith(cluster):
+            ansible_host = next(iter(pve_inventory[cluster].values()))["ansible_host"]
+
+    if not ansible_host:
+        raise RuntimeError(f"Could not find online host for {args.target_pve}!")
+
+    cluster_vars, patroni_pass, bind_internal_key = get_cloud_env(ansible_host)
+
+    print(
+        f"export PG_CONN_STR=\"postgres://postgres:{patroni_pass}@{cluster_vars['pve_haproxy_floating_ip_internal']}:5000/tf_states?sslmode=disable\""
+    )
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="PVE Cloud utility cli. Should be called with bash eval."
+    )
+
+    base_parser = argparse.ArgumentParser(add_help=False)
+
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    export_envr_parser = subparsers.add_parser(
+        "export-psql", help="Export variables for k8s .envrc", parents=[base_parser]
+    )
+    export_envr_parser.add_argument(
+        "--target-pve", type=str, help="The target pve cluster.", required=True
+    )
+    export_envr_parser.set_defaults(func=export_pg_conn_str)
+
+    get_online_pve_host_parser = subparsers.add_parser(
+        "get-online-host",
+        help="Gets the ip for the first online proxmox host in the cluster.",
+        parents=[base_parser],
+    )
+    get_online_pve_host_parser.add_argument(
+        "--target-pve",
+        type=str,
+        help="The target pve cluster to get the first online ip of.",
+        required=True,
+    )
+    get_online_pve_host_parser.set_defaults(func=get_online_pve_host_prsr)
+
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

{py_pve_cloud-0.13.22 → py_pve_cloud-0.14.2rc14}/src/pve_cloud/lib/inventory.py

@@ -1,15 +1,22 @@
+import os
+import shutil
+import socket
 import subprocess
+
+import yaml
 from proxmoxer import ProxmoxAPI
+
 from pve_cloud.lib.validate import raise_on_py_cloud_missmatch
-import shutil
-import yaml
-import os
-import socket
 
 
-def get_cloud_domain(target_pve, suppress_warnings = False):
+def get_cloud_domain(target_pve, suppress_warnings=False):
     if shutil.which("avahi-browse"):
-        avahi_disc = subprocess.run(["avahi-browse", "-rpt", "_pxc._tcp"], stdout=subprocess.PIPE, text=True, check=True)
+        avahi_disc = subprocess.run(
+            ["avahi-browse", "-rpt", "_pxc._tcp"],
+            stdout=subprocess.PIPE,
+            text=True,
+            check=True,
+        )
         services = avahi_disc.stdout.splitlines()
 
         # find cloud domain hosts and get first online per proxmox cluster
@@ -22,7 +29,7 @@ def get_cloud_domain(target_pve, suppress_warnings = False):
                 cluster_name = None
 
                 for txt_arg in svc_args[9].split():
-                    txt_arg = txt_arg.replace('"', '')
+                    txt_arg = txt_arg.replace('"', "")
                     if txt_arg.startswith("cloud_domain"):
                         cloud_domain = txt_arg.split("=")[1]
 
@@ -30,16 +37,19 @@ def get_cloud_domain(target_pve, suppress_warnings = False):
                         cluster_name = txt_arg.split("=")[1]
 
                 if not cloud_domain or not cluster_name:
-                    raise ValueError(f"Missconfigured proxmox cloud avahi service: {service}")
-                
+                    raise ValueError(
+                        f"Missconfigured proxmox cloud avahi service: {service}"
+                    )
+
                 if target_pve.endswith(cloud_domain):
                     return cloud_domain
-                
 
         raise RuntimeError("Could not get cloud domain via avahi mdns!")
     else:
         if not suppress_warnings:
-            print("avahi-browse not available, falling back to local inventory file from pvcli connect-cluster!")
+            print(
+                "avahi-browse not available, falling back to local inventory file from pvcli connect-cluster!"
+            )
 
         with open(os.path.expanduser("~/.pve-cloud-dyn-inv.yaml"), "r") as f:
             pve_inventory = yaml.safe_load(f)
@@ -48,13 +58,18 @@ def get_cloud_domain(target_pve, suppress_warnings = False):
             for pve_cluster in pve_inventory[pve_cloud]:
                 if pve_cluster + "." + pve_cloud == target_pve:
                     return pve_cloud
-        
+
         raise Exception(f"Could not identify cloud domain for {target_pve}")
 
 
-def get_online_pve_host(target_pve, suppress_warnings = False, skip_py_cloud_check=False):
+def get_online_pve_host(target_pve, suppress_warnings=False, skip_py_cloud_check=False):
     if shutil.which("avahi-browse"):
-        avahi_disc = subprocess.run(["avahi-browse", "-rpt", "_pxc._tcp"], stdout=subprocess.PIPE, text=True, check=True)
+        avahi_disc = subprocess.run(
+            ["avahi-browse", "-rpt", "_pxc._tcp"],
+            stdout=subprocess.PIPE,
+            text=True,
+            check=True,
+        )
         services = avahi_disc.stdout.splitlines()
 
         for service in services:
@@ -67,7 +82,7 @@ def get_online_pve_host(target_pve, suppress_warnings = False, skip_py_cloud_che
                 cluster_name = None
 
                 for txt_arg in svc_args[9].split():
-                    txt_arg = txt_arg.replace('"', '')
+                    txt_arg = txt_arg.replace('"', "")
                     if txt_arg.startswith("cloud_domain"):
                         cloud_domain = txt_arg.split("=")[1]
 
@@ -75,19 +90,25 @@ def get_online_pve_host(target_pve, suppress_warnings = False, skip_py_cloud_che
                         cluster_name = txt_arg.split("=")[1]
 
                 if not cloud_domain or not cluster_name:
-                    raise ValueError(f"Missconfigured proxmox cloud avahi service: {service}")
-                
+                    raise ValueError(
+                        f"Missconfigured proxmox cloud avahi service: {service}"
+                    )
+
                 # main pve cloud inventory
                 if f"{cluster_name}.{cloud_domain}" == target_pve:
                     if not skip_py_cloud_check:
-                        raise_on_py_cloud_missmatch(host_ip) # validate that versions of dev machine and running on cluster match
+                        raise_on_py_cloud_missmatch(
+                            host_ip
+                        )  # validate that versions of dev machine and running on cluster match
 
                     return host_ip
-        
+
         raise RuntimeError(f"No online host found for {target_pve}!")
     else:
         if not suppress_warnings:
-            print("avahi-browse not available, falling back to local inventory file from pvcli connect-cluster!")
+            print(
+                "avahi-browse not available, falling back to local inventory file from pvcli connect-cluster!"
+            )
 
         with open(os.path.expanduser("~/.pve-cloud-dyn-inv.yaml"), "r") as f:
             pve_inventory = yaml.safe_load(f)
@@ -97,29 +118,39 @@ def get_online_pve_host(target_pve, suppress_warnings = False, skip_py_cloud_che
                 if pve_cluster + "." + pve_cloud == target_pve:
                     for pve_host in pve_inventory[pve_cloud][pve_cluster]:
                         # check if host is available
-                        pve_host_ip = pve_inventory[pve_cloud][pve_cluster][pve_host]["ansible_host"]
+                        pve_host_ip = pve_inventory[pve_cloud][pve_cluster][pve_host][
+                            "ansible_host"
+                        ]
                         try:
                             with socket.create_connection((pve_host_ip, 22), timeout=3):
-                                
+
                                 if not skip_py_cloud_check:
-                                    raise_on_py_cloud_missmatch(pve_host_ip) # validate that versions of dev machine and running on cluster match
+                                    raise_on_py_cloud_missmatch(
+                                        pve_host_ip
+                                    )  # validate that versions of dev machine and running on cluster match
 
                                 return pve_host_ip
                         except Exception as e:
                             # debug
                             print(e, type(e))
-                            pass
-            
+
         raise RuntimeError(f"Could not find online pve host for {target_pve}")
 
 
-def get_pve_inventory(pve_cloud_domain, suppress_warnings = False, skip_py_cloud_check=False):
+def get_pve_inventory(
+    pve_cloud_domain, suppress_warnings=False, skip_py_cloud_check=False
+):
     if shutil.which("avahi-browse"):
         # avahi is available
 
         # call avahi-browse -rpt _pxc._tcp and find online host matching pve cloud domain
         # connect via ssh and fetch all other hosts via proxmox api => build inventory
-        avahi_disc = subprocess.run(["avahi-browse", "-rpt", "_pxc._tcp"], stdout=subprocess.PIPE, text=True, check=True)
+        avahi_disc = subprocess.run(
+            ["avahi-browse", "-rpt", "_pxc._tcp"],
+            stdout=subprocess.PIPE,
+            text=True,
+            check=True,
+        )
         services = avahi_disc.stdout.splitlines()
 
         pve_inventory = {}
@@ -138,7 +169,7 @@ def get_pve_inventory(pve_cloud_domain, suppress_warnings = False, skip_py_cloud
                 cluster_name = None
 
                 for txt_arg in svc_args[9].split():
-                    txt_arg = txt_arg.replace('"', '')
+                    txt_arg = txt_arg.replace('"', "")
                     if txt_arg.startswith("cloud_domain"):
                         cloud_domain = txt_arg.split("=")[1]
 
@@ -146,36 +177,49 @@ def get_pve_inventory(pve_cloud_domain, suppress_warnings = False, skip_py_cloud
                         cluster_name = txt_arg.split("=")[1]
 
                 if not cloud_domain or not cluster_name:
-                    raise ValueError(f"Missconfigured proxmox cloud avahi service: {service}")
-                
+                    raise ValueError(
+                        f"Missconfigured proxmox cloud avahi service: {service}"
+                    )
+
                 # main pve cloud inventory
-                if cloud_domain == pve_cloud_domain and cluster_name not in cloud_domain_first_hosts:
-                    if not skip_py_cloud_check and f"{cluster_name}.{cloud_domain}" not in py_pve_cloud_performed_version_checks:
-                        raise_on_py_cloud_missmatch(host_ip) # validate that versions of dev machine and running on cluster match
-                        py_pve_cloud_performed_version_checks.add(f"{cluster_name}.{cloud_domain}") # perform version check only once per cluster
+                if (
+                    cloud_domain == pve_cloud_domain
+                    and cluster_name not in cloud_domain_first_hosts
+                ):
+                    if (
+                        not skip_py_cloud_check
+                        and f"{cluster_name}.{cloud_domain}"
+                        not in py_pve_cloud_performed_version_checks
+                    ):
+                        raise_on_py_cloud_missmatch(
+                            host_ip
+                        )  # validate that versions of dev machine and running on cluster match
+                        py_pve_cloud_performed_version_checks.add(
+                            f"{cluster_name}.{cloud_domain}"
+                        )  # perform version check only once per cluster
 
                     cloud_domain_first_hosts[cluster_name] = host_ip
-                
+
         # iterate over hosts and build pve inv via proxmox api
         # todo: this needs to be hugely optimized it blocks the grpc server
         for cluster_first, first_host in cloud_domain_first_hosts.items():
-            proxmox = ProxmoxAPI(
-                first_host, user="root", backend='ssh_paramiko'
-            )
+            proxmox = ProxmoxAPI(first_host, user="root", backend="ssh_paramiko")
 
             cluster_name = None
             status_resp = proxmox.cluster.status.get()
             for entry in status_resp:
-                if entry['id'] == "cluster":
-                    cluster_name = entry['name']
+                if entry["id"] == "cluster":
+                    cluster_name = entry["name"]
                 break
 
             if cluster_name is None:
                 raise RuntimeError("Could not get cluster name")
-            
+
             if cluster_name != cluster_first:
-                raise ValueError(f"Proxmox cluster name missconfigured in avahi service {cluster_name}/{cluster_first}")
-            
+                raise ValueError(
+                    f"Proxmox cluster name missconfigured in avahi service {cluster_name}/{cluster_first}"
+                )
+
             pve_inventory[cluster_name] = {}
 
             # fetch other hosts via api
@@ -187,41 +231,46 @@ def get_pve_inventory(pve_cloud_domain, suppress_warnings = False, skip_py_cloud
                 if node["status"] == "offline":
                     print(f"skipping offline node {node_name}")
                     continue
-                
+
                 # get the main ip
                 ifaces = proxmox.nodes(node_name).network.get()
                 node_ip_address = None
                 for iface in ifaces:
-                    if 'gateway' in iface:
+                    if "gateway" in iface:
                         if node_ip_address is not None:
-                            raise RuntimeError(f"found multiple ifaces with gateways for node {node_name}")
+                            raise RuntimeError(
+                                f"found multiple ifaces with gateways for node {node_name}"
+                            )
                         node_ip_address = iface.get("address")
 
                 if node_ip_address is None:
                     raise RuntimeError(f"Could not find ip for node {node_name}")
-                
+
                 pve_inventory[cluster_name][node_name] = {
                     "ansible_user": "root",
-                    "ansible_host": node_ip_address
+                    "ansible_host": node_ip_address,
                 }
-                        
 
         return pve_inventory
 
     else:
         if not suppress_warnings:
-            print("avahi-browse not available, falling back to local inventory file from pvcli connect-cluster!")
+            print(
+                "avahi-browse not available, falling back to local inventory file from pvcli connect-cluster!"
+            )
         # try load fallback manual inventory from disk
         inv_path = os.path.expanduser("~/.pve-cloud-dyn-inv.yaml")
         if not os.path.exists(inv_path):
-            raise RuntimeError("Local pve inventory file missing (~/.pve-cloud-dyn-inv.yaml), execute `pvcli connect-cluster` or setup avahi mdns discovery!")
+            raise RuntimeError(
+                "Local pve inventory file missing (~/.pve-cloud-dyn-inv.yaml), execute `pvcli connect-cluster` or setup avahi mdns discovery!"
+            )
 
         with open(inv_path, "r") as file:
             dynamic_inventory = yaml.safe_load(file)
 
         if pve_cloud_domain not in dynamic_inventory:
-            raise RuntimeError(f"{pve_cloud_domain} not in local dynamic inventory (~/.pve-cloud-dyn-inv.yaml created by `pvcli connect-cluster`)!")
+            raise RuntimeError(
+                f"{pve_cloud_domain} not in local dynamic inventory (~/.pve-cloud-dyn-inv.yaml created by `pvcli connect-cluster`)!"
+            )
 
         return dynamic_inventory[pve_cloud_domain]
-
-   

{py_pve_cloud-0.13.22 → py_pve_cloud-0.14.2rc14}/src/pve_cloud/lib/validate.py

@@ -1,7 +1,8 @@
+import os
+
 import paramiko
-import yaml
 import pve_cloud._version
-import os
+import yaml
 
 
 def raise_on_py_cloud_missmatch(proxmox_host):
@@ -16,7 +17,9 @@ def raise_on_py_cloud_missmatch(proxmox_host):
     # since we need root we cant use sftp and root via ssh is disabled
     _, stdout, _ = ssh.exec_command("cat /etc/pve/cloud/cluster_vars.yaml")
 
-    cluster_vars = yaml.safe_load(stdout.read().decode('utf-8'))
-  
+    cluster_vars = yaml.safe_load(stdout.read().decode("utf-8"))
+
     if cluster_vars["py_pve_cloud_version"] != pve_cloud._version.__version__:
-        raise RuntimeError(f"Version missmatch! py_pve_cloud_version for cluster is {cluster_vars['py_pve_cloud_version']}, while you are using {pve_cloud._version.__version__}")
+        raise RuntimeError(
+            f"Version missmatch! py_pve_cloud_version for cluster is {cluster_vars['py_pve_cloud_version']}, while you are using {pve_cloud._version.__version__}"
+        )

{py_pve_cloud-0.13.22 → py_pve_cloud-0.14.2rc14/src/py_pve_cloud.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: py-pve-cloud
-Version: 0.13.22
+Version: 0.14.2rc14
 Author-email: Tobias Huebner <tobias.huebner@vmzberlin.com>
 License-Expression: GPL-3.0-or-later
 License-File: LICENSE.md

py_pve_cloud-0.13.22/src/pve_cloud/_version.py

@@ -1 +0,0 @@
-__version__ = "0.13.22"

py_pve_cloud-0.13.22/src/pve_cloud/cli/pvcli.py

@@ -1,158 +0,0 @@
-import argparse
-import yaml
-from proxmoxer import ProxmoxAPI
-import os
-import paramiko
-from pve_cloud.cli.pvclu import get_ssh_master_kubeconfig
-from pve_cloud.lib.inventory import *
-
-
-
-def connect_cluster(args):
-  # try load current dynamic inventory
-  inv_path = os.path.expanduser("~/.pve-cloud-dyn-inv.yaml")
-  if os.path.exists(inv_path):
-    with open(inv_path, "r") as file:
-      dynamic_inventory = yaml.safe_load(file)
-  else:
-    # initialize empty
-    dynamic_inventory = {}
-
-  # connect to the cluster via paramiko and check if cloud files are already there
-  ssh = paramiko.SSHClient()
-  ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-
-  ssh.connect(args.pve_host, username="root")
-
-  # since we need root we cant use sftp and root via ssh is disabled
-  _, stdout, _ = ssh.exec_command("cat /etc/pve/cloud/cluster_vars.yaml")
-  cluster_vars = yaml.safe_load(stdout.read().decode('utf-8'))
-
-  if not cluster_vars:
-    # cluster has not been yet initialized
-    pve_cloud_domain = input("Cluster has not yet been fully initialized, assign the cluster a cloud domain and press ENTER:")
-  else:
-    pve_cloud_domain = cluster_vars["pve_cloud_domain"]
-  
-  # init cloud domain if not there
-  if pve_cloud_domain not in dynamic_inventory:
-    dynamic_inventory[pve_cloud_domain] = {}
-
-  # connect to the passed host
-  proxmox = ProxmoxAPI(
-    args.pve_host, user="root", backend='ssh_paramiko'
-  )
-
-  # try get the cluster name
-  cluster_name = None
-  status_resp = proxmox.cluster.status.get()
-  for entry in status_resp:
-    if entry['id'] == "cluster":
-      cluster_name = entry['name']
-      break
-
-  if cluster_name is None:
-    raise Exception("Could not get cluster name")
-  
-  if cluster_name in dynamic_inventory[pve_cloud_domain] and not args.force:
-    print(f"cluster {cluster_name} already in dynamic inventory, add --force to overwrite current local inv.")
-    return
-  
-  # overwrite on force / create fresh
-  dynamic_inventory[pve_cloud_domain][cluster_name] = {}
-  
-  # not present => add and safe the dynamic inventory
-  cluster_hosts = proxmox.nodes.get()
-
-  for node in cluster_hosts:
-    node_name = node["node"]
-
-    if node["status"] == "offline":
-      print(f"skipping offline node {node_name}")
-      continue
-    
-    # get the main ip
-    ifaces = proxmox.nodes(node_name).network.get()
-    node_ip_address = None
-    for iface in ifaces:
-      if 'gateway' in iface:
-        if node_ip_address is not None:
-          raise Exception(f"found multiple ifaces with gateways for node {node_name}")
-        node_ip_address = iface.get("address")
-
-    if node_ip_address is None:
-      raise Exception(f"Could not find ip for node {node_name}")
-    
-    print(f"adding {node_name}")
-    dynamic_inventory[pve_cloud_domain][cluster_name][node_name] = {
-      "ansible_user": "root",
-      "ansible_host": node_ip_address
-    }
-
-  print(f"writing dyn inv to {inv_path}")
-  with open(inv_path, "w") as file:
-    yaml.dump(dynamic_inventory, file)
-
-
-def print_kubeconfig(args):
-  if not os.path.exists(args.inventory):
-    print("The specified inventory file does not exist!")
-    return
-
-  with open(args.inventory, "r") as f:
-    inventory = yaml.safe_load(f)
-
-  target_pve = inventory["target_pve"]
-
-  target_cloud_domain = get_cloud_domain(target_pve)
-  pve_inventory = get_pve_inventory(target_cloud_domain)
-
-  # find target cluster in loaded inventory
-  target_cluster = None
-
-  for cluster in pve_inventory:
-    if target_pve.endswith((cluster + "." + target_cloud_domain)):
-      target_cluster = cluster
-      break
-
-  if not target_cluster:
-    print("could not find target cluster in pve inventory!")
-    return
-
-  first_host = list(pve_inventory[target_cluster].keys())[0]
-
-  # connect to the first pve host in the dyn inv, assumes they are all online
-  ssh = paramiko.SSHClient()
-  ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-  ssh.connect(pve_inventory[target_cluster][first_host]["ansible_host"], username="root")
-
-  # since we need root we cant use sftp and root via ssh is disabled
-  _, stdout, _ = ssh.exec_command("cat /etc/pve/cloud/cluster_vars.yaml")
-
-  cluster_vars = yaml.safe_load(stdout.read().decode('utf-8'))
-
-  print(get_ssh_master_kubeconfig(cluster_vars, inventory["stack_name"]))
-
-
-def main():
-  parser = argparse.ArgumentParser(description="PVE general purpose cli for setting up.")
-
-  base_parser = argparse.ArgumentParser(add_help=False)
-
-  subparsers = parser.add_subparsers(dest="command", required=True)
-
-  connect_cluster_parser = subparsers.add_parser("connect-cluster", help="Add an entire pve cluster to this machine for use.", parents=[base_parser])
-  connect_cluster_parser.add_argument("--pve-host", type=str, help="PVE Host to connect to and add the entire cluster for the local machine.", required=True)
-  connect_cluster_parser.add_argument("--force", action="store_true", help="Will read the cluster if set.")
-  connect_cluster_parser.set_defaults(func=connect_cluster)
-
-  print_kconf_parser = subparsers.add_parser("print-kubeconfig", help="Print the kubeconfig from a k8s cluster deployed with pve cloud.", parents=[base_parser])
-  print_kconf_parser.add_argument("--inventory", type=str, help="PVE cloud kubespray inventory yaml file.", required=True)
-  print_kconf_parser.set_defaults(func=print_kubeconfig)
-
-  args = parser.parse_args()
-  args.func(args)
-
-
-if __name__ == "__main__":
-  main()

py_pve_cloud-0.13.22/src/pve_cloud/cli/pvclu.py

@@ -1,122 +0,0 @@
-import argparse
-import yaml 
-import pprint
-import paramiko
-import dns.resolver
-import base64
-import re
-from pve_cloud.lib.inventory import *
-
-
-def get_cluster_vars(pve_host):
-  ssh = paramiko.SSHClient()
-  ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-
-  ssh.connect(pve_host, username="root")
-
-  # since we need root we cant use sftp and root via ssh is disabled
-  _, stdout, _ = ssh.exec_command("cat /etc/pve/cloud/cluster_vars.yaml")
-
-  cluster_vars = yaml.safe_load(stdout.read().decode('utf-8'))
-
-  return cluster_vars
-
-
-def get_cloud_env(pve_host):
-  ssh = paramiko.SSHClient()
-  ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-
-  ssh.connect(pve_host, username="root")
-
-  # since we need root we cant use sftp and root via ssh is disabled
-  _, stdout, _ = ssh.exec_command("cat /etc/pve/cloud/cluster_vars.yaml")
-
-  cluster_vars = yaml.safe_load(stdout.read().decode('utf-8'))
-
-  _, stdout, _ = ssh.exec_command("cat /etc/pve/cloud/secrets/patroni.pass")
-
-  patroni_pass = stdout.read().decode('utf-8').strip()
-
-  # fetch bind update key for ingress dns validation
-  _, stdout, _ = ssh.exec_command("sudo cat /etc/pve/cloud/secrets/internal.key")
-  bind_key_file = stdout.read().decode('utf-8')
-
-  bind_internal_key = re.search(r'secret\s+"([^"]+)";', bind_key_file).group(1)
-
-  return cluster_vars, patroni_pass, bind_internal_key
-
-
-def get_online_pve_host_prsr(args):
-  print(f"export PVE_ANSIBLE_HOST='{get_online_pve_host(args.target_pve, suppress_warnings=True)}'")
-
-
-def get_ssh_master_kubeconfig(cluster_vars, stack_name):
-  resolver = dns.resolver.Resolver()
-  resolver.nameservers = [cluster_vars['bind_master_ip'], cluster_vars['bind_slave_ip']]
-
-  ddns_answer = resolver.resolve(f"masters-{stack_name}.{cluster_vars['pve_cloud_domain']}")
-  ddns_ips = [rdata.to_text() for rdata in ddns_answer]
-
-  if not ddns_ips:
-    raise Exception("No master could be found via DNS!")
-
-  ssh = paramiko.SSHClient()
-  ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-
-  ssh.connect(ddns_ips[0], username="admin")
-
-  # since we need root we cant use sftp and root via ssh is disabled
-  _, stdout, _ = ssh.exec_command("sudo cat /etc/kubernetes/admin.conf")
-
-  admin_conf = yaml.safe_load(stdout.read().decode('utf-8'))
-  # rewrite variables for external access
-  admin_conf["clusters"][0]["cluster"]["server"] = f"https://{ddns_ips[0]}:6443"
-  admin_conf["clusters"][0]["name"] = stack_name
-
-  admin_conf["contexts"][0]["context"]["cluster"] = stack_name
-  admin_conf["contexts"][0]["name"] = stack_name
-
-  admin_conf["current-context"] = stack_name
-
-  return yaml.safe_dump(admin_conf)
-
-
-def export_pg_conn_str(args):
-  cloud_domain = get_cloud_domain(args.target_pve, suppress_warnings=True)
-  pve_inventory = get_pve_inventory(cloud_domain, suppress_warnings=True)
-
-  # get ansible ip for first host in target cluster
-  ansible_host = None
-  for cluster in pve_inventory:
-    if args.target_pve.startswith(cluster):
-      ansible_host = next(iter(pve_inventory[cluster].values()))["ansible_host"]
-
-  if not ansible_host:
-    raise RuntimeError(f"Could not find online host for {args.target_pve}!")
-
-  cluster_vars, patroni_pass, bind_internal_key = get_cloud_env(ansible_host)
-
-  print(f"export PG_CONN_STR=\"postgres://postgres:{patroni_pass}@{cluster_vars['pve_haproxy_floating_ip_internal']}:5000/tf_states?sslmode=disable\"")
-
-
-def main():
-  parser = argparse.ArgumentParser(description="PVE Cloud utility cli. Should be called with bash eval.")
-
-  base_parser = argparse.ArgumentParser(add_help=False)
-
-  subparsers = parser.add_subparsers(dest="command", required=True)
-
-  export_envr_parser = subparsers.add_parser("export-psql", help="Export variables for k8s .envrc", parents=[base_parser])
-  export_envr_parser.add_argument("--target-pve", type=str, help="The target pve cluster.", required=True)
-  export_envr_parser.set_defaults(func=export_pg_conn_str)
-
-  get_online_pve_host_parser = subparsers.add_parser("get-online-host", help="Gets the ip for the first online proxmox host in the cluster.", parents=[base_parser])
-  get_online_pve_host_parser.add_argument("--target-pve", type=str, help="The target pve cluster to get the first online ip of.", required=True)
-  get_online_pve_host_parser.set_defaults(func=get_online_pve_host_prsr)
-
-  args = parser.parse_args()
-  args.func(args)
-
-
-if __name__ == "__main__":
-  main()