py-pve-cloud 0.5.16__py3-none-any.whl → 0.14.2rc26__py3-none-any.whl

pve_cloud/_version.py

@@ -1 +1 @@
-__version__ = "0.5.16"
+__version__ = "0.14.2rc26"

pve_cloud/cli/pvcli.py

@@ -1,159 +1,187 @@
 import argparse
-import yaml
-from proxmoxer import ProxmoxAPI
 import os
+
 import paramiko
+import yaml
+from proxmoxer import ProxmoxAPI
+
 from pve_cloud.cli.pvclu import get_ssh_master_kubeconfig
+from pve_cloud.lib.inventory import *
 
 
 def connect_cluster(args):
-  # try load current dynamic inventory
-  inv_path = os.path.expanduser("~/.pve-cloud-dyn-inv.yaml")
-  if os.path.exists(inv_path):
-    with open(inv_path, "r") as file:
-      dynamic_inventory = yaml.safe_load(file)
-  else:
-    # initialize empty
-    dynamic_inventory = {}
-
-  # connect to the cluster via paramiko and check if cloud files are already there
-  ssh = paramiko.SSHClient()
-  ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-
-  ssh.connect(args.pve_host, username="root")
-
-  # since we need root we cant use sftp and root via ssh is disabled
-  _, stdout, _ = ssh.exec_command("cat /etc/pve/cloud/cluster_vars.yaml")
-  cluster_vars = yaml.safe_load(stdout.read().decode('utf-8'))
-
-  if not cluster_vars:
-    # cluster has not been yet initialized
-    pve_cloud_domain = input("Cluster has not yet been fully initialized, assign the cluster a cloud domain and press ENTER:")
-  else:
-    pve_cloud_domain = cluster_vars["pve_cloud_domain"]
-  
-  # init cloud domain if not there
-  if pve_cloud_domain not in dynamic_inventory:
-    dynamic_inventory[pve_cloud_domain] = {}
-
-  # connect to the passed host
-  proxmox = ProxmoxAPI(
-    args.pve_host, user="root", backend='ssh_paramiko'
-  )
-
-  # try get the cluster name
-  cluster_name = None
-  status_resp = proxmox.cluster.status.get()
-  for entry in status_resp:
-    if entry['id'] == "cluster":
-      cluster_name = entry['name']
-      break
-
-  if cluster_name is None:
-    raise Exception("Could not get cluster name")
-  
-  if cluster_name in dynamic_inventory[pve_cloud_domain] and not args.force:
-    print(f"cluster {cluster_name} already in dynamic inventory, add --force to overwrite current local inv.")
-    return
-  
-  # overwrite on force / create fresh
-  dynamic_inventory[pve_cloud_domain][cluster_name] = {}
-  
-  # not present => add and safe the dynamic inventory
-  cluster_hosts = proxmox.nodes.get()
-
-  for node in cluster_hosts:
-    node_name = node["node"]
-
-    if node["status"] == "offline":
-      print(f"skipping offline node {node_name}")
-      continue
-    
-    # get the main ip
-    ifaces = proxmox.nodes(node_name).network.get()
-    node_ip_address = None
-    for iface in ifaces:
-      if 'gateway' in iface:
-        if node_ip_address is not None:
-          raise Exception(f"found multiple ifaces with gateways for node {node_name}")
-        node_ip_address = iface.get("address")
-
-    if node_ip_address is None:
-      raise Exception(f"Could not find ip for node {node_name}")
-    
-    dynamic_inventory[pve_cloud_domain][cluster_name][node_name] = {
-      "ansible_user": "root",
-      "ansible_host": node_ip_address
-    }
-
-  with open(inv_path, "w") as file:
-    yaml.dump(dynamic_inventory, file)
+    # try load current dynamic inventory
+    inv_path = os.path.expanduser("~/.pve-cloud-dyn-inv.yaml")
+    if os.path.exists(inv_path):
+        with open(inv_path, "r") as file:
+            dynamic_inventory = yaml.safe_load(file)
+    else:
+        # initialize empty
+        dynamic_inventory = {}
+
+    # connect to the cluster via paramiko and check if cloud files are already there
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+    ssh.connect(args.pve_host, username="root")
+
+    # since we need root we cant use sftp and root via ssh is disabled
+    _, stdout, _ = ssh.exec_command("cat /etc/pve/cloud/cluster_vars.yaml")
+    cluster_vars = yaml.safe_load(stdout.read().decode("utf-8"))
+
+    if not cluster_vars:
+        # cluster has not been yet initialized
+        pve_cloud_domain = input(
+            "Cluster has not yet been fully initialized, assign the cluster a cloud domain and press ENTER:"
+        )
+    else:
+        pve_cloud_domain = cluster_vars["pve_cloud_domain"]
+
+    # init cloud domain if not there
+    if pve_cloud_domain not in dynamic_inventory:
+        dynamic_inventory[pve_cloud_domain] = {}
+
+    # connect to the passed host
+    proxmox = ProxmoxAPI(args.pve_host, user="root", backend="ssh_paramiko")
+
+    # try get the cluster name
+    cluster_name = None
+    status_resp = proxmox.cluster.status.get()
+    for entry in status_resp:
+        if entry["id"] == "cluster":
+            cluster_name = entry["name"]
+            break
+
+    if cluster_name is None:
+        raise Exception("Could not get cluster name")
+
+    if cluster_name in dynamic_inventory[pve_cloud_domain] and not args.force:
+        print(
+            f"cluster {cluster_name} already in dynamic inventory, add --force to overwrite current local inv."
+        )
+        return
+
+    # overwrite on force / create fresh
+    dynamic_inventory[pve_cloud_domain][cluster_name] = {}
+
+    # not present => add and safe the dynamic inventory
+    cluster_hosts = proxmox.nodes.get()
+
+    for node in cluster_hosts:
+        node_name = node["node"]
+
+        if node["status"] == "offline":
+            print(f"skipping offline node {node_name}")
+            continue
+
+        # get the main ip
+        ifaces = proxmox.nodes(node_name).network.get()
+        node_ip_address = None
+        for iface in ifaces:
+            if "gateway" in iface:
+                if node_ip_address is not None:
+                    raise Exception(
+                        f"found multiple ifaces with gateways for node {node_name}"
+                    )
+                node_ip_address = iface.get("address")
+
+        if node_ip_address is None:
+            raise Exception(f"Could not find ip for node {node_name}")
+
+        print(f"adding {node_name}")
+        dynamic_inventory[pve_cloud_domain][cluster_name][node_name] = {
+            "ansible_user": "root",
+            "ansible_host": node_ip_address,
+        }
+
+    print(f"writing dyn inv to {inv_path}")
+    with open(inv_path, "w") as file:
+        yaml.dump(dynamic_inventory, file)
 
 
 def print_kubeconfig(args):
-  if not os.path.exists(args.inventory):
-    print("The specified inventory file does not exist!")
-    return
-
-  with open(args.inventory, "r") as f:
-    inventory = yaml.safe_load(f)
+    if not os.path.exists(args.inventory):
+        print("The specified inventory file does not exist!")
+        return
 
-  target_pve = inventory["target_pve"]
+    with open(args.inventory, "r") as f:
+        inventory = yaml.safe_load(f)
 
-  # load dyn inv to init proxmoxer
-  inv_path = os.path.expanduser("~/.pve-cloud-dyn-inv.yaml")
-  with open(inv_path, "r") as file:
-    dynamic_inventory = yaml.safe_load(file)
+    target_pve = inventory["target_pve"]
 
-  target_cloud = None
-  target_cluster = None
-  for cloud in dynamic_inventory:
-    for cluster in dynamic_inventory[cloud]:
-      if target_pve.endswith((cluster + "." + cloud)):
-        target_cloud = cloud
-        target_cluster = cluster
-        break
+    target_cloud_domain = get_cloud_domain(target_pve)
+    pve_inventory = get_pve_inventory(target_cloud_domain)
 
-  if not target_cloud:
-    print("could not find cloud in dyn inv!")
-    return
+    # find target cluster in loaded inventory
+    target_cluster = None
 
-  first_host = list(dynamic_inventory[target_cloud][target_cluster].keys())[0]
+    for cluster in pve_inventory:
+        if target_pve.endswith((cluster + "." + target_cloud_domain)):
+            target_cluster = cluster
+            break
 
-  # connect to the first pve host in the dyn inv, assumes they are all online
-  ssh = paramiko.SSHClient()
-  ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-  ssh.connect(dynamic_inventory[target_cloud][target_cluster][first_host]["ansible_host"], username="root")
+    if not target_cluster:
+        print("could not find target cluster in pve inventory!")
+        return
 
-  # since we need root we cant use sftp and root via ssh is disabled
-  _, stdout, _ = ssh.exec_command("cat /etc/pve/cloud/cluster_vars.yaml")
+    first_host = list(pve_inventory[target_cluster].keys())[0]
 
-  cluster_vars = yaml.safe_load(stdout.read().decode('utf-8'))
+    # connect to the first pve host in the dyn inv, assumes they are all online
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect(
+        pve_inventory[target_cluster][first_host]["ansible_host"], username="root"
+    )
 
-  print(get_ssh_master_kubeconfig(cluster_vars, inventory["stack_name"]))
+    # since we need root we cant use sftp and root via ssh is disabled
+    _, stdout, _ = ssh.exec_command("cat /etc/pve/cloud/cluster_vars.yaml")
 
+    cluster_vars = yaml.safe_load(stdout.read().decode("utf-8"))
 
-def main():
-  parser = argparse.ArgumentParser(description="PVE general purpose cli for setting up.")
-
-  base_parser = argparse.ArgumentParser(add_help=False)
-
-  subparsers = parser.add_subparsers(dest="command", required=True)
-
-  connect_cluster_parser = subparsers.add_parser("connect-cluster", help="Add an entire pve cluster to this machine for use.", parents=[base_parser])
-  connect_cluster_parser.add_argument("--pve-host", type=str, help="PVE Host to connect to and add the entire cluster for the local machine.", required=True)
-  connect_cluster_parser.add_argument("--force", action="store_true", help="Will read the cluster if set.")
-  connect_cluster_parser.set_defaults(func=connect_cluster)
+    print(get_ssh_master_kubeconfig(cluster_vars, inventory["stack_name"]))
 
 
-  print_kconf_parser = subparsers.add_parser("print-kubeconfig", help="Print the kubeconfig from a k8s cluster deployed with pve cloud.", parents=[base_parser])
-  print_kconf_parser.add_argument("--inventory", type=str, help="PVE cloud kubespray inventory yaml file.", required=True)
-  print_kconf_parser.set_defaults(func=print_kubeconfig)
-
-
-  args = parser.parse_args()
-  args.func(args)
+def main():
+    parser = argparse.ArgumentParser(
+        description="PVE general purpose cli for setting up."
+    )
+
+    base_parser = argparse.ArgumentParser(add_help=False)
+
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    connect_cluster_parser = subparsers.add_parser(
+        "connect-cluster",
+        help="Add an entire pve cluster to this machine for use.",
+        parents=[base_parser],
+    )
+    connect_cluster_parser.add_argument(
+        "--pve-host",
+        type=str,
+        help="PVE Host to connect to and add the entire cluster for the local machine.",
+        required=True,
+    )
+    connect_cluster_parser.add_argument(
+        "--force", action="store_true", help="Will read the cluster if set."
+    )
+    connect_cluster_parser.set_defaults(func=connect_cluster)
+
+    print_kconf_parser = subparsers.add_parser(
+        "print-kubeconfig",
+        help="Print the kubeconfig from a k8s cluster deployed with pve cloud.",
+        parents=[base_parser],
+    )
+    print_kconf_parser.add_argument(
+        "--inventory",
+        type=str,
+        help="PVE cloud kubespray inventory yaml file.",
+        required=True,
+    )
+    print_kconf_parser.set_defaults(func=print_kubeconfig)
+
+    args = parser.parse_args()
+    args.func(args)
 
 
 if __name__ == "__main__":
-  main()
+    main()

pve_cloud/cli/pvclu.py

@@ -1,140 +1,146 @@
 import argparse
-import yaml
-import os 
-import socket
-import paramiko
-import dns.resolver
-import base64
 import re
 
+import dns.resolver
+import paramiko
+import yaml
 
-def get_cloud_domain(target_pve):
-  with open(os.path.expanduser("~/.pve-cloud-dyn-inv.yaml"), "r") as f:
-    pve_inventory = yaml.safe_load(f)
+from pve_cloud.lib.inventory import *
 
-  for pve_cloud in pve_inventory:
-    for pve_cluster in pve_inventory[pve_cloud]:
-      if pve_cluster + "." + pve_cloud == target_pve:
-        return pve_cloud
-  
-  raise Exception(f"Could not identify cloud domain for {target_pve}")
 
+def get_cluster_vars(pve_host):
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 
-def get_cld_domain_prsr(args):
-  print(f"export PVE_CLOUD_DOMAIN='{get_cloud_domain(args.target_pve)}'")
+    ssh.connect(pve_host, username="root")
 
+    # since we need root we cant use sftp and root via ssh is disabled
+    _, stdout, _ = ssh.exec_command("cat /etc/pve/cloud/cluster_vars.yaml")
 
-def get_online_pve_host(target_pve):
-  with open(os.path.expanduser("~/.pve-cloud-dyn-inv.yaml"), "r") as f:
-    pve_inventory = yaml.safe_load(f)
+    cluster_vars = yaml.safe_load(stdout.read().decode("utf-8"))
 
-  for pve_cloud in pve_inventory:
-    for pve_cluster in pve_inventory[pve_cloud]:
-      if pve_cluster + "." + pve_cloud == target_pve:
-        for pve_host in pve_inventory[pve_cloud][pve_cluster]:
-          # check if host is available
-          pve_host_ip = pve_inventory[pve_cloud][pve_cluster][pve_host]["ansible_host"]
-          try:
-              with socket.create_connection((pve_host_ip, 22), timeout=3):
-                  return pve_host_ip
-          except Exception as e:
-              # debug
-              print(e, type(e))
-              pass
-  
-  raise Exception(f"Could not find online pve host for {target_pve}")
+    return cluster_vars
 
 
 def get_cloud_env(pve_host):
-  ssh = paramiko.SSHClient()
-  ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 
-  ssh.connect(pve_host, username="root")
+    ssh.connect(pve_host, username="root")
 
-  # since we need root we cant use sftp and root via ssh is disabled
-  _, stdout, _ = ssh.exec_command("cat /etc/pve/cloud/cluster_vars.yaml")
+    # since we need root we cant use sftp and root via ssh is disabled
+    _, stdout, _ = ssh.exec_command("cat /etc/pve/cloud/cluster_vars.yaml")
 
-  cluster_vars = yaml.safe_load(stdout.read().decode('utf-8'))
+    cluster_vars = yaml.safe_load(stdout.read().decode("utf-8"))
 
-  _, stdout, _ = ssh.exec_command("cat /etc/pve/cloud/secrets/patroni.pass")
+    _, stdout, _ = ssh.exec_command("cat /etc/pve/cloud/secrets/patroni.pass")
 
-  patroni_pass = stdout.read().decode('utf-8').strip()
+    patroni_pass = stdout.read().decode("utf-8").strip()
 
-  # fetch bind update key for ingress dns validation
-  _, stdout, _ = ssh.exec_command("sudo cat /etc/pve/cloud/secrets/internal.key")
-  bind_key_file = stdout.read().decode('utf-8')
+    # fetch bind update key for ingress dns validation
+    _, stdout, _ = ssh.exec_command("sudo cat /etc/pve/cloud/secrets/internal.key")
+    bind_key_file = stdout.read().decode("utf-8")
 
-  bind_internal_key = re.search(r'secret\s+"([^"]+)";', bind_key_file).group(1)
+    bind_internal_key = re.search(r'secret\s+"([^"]+)";', bind_key_file).group(1)
 
-  return cluster_vars, patroni_pass, bind_internal_key
+    return cluster_vars, patroni_pass, bind_internal_key
 
 
 def get_online_pve_host_prsr(args):
-  print(f"export PVE_ANSIBLE_HOST='{get_online_pve_host(args.target_pve)}'")
+    print(
+        f"export PVE_ANSIBLE_HOST='{get_online_pve_host(args.target_pve, suppress_warnings=True)}'"
+    )
 
 
 def get_ssh_master_kubeconfig(cluster_vars, stack_name):
-  resolver = dns.resolver.Resolver()
-  resolver.nameservers = [cluster_vars['bind_master_ip'], cluster_vars['bind_slave_ip']]
+    resolver = dns.resolver.Resolver()
+    resolver.nameservers = [
+        cluster_vars["bind_master_ip"],
+        cluster_vars["bind_slave_ip"],
+    ]
 
-  ddns_answer = resolver.resolve(f"masters-{stack_name}.{cluster_vars['pve_cloud_domain']}")
-  ddns_ips = [rdata.to_text() for rdata in ddns_answer]
+    ddns_answer = resolver.resolve(
+        f"masters-{stack_name}.{cluster_vars['pve_cloud_domain']}"
+    )
+    ddns_ips = [rdata.to_text() for rdata in ddns_answer]
 
-  if not ddns_ips:
-    raise Exception("No master could be found via DNS!")
+    if not ddns_ips:
+        raise Exception("No master could be found via DNS!")
 
-  ssh = paramiko.SSHClient()
-  ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 
-  ssh.connect(ddns_ips[0], username="admin")
+    ssh.connect(ddns_ips[0], username="admin")
 
-  # since we need root we cant use sftp and root via ssh is disabled
-  _, stdout, _ = ssh.exec_command("sudo cat /etc/kubernetes/admin.conf")
+    # since we need root we cant use sftp and root via ssh is disabled
+    _, stdout, _ = ssh.exec_command("sudo cat /etc/kubernetes/admin.conf")
 
-  return stdout.read().decode('utf-8').replace("https://127.0.0.1:6443", f"https://{ddns_ips[0]}:6443")
+    admin_conf = yaml.safe_load(stdout.read().decode("utf-8"))
+    # rewrite variables for external access
+    admin_conf["clusters"][0]["cluster"]["server"] = f"https://{ddns_ips[0]}:6443"
+    admin_conf["clusters"][0]["name"] = stack_name
 
+    admin_conf["contexts"][0]["context"]["cluster"] = stack_name
+    admin_conf["contexts"][0]["name"] = stack_name
 
-def export_envr(args):
-  ansible_host = get_online_pve_host(args.target_pve)
-  cloud_domain = get_cloud_domain(args.target_pve)
-  cluster_vars, patroni_pass, bind_internal_key = get_cloud_env(ansible_host)
-  print(f"export PVE_ANSIBLE_HOST='{ansible_host}'")
-  print(f"export PVE_CLOUD_DOMAIN='{cloud_domain}'")
+    admin_conf["current-context"] = stack_name
 
-  # tf vars
-  print(f"export PG_CONN_STR=\"postgres://postgres:{patroni_pass}@{cluster_vars['pve_haproxy_floating_ip_internal']}:5000/tf_states?sslmode=disable\"")
-  print(f"export TF_VAR_pve_cloud_domain='{cloud_domain}'")
-  print(f"export TF_VAR_pve_host='{ansible_host}'")
-  print(f"export TF_VAR_cluster_proxy_ip='{cluster_vars['pve_haproxy_floating_ip_internal']}'")
-  print(f"export TF_VAR_pve_cloud_pg_cstr=\"postgresql+psycopg2://postgres:{patroni_pass}@{cluster_vars['pve_haproxy_floating_ip_internal']}:5000/pve_cloud?sslmode=disable\"")
-  print(f"export TF_VAR_master_b64_kubeconf='{base64.b64encode(get_ssh_master_kubeconfig(cluster_vars, args.stack_name).encode('utf-8')).decode('utf-8')}'")
-  print(f"export TF_VAR_bind_master_ip='{cluster_vars['bind_master_ip']}'")
-  print(f"export TF_VAR_bind_internal_key='{bind_internal_key}'")
+    return yaml.safe_dump(admin_conf)
 
-  
-def main():
-  parser = argparse.ArgumentParser(description="PVE Cloud utility cli. Should be called with bash eval.")
 
-  base_parser = argparse.ArgumentParser(add_help=False)
+def export_pg_conn_str(args):
+    cloud_domain = get_cloud_domain(args.target_pve, suppress_warnings=True)
+    pve_inventory = get_pve_inventory(cloud_domain, suppress_warnings=True)
 
-  subparsers = parser.add_subparsers(dest="command", required=True)
+    # get ansible ip for first host in target cluster
+    ansible_host = None
+    for cluster in pve_inventory:
+        if args.target_pve.startswith(cluster):
+            ansible_host = next(iter(pve_inventory[cluster].values()))["ansible_host"]
 
-  get_cld_domain_parser = subparsers.add_parser("get-cloud-domain", help="Get the cloud domain of a pve cluster.", parents=[base_parser])
-  get_cld_domain_parser.add_argument("--target-pve", type=str, help="The target pve cluster to get the cloud domain of.", required=True)
-  get_cld_domain_parser .set_defaults(func=get_cld_domain_prsr)
+    if not ansible_host:
+        raise RuntimeError(f"Could not find online host for {args.target_pve}!")
 
-  export_envr_parser = subparsers.add_parser("export-envrc", help="Export variables for k8s .envrc", parents=[base_parser])
-  export_envr_parser.add_argument("--target-pve", type=str, help="The target pve cluster.", required=True)
-  export_envr_parser.add_argument("--stack-name", type=str, help="Stack name of the deployment.", required=True)
-  export_envr_parser.set_defaults(func=export_envr)
+    cluster_vars, patroni_pass, bind_internal_key = get_cloud_env(ansible_host)
 
-  get_online_pve_host_parser = subparsers.add_parser("get-online-host", help="Gets the ip for the first online proxmox host in the cluster.", parents=[base_parser])
-  get_online_pve_host_parser.add_argument("--target-pve", type=str, help="The target pve cluster to get the first online ip of.", required=True)
-  get_online_pve_host_parser.set_defaults(func=get_online_pve_host_prsr)
+    print(
+        f"export PG_CONN_STR=\"postgres://postgres:{patroni_pass}@{cluster_vars['pve_haproxy_floating_ip_internal']}:5000/tf_states?sslmode=disable\""
+    )
 
-  args = parser.parse_args()
-  args.func(args)
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="PVE Cloud utility cli. Should be called with bash eval."
+    )
+
+    base_parser = argparse.ArgumentParser(add_help=False)
+
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    export_envr_parser = subparsers.add_parser(
+        "export-psql", help="Export variables for k8s .envrc", parents=[base_parser]
+    )
+    export_envr_parser.add_argument(
+        "--target-pve", type=str, help="The target pve cluster.", required=True
+    )
+    export_envr_parser.set_defaults(func=export_pg_conn_str)
+
+    get_online_pve_host_parser = subparsers.add_parser(
+        "get-online-host",
+        help="Gets the ip for the first online proxmox host in the cluster.",
+        parents=[base_parser],
+    )
+    get_online_pve_host_parser.add_argument(
+        "--target-pve",
+        type=str,
+        help="The target pve cluster to get the first online ip of.",
+        required=True,
+    )
+    get_online_pve_host_parser.set_defaults(func=get_online_pve_host_prsr)
+
+    args = parser.parse_args()
+    args.func(args)
 
 
 if __name__ == "__main__":
-  main()
+    main()