py-pve-cloud 0.5.16__py3-none-any.whl → 0.14.2rc26__py3-none-any.whl

pve_cloud/lib/inventory.py

@@ -1,5 +1,276 @@
+import os
+import shutil
+import socket
+import subprocess
 
-def get_pve_cloud_inventory():
-  pass
+import yaml
+from proxmoxer import ProxmoxAPI
 
+from pve_cloud.lib.validate import raise_on_py_cloud_missmatch
 
+
+def get_cloud_domain(target_pve, suppress_warnings=False):
+    if shutil.which("avahi-browse"):
+        avahi_disc = subprocess.run(
+            ["avahi-browse", "-rpt", "_pxc._tcp"],
+            stdout=subprocess.PIPE,
+            text=True,
+            check=True,
+        )
+        services = avahi_disc.stdout.splitlines()
+
+        # find cloud domain hosts and get first online per proxmox cluster
+        for service in services:
+            if service.startswith("="):
+                # avahi service def
+                svc_args = service.split(";")
+
+                cloud_domain = None
+                cluster_name = None
+
+                for txt_arg in svc_args[9].split():
+                    txt_arg = txt_arg.replace('"', "")
+                    if txt_arg.startswith("cloud_domain"):
+                        cloud_domain = txt_arg.split("=")[1]
+
+                    if txt_arg.startswith("cluster_name"):
+                        cluster_name = txt_arg.split("=")[1]
+
+                if not cloud_domain or not cluster_name:
+                    raise ValueError(
+                        f"Missconfigured proxmox cloud avahi service: {service}"
+                    )
+
+                if target_pve.endswith(cloud_domain):
+                    return cloud_domain
+
+        raise RuntimeError("Could not get cloud domain via avahi mdns!")
+    else:
+        if not suppress_warnings:
+            print(
+                "avahi-browse not available, falling back to local inventory file from pvcli connect-cluster!"
+            )
+
+        with open(os.path.expanduser("~/.pve-cloud-dyn-inv.yaml"), "r") as f:
+            pve_inventory = yaml.safe_load(f)
+
+        for pve_cloud in pve_inventory:
+            for pve_cluster in pve_inventory[pve_cloud]:
+                if pve_cluster + "." + pve_cloud == target_pve:
+                    return pve_cloud
+
+        raise Exception(f"Could not identify cloud domain for {target_pve}")
+
+
+def get_online_pve_host(target_pve, suppress_warnings=False, skip_py_cloud_check=False):
+    if shutil.which("avahi-browse"):
+        avahi_disc = subprocess.run(
+            ["avahi-browse", "-rpt", "_pxc._tcp"],
+            stdout=subprocess.PIPE,
+            text=True,
+            check=True,
+        )
+        services = avahi_disc.stdout.splitlines()
+
+        for service in services:
+            if service.startswith("="):
+                # avahi service def
+                svc_args = service.split(";")
+                host_ip = svc_args[7]
+
+                cloud_domain = None
+                cluster_name = None
+
+                for txt_arg in svc_args[9].split():
+                    txt_arg = txt_arg.replace('"', "")
+                    if txt_arg.startswith("cloud_domain"):
+                        cloud_domain = txt_arg.split("=")[1]
+
+                    if txt_arg.startswith("cluster_name"):
+                        cluster_name = txt_arg.split("=")[1]
+
+                if not cloud_domain or not cluster_name:
+                    raise ValueError(
+                        f"Missconfigured proxmox cloud avahi service: {service}"
+                    )
+
+                # main pve cloud inventory
+                if f"{cluster_name}.{cloud_domain}" == target_pve:
+                    if not skip_py_cloud_check:
+                        raise_on_py_cloud_missmatch(
+                            host_ip
+                        )  # validate that versions of dev machine and running on cluster match
+
+                    return host_ip
+
+        raise RuntimeError(f"No online host found for {target_pve}!")
+    else:
+        if not suppress_warnings:
+            print(
+                "avahi-browse not available, falling back to local inventory file from pvcli connect-cluster!"
+            )
+
+        with open(os.path.expanduser("~/.pve-cloud-dyn-inv.yaml"), "r") as f:
+            pve_inventory = yaml.safe_load(f)
+
+        for pve_cloud in pve_inventory:
+            for pve_cluster in pve_inventory[pve_cloud]:
+                if pve_cluster + "." + pve_cloud == target_pve:
+                    for pve_host in pve_inventory[pve_cloud][pve_cluster]:
+                        # check if host is available
+                        pve_host_ip = pve_inventory[pve_cloud][pve_cluster][pve_host][
+                            "ansible_host"
+                        ]
+                        try:
+                            with socket.create_connection((pve_host_ip, 22), timeout=3):
+
+                                if not skip_py_cloud_check:
+                                    raise_on_py_cloud_missmatch(
+                                        pve_host_ip
+                                    )  # validate that versions of dev machine and running on cluster match
+
+                                return pve_host_ip
+                        except Exception as e:
+                            # debug
+                            print(e, type(e))
+
+        raise RuntimeError(f"Could not find online pve host for {target_pve}")
+
+
+def get_pve_inventory(
+    pve_cloud_domain, suppress_warnings=False, skip_py_cloud_check=False
+):
+    if shutil.which("avahi-browse"):
+        # avahi is available
+
+        # call avahi-browse -rpt _pxc._tcp and find online host matching pve cloud domain
+        # connect via ssh and fetch all other hosts via proxmox api => build inventory
+        avahi_disc = subprocess.run(
+            ["avahi-browse", "-rpt", "_pxc._tcp"],
+            stdout=subprocess.PIPE,
+            text=True,
+            check=True,
+        )
+        services = avahi_disc.stdout.splitlines()
+
+        pve_inventory = {}
+
+        py_pve_cloud_performed_version_checks = set()
+
+        # find cloud domain hosts and get first online per proxmox cluster
+        cloud_domain_first_hosts = {}
+        for service in services:
+            if service.startswith("="):
+                # avahi service def
+                svc_args = service.split(";")
+                host_ip = svc_args[7]
+
+                cloud_domain = None
+                cluster_name = None
+
+                for txt_arg in svc_args[9].split():
+                    txt_arg = txt_arg.replace('"', "")
+                    if txt_arg.startswith("cloud_domain"):
+                        cloud_domain = txt_arg.split("=")[1]
+
+                    if txt_arg.startswith("cluster_name"):
+                        cluster_name = txt_arg.split("=")[1]
+
+                if not cloud_domain or not cluster_name:
+                    raise ValueError(
+                        f"Missconfigured proxmox cloud avahi service: {service}"
+                    )
+
+                # main pve cloud inventory
+                if (
+                    cloud_domain == pve_cloud_domain
+                    and cluster_name not in cloud_domain_first_hosts
+                ):
+                    if (
+                        not skip_py_cloud_check
+                        and f"{cluster_name}.{cloud_domain}"
+                        not in py_pve_cloud_performed_version_checks
+                    ):
+                        raise_on_py_cloud_missmatch(
+                            host_ip
+                        )  # validate that versions of dev machine and running on cluster match
+                        py_pve_cloud_performed_version_checks.add(
+                            f"{cluster_name}.{cloud_domain}"
+                        )  # perform version check only once per cluster
+
+                    cloud_domain_first_hosts[cluster_name] = host_ip
+
+        # iterate over hosts and build pve inv via proxmox api
+        # todo: this needs to be hugely optimized it blocks the grpc server
+        for cluster_first, first_host in cloud_domain_first_hosts.items():
+            proxmox = ProxmoxAPI(first_host, user="root", backend="ssh_paramiko")
+
+            cluster_name = None
+            status_resp = proxmox.cluster.status.get()
+            for entry in status_resp:
+                if entry["id"] == "cluster":
+                    cluster_name = entry["name"]
+                break
+
+            if cluster_name is None:
+                raise RuntimeError("Could not get cluster name")
+
+            if cluster_name != cluster_first:
+                raise ValueError(
+                    f"Proxmox cluster name missconfigured in avahi service {cluster_name}/{cluster_first}"
+                )
+
+            pve_inventory[cluster_name] = {}
+
+            # fetch other hosts via api
+            cluster_hosts = proxmox.nodes.get()
+
+            for node in cluster_hosts:
+                node_name = node["node"]
+
+                if node["status"] == "offline":
+                    print(f"skipping offline node {node_name}")
+                    continue
+
+                # get the main ip
+                ifaces = proxmox.nodes(node_name).network.get()
+                node_ip_address = None
+                for iface in ifaces:
+                    if "gateway" in iface:
+                        if node_ip_address is not None:
+                            raise RuntimeError(
+                                f"found multiple ifaces with gateways for node {node_name}"
+                            )
+                        node_ip_address = iface.get("address")
+
+                if node_ip_address is None:
+                    raise RuntimeError(f"Could not find ip for node {node_name}")
+
+                pve_inventory[cluster_name][node_name] = {
+                    "ansible_user": "root",
+                    "ansible_host": node_ip_address,
+                }
+
+        return pve_inventory
+
+    else:
+        if not suppress_warnings:
+            print(
+                "avahi-browse not available, falling back to local inventory file from pvcli connect-cluster!"
+            )
+        # try load fallback manual inventory from disk
+        inv_path = os.path.expanduser("~/.pve-cloud-dyn-inv.yaml")
+        if not os.path.exists(inv_path):
+            raise RuntimeError(
+                "Local pve inventory file missing (~/.pve-cloud-dyn-inv.yaml), execute `pvcli connect-cluster` or setup avahi mdns discovery!"
+            )
+
+        with open(inv_path, "r") as file:
+            dynamic_inventory = yaml.safe_load(file)
+
+        if pve_cloud_domain not in dynamic_inventory:
+            raise RuntimeError(
+                f"{pve_cloud_domain} not in local dynamic inventory (~/.pve-cloud-dyn-inv.yaml created by `pvcli connect-cluster`)!"
+            )
+
+        return dynamic_inventory[pve_cloud_domain]

pve_cloud/lib/validate.py

@@ -0,0 +1,25 @@
+import os
+
+import paramiko
+import pve_cloud._version
+import yaml
+
+
+def raise_on_py_cloud_missmatch(proxmox_host):
+    # dont raise in tdd
+    if os.getenv("PYTEST_CURRENT_TEST"):
+        return
+
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect(proxmox_host, username="root")
+
+    # since we need root we cant use sftp and root via ssh is disabled
+    _, stdout, _ = ssh.exec_command("cat /etc/pve/cloud/cluster_vars.yaml")
+
+    cluster_vars = yaml.safe_load(stdout.read().decode("utf-8"))
+
+    if cluster_vars["py_pve_cloud_version"] != pve_cloud._version.__version__:
+        raise RuntimeError(
+            f"Version missmatch! py_pve_cloud_version for cluster is {cluster_vars['py_pve_cloud_version']}, while you are using {pve_cloud._version.__version__}"
+        )

pve_cloud/orm/alchemy.py

@@ -1,5 +1,5 @@
 from sqlalchemy import Column, Integer, String, Boolean, SmallInteger, Text
-from sqlalchemy.dialects.postgresql import MACADDR, INET, JSONB, insert
+from sqlalchemy.dialects.postgresql import MACADDR, INET, JSONB, insert, ENUM
 from sqlalchemy import create_engine, MetaData, Table, select, delete, update
 from sqlalchemy.orm import declarative_base
 from alembic.config import Config
@@ -33,6 +33,7 @@ class KeaReservations(Base):
   hostname = Column(String(253), nullable=False)
   client_classes = Column(String(1000)) # csv seperated client classes
   stack_fqdn = Column(String(253), nullable=False)
+  machine_type = Column(String(50), nullable=False)
 
 
 class KeaClientClassDefs(Base):
@@ -43,24 +44,7 @@ class KeaClientClassDefs(Base):
   class_content = Column(JSONB, nullable=False)
 
 
-class K8SWorkers(Base):
-  __tablename__ = "k8s_workers"
-
-  ip = Column(INET, primary_key=True)
-  hostname = Column(String(253), nullable=False)
-  stack_fqdn = Column(String(253), nullable=False)
-  proxy_stack_fqdn = Column(String(253), nullable=False)
-
-
-class K8SMasters(Base):
-  __tablename__ = "k8s_masters"
-
-  ip = Column(INET, primary_key=True)
-  hostname = Column(String(253), nullable=False)
-  stack_fqdn = Column(String(253), nullable=False)
-  proxy_stack_fqdn = Column(String(253), nullable=False)
-
-
+# todo: rename just ingress rules
 class K8SIngressRules(Base):
   __tablename__ = "k8s_ingress_rules"
 
@@ -70,8 +54,10 @@ class K8SIngressRules(Base):
   proxy_stack_fqdn = Column(String(253), nullable=False)
   external = Column(Boolean, default=False)
   rule_len = Column(Integer, nullable=False)
+  is_k8s = Column(Boolean, nullable=False)
 
 
+# todo: rename to just tcp proxies
 class K8STcpProxies(Base):
   __tablename__ = "k8s_tcp_proxies"
 
@@ -79,6 +65,8 @@ class K8STcpProxies(Base):
   haproxy_port = Column(SmallInteger, primary_key=True)
   node_port = Column(SmallInteger, nullable=False)
   stack_fqdn = Column(String(253), nullable=False)
+  # determines backend routing in haproxy, if false will go 
+  is_k8s = Column(Boolean, nullable=False) 
   proxy_snippet = Column(Text)
   proxy_stack_fqdn = Column(String(253), primary_key=True)
   external = Column(Boolean, default=False)

pve_cloud/orm/migrations/versions/0ad803c51325_machine_type_refactor.py

@@ -0,0 +1,65 @@
+"""machine type refactor
+
+Revision ID: 0ad803c51325
+Revises: fdcb5aa33b76
+Create Date: 2025-12-22 13:46:51.857049
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = '0ad803c51325'
+down_revision: Union[str, Sequence[str], None] = 'fdcb5aa33b76'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+
+    op.add_column('k8s_tcp_proxies', sa.Column('is_k8s', sa.Boolean(), nullable=False, server_default='true'))
+    op.add_column('kea_reservations', sa.Column('machine_type', sa.String(length=50), nullable=False, server_default='k8s_worker'))
+    
+    # remove defaults
+    op.alter_column('k8s_tcp_proxies', 'is_k8s', server_default=None)
+    op.alter_column('kea_reservations', 'machine_type', server_default=None)
+
+    # set master type
+    op.execute("""
+        UPDATE kea_reservations kr
+        SET machine_type = 'k8s_master'
+        FROM k8s_masters km
+        WHERE kr.ip = km.ip
+    """)
+
+    op.drop_table('k8s_masters')
+    op.drop_table('k8s_workers')
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('kea_reservations', 'machine_type')
+    op.drop_column('k8s_tcp_proxies', 'is_k8s')
+    op.create_table('k8s_workers',
+    sa.Column('ip', postgresql.INET(), autoincrement=False, nullable=False),
+    sa.Column('hostname', sa.VARCHAR(length=253), autoincrement=False, nullable=False),
+    sa.Column('stack_fqdn', sa.VARCHAR(length=253), autoincrement=False, nullable=False),
+    sa.Column('proxy_stack_fqdn', sa.VARCHAR(length=253), autoincrement=False, nullable=False),
+    sa.PrimaryKeyConstraint('ip', name=op.f('k8s_workers_pkey'))
+    )
+    op.create_table('k8s_masters',
+    sa.Column('ip', postgresql.INET(), autoincrement=False, nullable=False),
+    sa.Column('hostname', sa.VARCHAR(length=253), autoincrement=False, nullable=False),
+    sa.Column('stack_fqdn', sa.VARCHAR(length=253), autoincrement=False, nullable=False),
+    sa.Column('proxy_stack_fqdn', sa.VARCHAR(length=253), autoincrement=False, nullable=False),
+    sa.PrimaryKeyConstraint('ip', name=op.f('k8s_masters_pkey'))
+    )
+    # ### end Alembic commands ###

pve_cloud/orm/migrations/versions/e60b9cc63413_ingress_generic.py

@@ -0,0 +1,33 @@
+"""ingress generic
+
+Revision ID: e60b9cc63413
+Revises: 0ad803c51325
+Create Date: 2025-12-22 21:29:59.987939
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'e60b9cc63413'
+down_revision: Union[str, Sequence[str], None] = '0ad803c51325'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('k8s_ingress_rules', sa.Column('is_k8s', sa.Boolean(), nullable=False, server_default='true'))
+    op.alter_column('k8s_ingress_rules', 'is_k8s', server_default=None) # default create as true since to this point its only for k8s
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('k8s_ingress_rules', 'is_k8s')
+    # ### end Alembic commands ###

{py_pve_cloud-0.5.16.dist-info → py_pve_cloud-0.14.2rc26.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: py-pve-cloud
-Version: 0.5.16
+Version: 0.14.2rc26
 Author-email: Tobias Huebner <tobias.huebner@vmzberlin.com>
 License-Expression: GPL-3.0-or-later
-License-File: LICENSE
+License-File: LICENSE.md
 Requires-Dist: PyYAML==6.0.2
 Requires-Dist: psycopg2-binary==2.9.10
 Requires-Dist: SQLAlchemy==2.0.43

py_pve_cloud-0.14.2rc26.dist-info/RECORD

@@ -0,0 +1,25 @@
+pve_cloud/_version.py,sha256=PeTrwMVliRFapYHocR8c3EJDBUJO0_wu89ViTvJwiS4,27
+pve_cloud/cli/pvcli.py,sha256=Sew-_ZVmugskLYncEzVE_-F1R90O9vM9rPZ2DuxumQQ,5977
+pve_cloud/cli/pvclu.py,sha256=IZRhgbVbEP5wo3gHpiEs3zEu_xmgcpuQAPRWPi18APc,4660
+pve_cloud/lib/inventory.py,sha256=76JVyYbVDKGHYnEab4DollNwxzZTzvgVBdqlPMIkGcw,10598
+pve_cloud/lib/validate.py,sha256=IQ4_1pknjFt_tbqhl497aoaVDnUSgxq5eSLkantdub8,830
+pve_cloud/orm/alchemy.py,sha256=fGDBpO22quf_d7jCeAkDPYxwXPDFSUH99DKe2JuJOUk,5031
+pve_cloud/orm/alembic.ini,sha256=7140n-YUj06aAIHOHACm8U0xhUFUoBZ4Jw23KlYB9EA,4865
+pve_cloud/orm/migrations/env.py,sha256=xtOgjF1KLmRUkG1-yb4eV4F2JzarDKFU1tdWJovNHDc,2200
+pve_cloud/orm/migrations/versions/04398db10434_external_cp_extra_sans.py,sha256=HUrnRni5D_xpqVruXoywekA-u3bIjZb-LvyMi8Ea65E,1465
+pve_cloud/orm/migrations/versions/0ad803c51325_machine_type_refactor.py,sha256=Tp3_KPp9SlEAc1Nh7BdIll_rTAWbQUFP1Evjdx_g-So,2480
+pve_cloud/orm/migrations/versions/24a548bfce3e_len_rules_enforcements.py,sha256=rWVywDbSLs16bEcncdXuwcij1axp-K0mleTXqFRhs3s,5238
+pve_cloud/orm/migrations/versions/27724e407e2b_proxy_fqdn.py,sha256=va2I9uPkn6QEQf_sn986-ogX9HeN08ScmFhzqk-eYsY,918
+pve_cloud/orm/migrations/versions/3c95509a5de9_fix.py,sha256=yAfRQerPhlHoRrW6F2aAi2ZyY5IIuGUOCieyg7IK544,1491
+pve_cloud/orm/migrations/versions/7868bcd05006_migrate_old.py,sha256=rU8Bw2tYDynM1Ny0zFolDbcn6Oe6JAhndbpJAewiTac,3475
+pve_cloud/orm/migrations/versions/7dea8c4ee39f_init.py,sha256=iMDyHhtyvpSywMnLhiSEL3W12YSm6sPa18XRgzQcwDg,954
+pve_cloud/orm/migrations/versions/944a8fd5d5bc_ext_ctrl_plns.py,sha256=LnVAShLaU1asz1L7TYs7oI9SnLxPp2IOA6K83kHNkN0,1674
+pve_cloud/orm/migrations/versions/d9b711555be8_ext_control_plane.py,sha256=uBqv1r5pLX-RjqciKYx0zvWyygJMa5u58DTnVfIEAF0,1073
+pve_cloud/orm/migrations/versions/e60b9cc63413_ingress_generic.py,sha256=fSVHYbRvkssMsxnuZ3TIOflBseuuQhgYa1G5Xrn5m88,1044
+pve_cloud/orm/migrations/versions/fdcb5aa33b76_slop_firewall_seperation.py,sha256=1qM2weneVfnPFG921lnQSJ70QF9mW4oZmn0ZJguBO8U,1948
+py_pve_cloud-0.14.2rc26.dist-info/licenses/LICENSE.md,sha256=ADUqsZhl4juwq34PRTMiBqumpm11s_PMli_dZQjWPqQ,34260
+py_pve_cloud-0.14.2rc26.dist-info/METADATA,sha256=Fk7mX718ZSU7rukjaF3KJ6l3rKtVAZkN6-eKxvLuUvI,433
+py_pve_cloud-0.14.2rc26.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+py_pve_cloud-0.14.2rc26.dist-info/entry_points.txt,sha256=VvncsKmTJ46irz-9wQZ4Zo1FgNBjRltGDBKR9ht18mE,84
+py_pve_cloud-0.14.2rc26.dist-info/top_level.txt,sha256=mpT7ttGRyZJVt_obhPLBHyIBcjKhUdJ-qVsMEVX5WJg,10
+py_pve_cloud-0.14.2rc26.dist-info/RECORD,,