osi-dump 0.1.2.2__tar.gz → 0.1.2.3__tar.gz

{osi_dump-0.1.2.2 → osi_dump-0.1.2.3}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.1
 Name: osi-dump
-Version: 0.1.2.2
-Summary: OpenStack instances information dump tool
+Version: 0.1.2.3
+Summary: OpenStack information dump tool
 Author: TVKain
 License: Apache-2.0
 Classifier: License :: OSI Approved :: Apache Software License
@@ -12,6 +12,7 @@ Requires-Dist: openstacksdk
 Requires-Dist: pydantic
 Requires-Dist: pandas
 Requires-Dist: openpyxl
+Requires-Dist: numpy
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 

{osi_dump-0.1.2.2 → osi_dump-0.1.2.3}/pyproject.toml

@@ -1,13 +1,20 @@
 [project]
 name = "osi-dump"
-version = "0.1.2.2"
-description = "OpenStack instances information dump tool"
+version = "0.1.2.3"
+description = "OpenStack information dump tool"
 readme = "README.md"
 authors = [{ name = "TVKain" }]
 license = { text = "Apache-2.0" }
 requires-python = ">=3.10"
 classifiers = ["License :: OSI Approved :: Apache Software License"]
-dependencies = ["typer", "openstacksdk", "pydantic", "pandas", "openpyxl"]
+dependencies = [
+    "typer",
+    "openstacksdk",
+    "pydantic",
+    "pandas",
+    "openpyxl",
+    "numpy",
+]
 
 
 [project.scripts]

{osi_dump-0.1.2.2 → osi_dump-0.1.2.3}/src/osi_dump/api/octavia.py

@@ -17,13 +17,13 @@ def get_load_balancers(connection: Connection) -> list[LoadBalancer]:
     return data["loadbalancers"]
 
 
-def get_amphoraes(connection: Connection, load_balancer_id: str) -> list[str]:
+def get_amphoraes(connection: Connection, load_balancer_id: str) -> list[dict]:
 
     octavia_endpoint = connection.endpoint_for(
         service_type="load-balancer", interface="public"
     )
 
-    url = f"{octavia_endpoint}/v2/octavia/amphorae?load_balancer_id={load_balancer_id}&fields=id&fields=compute_id"
+    url = f"{octavia_endpoint}/v2/octavia/amphorae?load_balancer_id={load_balancer_id}&fields=status&fields=compute_id"
 
     response = connection.session.get(url)
 
@@ -31,4 +31,4 @@ def get_amphoraes(connection: Connection, load_balancer_id: str) -> list[str]:
 
     amphoraes = data["amphorae"]
 
-    return [amphorae["compute_id"] for amphorae in amphoraes]
+    return amphoraes

osi_dump-0.1.2.3/src/osi_dump/batch_handler/external_port_batch_handler.py

@@ -0,0 +1,57 @@
+import logging
+
+from openstack.connection import Connection
+
+from osi_dump.exporter.external_port.external_port_exporter import ExternalPortExporter
+from osi_dump.exporter.external_port.excel_external_port_exporter import (
+    ExcelExternalPortExporter,
+)
+
+from osi_dump.importer.external_port.external_port_importer import ExternalPortImporter
+from osi_dump.importer.external_port.openstack_external_port_importer import (
+    OpenStackExternalPortImporter,
+)
+
+
+from osi_dump import util
+
+logger = logging.getLogger(__name__)
+
+
+class ExternalPortBatchHandler:
+    def __init__(self):
+        self._importer_exporter_list: list[
+            tuple[ExternalPortImporter, ExternalPortExporter]
+        ] = []
+
+    def add_importer_exporter_from_openstack_connections(
+        self, connections: list[Connection], output_file: str
+    ):
+        for connection in connections:
+            importer = OpenStackExternalPortImporter(connection)
+
+            sheet_name = (
+                f"{util.extract_hostname(connection.auth['auth_url'])}-external_port"
+            )
+            exporter = ExcelExternalPortExporter(
+                sheet_name=sheet_name, output_file=output_file
+            )
+
+            self.add_importer_exporter(importer=importer, exporter=exporter)
+
+    def add_importer_exporter(
+        self, importer: ExternalPortImporter, exporter: ExternalPortExporter
+    ):
+        self._importer_exporter_list.append((importer, exporter))
+
+    def process(self):
+
+        for importer, exporter in self._importer_exporter_list:
+            try:
+
+                external_ports = importer.import_external_ports()
+
+                exporter.export_external_ports(external_ports=external_ports)
+            except Exception as e:
+                logger.warning(e)
+                logger.warning("Skipping...")

{osi_dump-0.1.2.2 → osi_dump-0.1.2.3}/src/osi_dump/cli.py

@@ -8,6 +8,7 @@ import typer
 
 from typing_extensions import Annotated
 
+from osi_dump.batch_handler.external_port_batch_handler import ExternalPortBatchHandler
 from osi_dump.batch_handler.load_balancer_batch_handler import LoadBalancerBatchHandler
 from osi_dump.batch_handler.role_assignment_batch_handler import (
     RoleAssignmentBatchHandler,
@@ -31,6 +32,11 @@ from osi_dump.batch_handler import (
 )
 
 
+from osi_dump.importer.external_port.openstack_external_port_importer import (
+    OpenStackExternalPortImporter,
+)
+
+
 from osi_dump import util
 
 
@@ -134,6 +140,16 @@ def _router(connections, output_path: str):
     _router_batch_handler.process()
 
 
+def _external_port(connections, output_path: str):
+    _external_batch_handler = ExternalPortBatchHandler()
+
+    _external_batch_handler.add_importer_exporter_from_openstack_connections(
+        connections, output_file=output_path
+    )
+
+    _external_batch_handler.process()
+
+
 def inner_main(file_path: str, output_path: str):
 
     logger = logging.getLogger(__name__)
@@ -160,6 +176,8 @@ def inner_main(file_path: str, output_path: str):
 
     _router(connections=connections, output_path=output_path)
 
+    _external_port(connections=connections, output_path=output_path)
+
     util.excel_autosize_column(output_path)
 
     util.excel_sort_sheet(output_path)

osi_dump-0.1.2.3/src/osi_dump/exporter/external_port/excel_external_port_exporter.py

@@ -0,0 +1,34 @@
+import pandas as pd
+
+import logging
+
+from openpyxl import load_workbook
+
+from osi_dump import util
+from osi_dump.exporter.external_port.external_port_exporter import ExternalPortExporter
+
+from osi_dump.model.external_port import ExternalPort
+
+logger = logging.getLogger(__name__)
+
+
+class ExcelExternalPortExporter(ExternalPortExporter):
+    def __init__(self, sheet_name: str, output_file: str):
+        self.sheet_name = sheet_name
+        self.output_file = output_file
+
+    def export_external_ports(self, external_ports: list[ExternalPort]):
+
+        df = pd.DataFrame(
+            [external_port.model_dump() for external_port in external_ports]
+        )
+
+        df = util.panda_excel.expand_list_column(df, "allowed_address_pairs")
+
+        logger.info(f"Exporting external_ports for {self.sheet_name}")
+        try:
+            util.export_data_excel(self.output_file, sheet_name=self.sheet_name, df=df)
+
+            logger.info(f"Exported external_ports for {self.sheet_name}")
+        except Exception as e:
+            logger.warning(f"Exporting external_ports for {self.sheet_name} error: {e}")

osi_dump-0.1.2.3/src/osi_dump/exporter/external_port/external_port_exporter.py

@@ -0,0 +1,7 @@
+from abc import ABC, abstractmethod
+
+
+class ExternalPortExporter(ABC):
+    @abstractmethod
+    def export_external_ports(self, projects, output_file: str):
+        pass

osi_dump-0.1.2.3/src/osi_dump/importer/external_port/external_port_importer.py

@@ -0,0 +1,9 @@
+from abc import ABC, abstractmethod
+
+from osi_dump.model.external_port import ExternalPort
+
+
+class ExternalPortImporter(ABC):
+    @abstractmethod
+    def import_external_ports(self) -> list[ExternalPort]:
+        pass

osi_dump-0.1.2.3/src/osi_dump/importer/external_port/openstack_external_port_importer.py

@@ -0,0 +1,129 @@
+import logging
+
+import ipaddress
+
+import concurrent
+
+import numpy as np
+
+from openstack.connection import Connection
+from openstack.network.v2.port import Port as OSPort
+from openstack.network.v2.subnet import Subnet as OSSubnet
+
+from osi_dump.importer.external_port.external_port_importer import ExternalPortImporter
+from osi_dump.model.external_port import ExternalPort
+
+logger = logging.getLogger(__name__)
+
+
+class OpenStackExternalPortImporter(ExternalPortImporter):
+    def __init__(self, connection: Connection):
+        self.connection = connection
+
+    def _get_external_ports(self) -> list[OSPort]:
+
+        self.connection.list_networks()
+        # Get all external networks
+        networks = self.connection.network.networks(
+            is_router_external=True,
+        )
+
+        external_ports = []
+
+        for network in networks:
+            ex_ports = list(self.connection.network.ports(network_id=network.id))
+
+            for ex_port in ex_ports:
+                ex_port["segmentation_id"] = network.provider_segmentation_id
+
+            external_ports.append(ex_ports)
+
+        # Get all port on those networks
+
+        external_ports = list(np.array(external_ports).flatten())
+
+        return external_ports
+
+    def import_external_ports(self) -> list[ExternalPort]:
+        """Import external_ports information from Openstack
+
+
+        Raises:
+            Exception: Raises exception if fetching external_port failed
+
+        Returns:
+            list[Instance]: _description_
+        """
+
+        logger.info(f"Importing external_ports for {self.connection.auth['auth_url']}")
+
+        try:
+            os_ports: list[OSPort] = self._get_external_ports()
+        except Exception as e:
+            raise Exception(
+                f"Can not fetch external_ports for {self.connection.auth['auth_url']}"
+            ) from e
+
+        external_ports: list[ExternalPort] = []
+
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            futures = [
+                executor.submit(self._get_external_port_info, external_port)
+                for external_port in os_ports
+            ]
+            for future in concurrent.futures.as_completed(futures):
+                external_ports.append(future.result())
+
+        logger.info(f"Imported external_ports for {self.connection.auth['auth_url']}")
+
+        external_ports = sorted(
+            external_ports,
+            key=lambda external_port: (
+                external_port.network_id,
+                (
+                    ipaddress.ip_address(external_port.ip_address)
+                    if external_port.ip_address
+                    else ipaddress.ip_address("0.0.0.0")
+                ),
+            ),
+        )
+
+        return external_ports
+
+    def _get_external_port_info(self, external_port: OSPort) -> ExternalPort:
+
+        subnet_id = None
+        ip_address = None
+        subnet_cidr = None
+
+        try:
+            ip_address = external_port.fixed_ips[0]["ip_address"]
+        except Exception as e:
+            logger.warning(f"No IP address found for port {external_port.id}")
+
+        try:
+            subnet_id = external_port.fixed_ips[0]["subnet_id"]
+        except Exception as e:
+            logger.warning(f"No subnet ID found for {external_port.id}")
+
+        try:
+            subnet: OSSubnet = self.connection.get_subnet_by_id(subnet_id)
+
+            subnet_cidr = subnet.cidr
+        except Exception as e:
+            logger.warning(f"No subnet cidr found for port {external_port.id}")
+
+        external_port_ret = ExternalPort(
+            port_id=external_port.id,
+            project_id=external_port.project_id,
+            network_id=external_port.network_id,
+            subnet_id=subnet_id,
+            subnet_cidr=subnet_cidr,
+            ip_address=ip_address,
+            allowed_address_pairs=external_port.allowed_address_pairs,
+            device_id=external_port.device_id,
+            device_owner=external_port.device_owner,
+            status=external_port.status,
+        )
+
+        return external_port_ret

{osi_dump-0.1.2.2 → osi_dump-0.1.2.3}/src/osi_dump/importer/hypervisor/openstack_hypervisor_importer.py

@@ -30,12 +30,11 @@ class OpenStackHypervisorImporter(HypervisorImporter):
             list[Hypervisor]: _description_
         """
 
-        logger.info(f"Importing hypervisors for {self.connection.auth['auth_url']}")
-
         try:
             oshypervisors: list[OSHypervisor] = list(
-                self.connection.compute.hypervisors(details=True)
+                self.connection.compute.hypervisors(details=True, with_servers=True)
             )
+
         except Exception as e:
             raise Exception(
                 f"Can not fetch hypervisor for {self.connection.auth['auth_url']}"
@@ -65,12 +64,6 @@ class OpenStackHypervisorImporter(HypervisorImporter):
             )
         )
 
-        servers = list(
-            self.connection.compute.servers(
-                details=True, all_project=True, hypervisor_hostname=hypervisor.name
-            )
-        )
-
         usage_data = get_usage(self.connection, resource_provider_id=hypervisor.id)
 
         vcpu = rpi[0]
@@ -89,7 +82,7 @@ class OpenStackHypervisorImporter(HypervisorImporter):
             vcpus_usage=usage_data["VCPU"],
             memory_usage=usage_data["MEMORY_MB"],
             local_disk_usage=usage_data["DISK_GB"],
-            vm_count=len(servers),
+            vm_count=len(hypervisor.servers),
         )
 
         return ret_hypervisor

{osi_dump-0.1.2.2 → osi_dump-0.1.2.3}/src/osi_dump/importer/load_balancer/openstack_load_balancer_importer.py

@@ -64,8 +64,9 @@ class OpenStackLoadBalancerImporter(LoadBalancerImporter):
         load_balancer_ret = LoadBalancer(
             id=load_balancer["id"],
             load_balancer_name=load_balancer["name"],
-            status=load_balancer["operating_status"],
+            operating_status=load_balancer["operating_status"],
             project_id=load_balancer["project_id"],
+            provisioning_status=load_balancer["provisioning_status"],
             amphoraes=amphoraes,
         )
 

osi_dump-0.1.2.3/src/osi_dump/model/external_port.py

@@ -0,0 +1,27 @@
+from typing import Optional
+
+from pydantic import BaseModel, ConfigDict, ValidationError
+
+
+class ExternalPort(BaseModel):
+    model_config = ConfigDict(strict=True)
+
+    port_id: str
+
+    project_id: Optional[str]
+
+    subnet_id: Optional[str]
+
+    subnet_cidr: Optional[str]
+
+    ip_address: Optional[str]
+
+    network_id: Optional[str]
+
+    allowed_address_pairs: Optional[list[dict]]
+
+    device_id: Optional[str]
+
+    device_owner: Optional[str]
+
+    status: Optional[str]

{osi_dump-0.1.2.2 → osi_dump-0.1.2.3}/src/osi_dump/model/image.py

@@ -18,7 +18,7 @@ class Image(BaseModel):
 
     protected: bool
     status: str
-    size: int
+    size: Optional[int]
     virtual_size: Optional[int]
     visibility: str
 

{osi_dump-0.1.2.2 → osi_dump-0.1.2.3}/src/osi_dump/model/load_balancer.py

@@ -10,8 +10,10 @@ class LoadBalancer(BaseModel):
 
     load_balancer_name: Optional[str]
 
-    status: str
+    provisioning_status: Optional[str]
 
-    amphoraes: list[str]
+    operating_status: Optional[str]
+
+    amphoraes: list[dict]
 
     project_id: Optional[str]

osi_dump-0.1.2.3/src/osi_dump/util/panda_excel.py

@@ -0,0 +1,33 @@
+import pandas as pd
+
+from pandas import DataFrame
+
+
+def expand_list_column(df, column):
+    # Find the maximum length of the list in the column
+    max_len = df[column].apply(len).max()
+
+    # Expand each dictionary in the list into columns
+    expanded_df = pd.DataFrame(
+        df[column]
+        .apply(lambda x: [{**item} for item in x] + [{}] * (max_len - len(x)))
+        .tolist(),
+        index=df.index,
+    )
+
+    # Flatten the nested dictionaries into separate columns
+    expanded_df = pd.json_normalize(expanded_df.to_dict(orient="records"))
+
+    new_columns = []
+    for i in range(max_len):
+        for key in expanded_df.columns[i::max_len]:
+            new_key = key.split(".")[-1]  # Get the actual key name
+            new_columns.append(f"{column}_{i+1}.{new_key}")
+
+    # Rename the columns to reflect the nested structure
+    expanded_df.columns = new_columns
+
+    # Drop the original column and join the expanded columns
+    df = df.drop(column, axis=1).join(expanded_df)
+
+    return df

{osi_dump-0.1.2.2 → osi_dump-0.1.2.3}/src/osi_dump.egg-info/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.1
 Name: osi-dump
-Version: 0.1.2.2
-Summary: OpenStack instances information dump tool
+Version: 0.1.2.3
+Summary: OpenStack information dump tool
 Author: TVKain
 License: Apache-2.0
 Classifier: License :: OSI Approved :: Apache Software License
@@ -12,6 +12,7 @@ Requires-Dist: openstacksdk
 Requires-Dist: pydantic
 Requires-Dist: pandas
 Requires-Dist: openpyxl
+Requires-Dist: numpy
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 

{osi_dump-0.1.2.2 → osi_dump-0.1.2.3}/src/osi_dump.egg-info/SOURCES.txt

@@ -13,6 +13,7 @@ src/osi_dump/api/__init__.py
 src/osi_dump/api/octavia.py
 src/osi_dump/api/placement.py
 src/osi_dump/batch_handler/__init__.py
+src/osi_dump/batch_handler/external_port_batch_handler.py
 src/osi_dump/batch_handler/flavor_batch_handler.py
 src/osi_dump/batch_handler/floating_ip_batch_handler.py
 src/osi_dump/batch_handler/hypervisor_batch_handler.py
@@ -24,6 +25,9 @@ src/osi_dump/batch_handler/role_assignment_batch_handler.py
 src/osi_dump/batch_handler/router_batch_handler.py
 src/osi_dump/batch_handler/volume_batch_handler.py
 src/osi_dump/exporter/__init__.py
+src/osi_dump/exporter/external_port/__init__.py
+src/osi_dump/exporter/external_port/excel_external_port_exporter.py
+src/osi_dump/exporter/external_port/external_port_exporter.py
 src/osi_dump/exporter/flavor/__init__.py
 src/osi_dump/exporter/flavor/excel_flavor_exporter.py
 src/osi_dump/exporter/flavor/flavor_exporter.py
@@ -54,6 +58,9 @@ src/osi_dump/exporter/router/router_exporter.py
 src/osi_dump/exporter/volume/__init__.py
 src/osi_dump/exporter/volume/excel_volume_exporter.py
 src/osi_dump/exporter/volume/volume_exporter.py
+src/osi_dump/importer/external_port/__init__.py
+src/osi_dump/importer/external_port/external_port_importer.py
+src/osi_dump/importer/external_port/openstack_external_port_importer.py
 src/osi_dump/importer/flavor/__init__.py
 src/osi_dump/importer/flavor/flavor_importer.py
 src/osi_dump/importer/flavor/openstack_flavor_importer.py
@@ -86,6 +93,7 @@ src/osi_dump/importer/volume/openstack_volume_importer.py
 src/osi_dump/importer/volume/volume_importer.py
 src/osi_dump/model/__init__.py
 src/osi_dump/model/authentication_info.py
+src/osi_dump/model/external_port.py
 src/osi_dump/model/flavor.py
 src/osi_dump/model/floating_ip.py
 src/osi_dump/model/hypervisor.py

{osi_dump-0.1.2.2 → osi_dump-0.1.2.3}/src/osi_dump.egg-info/requires.txt

@@ -3,6 +3,7 @@ openstacksdk
 pydantic
 pandas
 openpyxl
+numpy
 
 [test]
 pytest

osi_dump-0.1.2.2/src/osi_dump/util/panda_excel.py

@@ -1,22 +0,0 @@
-import pandas as pd
-
-from pandas import DataFrame
-
-
-def expand_list_column(df: DataFrame, column: str) -> DataFrame:
-    # Find the maximum length of the list in the column
-    max_len = df[column].apply(len).max()
-
-    # Create a DataFrame from the list column
-    expanded_df = pd.DataFrame(
-        df[column].apply(lambda x: x + [None] * (max_len - len(x))).tolist(),
-        index=df.index,
-    )
-
-    # Rename the columns
-    expanded_df.columns = [f"{column}_{i+1}" for i in range(expanded_df.shape[1])]
-
-    # Drop the original column and join the expanded columns
-    df = df.drop(column, axis=1).join(expanded_df)
-
-    return df