osi-dump 0.1.5__py3-none-any.whl → 0.1.6__py3-none-any.whl

osi_dump/exporter/image/excel_image_exporter.py

@@ -1,28 +1,31 @@
 import pandas as pd
-
 import logging
-
+from typing import Generator
 
 from osi_dump import util
 from osi_dump.exporter.image.image_exporter import ImageExporter
-
 from osi_dump.model.image import Image
 
 logger = logging.getLogger(__name__)
 
-
 class ExcelImageExporter(ImageExporter):
     def __init__(self, sheet_name: str, output_file: str):
         self.sheet_name = sheet_name
         self.output_file = output_file
 
-    def export_images(self, images: list[Image]):
-        df = pd.json_normalize([image.model_dump() for image in images])
+    def export_images(self, images: Generator[Image, None, None]):
+        df = pd.json_normalize(image.model_dump() for image in images)
+
+        if df.empty:
+            logger.info(f"No images to export for {self.sheet_name}")
+            return
+            
+        if 'image_name' in df.columns:
+            df.sort_values(by='image_name', inplace=True, na_position='last')
 
         logger.info(f"Exporting images for {self.sheet_name}")
         try:
             util.export_data_excel(self.output_file, sheet_name=self.sheet_name, df=df)
-
             logger.info(f"Exported images for {self.sheet_name}")
         except Exception as e:
             logger.warning(f"Exporting images for {self.sheet_name} error: {e}")

osi_dump/exporter/image/image_exporter.py

@@ -1,7 +1,9 @@
 from abc import ABC, abstractmethod
-
+from typing import Generator
+from osi_dump.model.image import Image
 
 class ImageExporter(ABC):
     @abstractmethod
-    def export_images(self, images, output_file: str):
+    def export_images(self, images: Generator[Image, None, None]):
         pass
+

osi_dump/exporter/network/excel_network_exporter.py

@@ -1,32 +1,34 @@
 import logging
-
-
 import pandas as pd
-
+from typing import Generator
 
 from osi_dump.exporter.network.network_exporter import NetworkExporter 
-
 from osi_dump.model.network import Network 
-
 from osi_dump import util
 
 logger = logging.getLogger(__name__)
 
-
 class ExcelNetworkExporter(NetworkExporter):
     def __init__(self, sheet_name: str, output_file: str):
         self.sheet_name = sheet_name
         self.output_file = output_file
 
-    def export_networks(self, networks: list[Network]):
-        df = pd.json_normalize([network.model_dump() for network in networks])
+    def export_networks(self, networks: Generator[Network, None, None]):
+        df = pd.DataFrame(network.model_dump() for network in networks)
         
-        df = util.expand_list_column(df, "subnets")
+        if df.empty:
+            logger.info(f"No networks to export for {self.sheet_name}")
+            return
+            
+        if 'subnets' in df.columns:
+            df = util.expand_list_column(df, "subnets")
+
+        if 'name' in df.columns:
+            df.sort_values(by='name', inplace=True, na_position='last')
 
         logger.info(f"Exporting networks for {self.sheet_name}")
         try:
             util.export_data_excel(self.output_file, sheet_name=self.sheet_name, df=df)
-
             logger.info(f"Exported networks for {self.sheet_name}")
         except Exception as e:
-            logger.warning(f"Exporting networks for {self.sheet_name} error: {e}")
+            logger.warning(f"Exporting networks for {self.sheet_name} error: {e}")

osi_dump/exporter/network/network_exporter.py

@@ -1,7 +1,8 @@
 from abc import ABC, abstractmethod
-
+from typing import Generator
+from osi_dump.model.network import Network
 
 class NetworkExporter(ABC):
     @abstractmethod
-    def export_networks(self, networks, output_file: str):
-        pass
+    def export_networks(self, networks: Generator[Network, None, None]):
+        pass

osi_dump/exporter/project/excel_project_exporter.py

@@ -1,30 +1,30 @@
 import pandas as pd
-
 import logging
-
-from openpyxl import load_workbook
-
+from typing import Generator
 from osi_dump import util
 from osi_dump.exporter.project.project_exporter import ProjectExporter
-
 from osi_dump.model.project import Project
 
 logger = logging.getLogger(__name__)
 
-
 class ExcelProjectExporter(ProjectExporter):
     def __init__(self, sheet_name: str, output_file: str):
         self.sheet_name = sheet_name
         self.output_file = output_file
 
-    def export_projects(self, projects: list[Project]):
-
-        df = pd.DataFrame([project.model_dump() for project in projects])
+    def export_projects(self, projects: Generator[Project, None, None]):
+        df = pd.DataFrame(p.model_dump() for p in projects)
+        
+        if df.empty:
+            logger.info(f"No projects to export for {self.sheet_name}")
+            return
+            
+        if 'project_name' in df.columns:
+            df.sort_values(by='project_name', inplace=True, na_position='last')
 
         logger.info(f"Exporting projects for {self.sheet_name}")
         try:
             util.export_data_excel(self.output_file, sheet_name=self.sheet_name, df=df)
-
             logger.info(f"Exported projects for {self.sheet_name}")
         except Exception as e:
-            logger.warning(f"Exporting projects for {self.sheet_name} error: {e}")
+            logger.warning(f"Exporting projects for {self.sheet_name} error: {e}")

osi_dump/exporter/project/project_exporter.py

@@ -1,7 +1,8 @@
 from abc import ABC, abstractmethod
-
+from typing import Generator
+from osi_dump.model.project import Project
 
 class ProjectExporter(ABC):
     @abstractmethod
-    def export_projects(self, projects, output_file: str):
-        pass
+    def export_projects(self, projects: Generator[Project, None, None]):
+        pass

osi_dump/exporter/router/excel_router_exporter.py

@@ -1,30 +1,31 @@
 import pandas as pd
-
 import logging
-
-from openpyxl import load_workbook
+from typing import Generator
 
 from osi_dump import util
 from osi_dump.exporter.router.router_exporter import RouterExporter
-
 from osi_dump.model.router import Router
 
 logger = logging.getLogger(__name__)
 
-
 class ExcelRouterExporter(RouterExporter):
     def __init__(self, sheet_name: str, output_file: str):
         self.sheet_name = sheet_name
         self.output_file = output_file
 
-    def export_routers(self, routers: list[Router]):
+    def export_routers(self, routers: Generator[Router, None, None]):
+        df = pd.DataFrame(router.model_dump() for router in routers)
 
-        df = pd.DataFrame([router.model_dump() for router in routers])
+        if df.empty:
+            logger.info(f"No routers to export for {self.sheet_name}")
+            return
+            
+        if 'name' in df.columns:
+            df.sort_values(by='name', inplace=True, na_position='last')
 
         logger.info(f"Exporting routers for {self.sheet_name}")
         try:
             util.export_data_excel(self.output_file, sheet_name=self.sheet_name, df=df)
-
             logger.info(f"Exported routers for {self.sheet_name}")
         except Exception as e:
             logger.warning(f"Exporting routers for {self.sheet_name} error: {e}")

osi_dump/exporter/router/router_exporter.py

@@ -1,7 +1,8 @@
 from abc import ABC, abstractmethod
-
+from typing import Generator
+from osi_dump.model.router import Router
 
 class RouterExporter(ABC):
     @abstractmethod
-    def export_routers(self, routers, output_file: str):
+    def export_routers(self, routers: Generator[Router, None, None]):
         pass

osi_dump/exporter/volume/excel_volume_exporter.py

@@ -1,29 +1,31 @@
 import pandas as pd
-
 import logging
-
+from typing import Generator
 
 from osi_dump import util
-
 from osi_dump.exporter.volume.volume_exporter import VolumeExporter
-
 from osi_dump.model.volume import Volume
 
 logger = logging.getLogger(__name__)
 
-
 class ExcelVolumeExporter(VolumeExporter):
     def __init__(self, sheet_name: str, output_file: str):
         self.sheet_name = sheet_name
         self.output_file = output_file
 
-    def export_volumes(self, volumes: list[Volume]):
-        df = pd.DataFrame([volume.model_dump() for volume in volumes])
+    def export_volumes(self, volumes: Generator[Volume, None, None]):
+        df = pd.DataFrame(volume.model_dump() for volume in volumes)
+
+        if df.empty:
+            logger.info(f"No volumes to export for {self.sheet_name}")
+            return
+
+        if 'volume_name' in df.columns:
+            df.sort_values(by='volume_name', inplace=True, na_position='last')
 
         logger.info(f"Exporting volumes for {self.sheet_name}")
         try:
             util.export_data_excel(self.output_file, sheet_name=self.sheet_name, df=df)
-
             logger.info(f"Exported volumes for {self.sheet_name}")
         except Exception as e:
             logger.warning(f"Exporting volumes for {self.sheet_name} error: {e}")

osi_dump/exporter/volume/volume_exporter.py

@@ -1,7 +1,9 @@
 from abc import ABC, abstractmethod
-
+from typing import Generator
+from osi_dump.model.volume import Volume
 
 class VolumeExporter(ABC):
     @abstractmethod
-    def export_volumes(self, volumes, output_file: str):
+    def export_volumes(self, volumes: Generator[Volume, None, None]):
         pass
+

osi_dump/importer/external_port/external_port_importer.py

@@ -1,9 +1,8 @@
 from abc import ABC, abstractmethod
-
+from typing import Generator
 from osi_dump.model.external_port import ExternalPort
 
-
 class ExternalPortImporter(ABC):
     @abstractmethod
-    def import_external_ports(self) -> list[ExternalPort]:
-        pass
+    def import_external_ports(self) -> Generator[ExternalPort, None, None]:
+        pass

osi_dump/importer/external_port/openstack_external_port_importer.py

@@ -1,146 +1,74 @@
 import logging
-
 import ipaddress
-
-import concurrent
-
-import numpy as np
-
+from typing import Generator
 from openstack.connection import Connection
 from openstack.network.v2.port import Port as OSPort
 from openstack.network.v2.subnet import Subnet as OSSubnet
-from openstack.network.v2.floating_ip import FloatingIP as OSFloatingIP
-from openstack.network.v2.router import Router as OSRouter
 from openstack.network.v2.network import Network as OSNetwork
 
 from osi_dump.importer.external_port.external_port_importer import ExternalPortImporter
 from osi_dump.model.external_port import ExternalPort
-
 import osi_dump.api.neutron as osi_neutron
 
 logger = logging.getLogger(__name__)
 
-
 class OpenStackExternalPortImporter(ExternalPortImporter):
     def __init__(self, connection: Connection):
         self.connection = connection
 
-    def _get_external_ports(self) -> list[OSPort]:
-
-        # Get all external networks
-        networks = self.connection.network.networks(
-            is_router_external=True,
-        )
-
-        external_ports = []
-
-        for network in networks:
-            ex_ports = list(self.connection.network.ports(network_id=network.id))
-
-            for ex_port in ex_ports:
-                external_ports.append(ex_port)
-
-        # Get all port on those networks
-
-        return external_ports
-
-    def import_external_ports(self) -> list[ExternalPort]:
-        """Import external_ports information from Openstack
-
-
-        Raises:
-            Exception: Raises exception if fetching external_port failed
-
-        Returns:
-            list[Instance]: _description_
-        """
-
-        logger.info(f"Importing external_ports for {self.connection.auth['auth_url']}")
-
+    def _get_external_ports_stream(self) -> Generator[OSPort, None, None]:
+        """Generator để lấy về từng port trên các network ngoại vi."""
         try:
-            os_ports: list[OSPort] = self._get_external_ports()
+            external_networks = self.connection.network.networks(is_router_external=True)
+            
+            for network in external_networks:
+                ports_on_network = self.connection.network.ports(network_id=network.id)
+                yield from ports_on_network 
         except Exception as e:
-            raise Exception(
-                f"Can not fetch external_ports for {self.connection.auth['auth_url']} {e}"
-            ) from e
+            logger.error(f"Failed to stream external networks/ports: {e}")
 
-        external_ports: list[ExternalPort] = []
 
-        with concurrent.futures.ThreadPoolExecutor() as executor:
-            futures = [
-                executor.submit(self._get_external_port_info, external_port)
-                for external_port in os_ports
-            ]
-            for future in concurrent.futures.as_completed(futures):
-                external_ports.append(future.result())
-
-        logger.info(f"Imported external_ports for {self.connection.auth['auth_url']}")
-
-        external_ports = sorted(
-            external_ports,
-            key=lambda external_port: (
-                external_port.network_id,
-                (
-                    ipaddress.ip_address(external_port.ip_address)
-                    if external_port.ip_address
-                    else ipaddress.ip_address("0.0.0.0")
-                ),
-            ),
-        )
-
-        return external_ports
+    def import_external_ports(self) -> Generator[ExternalPort, None, None]:
+        logger.info(f"Importing external ports for {self.connection.auth['auth_url']}")
+        
+        for os_port in self._get_external_ports_stream():
+            yield self._get_external_port_info(os_port)
+            
+        logger.info(f"Finished importing external ports for {self.connection.auth['auth_url']}")
 
     def _get_external_port_info(self, external_port: OSPort) -> ExternalPort:
-
         subnet_id = None
         ip_address = None
         subnet_cidr = None
 
-        try:
-            ip_address = external_port.fixed_ips[0]["ip_address"]
-        except Exception as e:
-            logger.warning(f"No IP address found for port {external_port.id}")
-
-        try:
-            subnet_id = external_port.fixed_ips[0]["subnet_id"]
-        except Exception as e:
-            logger.warning(f"No subnet ID found for {external_port.id}")
-
-        try:
-            subnet: OSSubnet = self.connection.get_subnet_by_id(subnet_id)
-
-            subnet_cidr = subnet.cidr
-        except Exception as e:
-            logger.warning(f"No subnet cidr found for port {external_port.id}")
-
-        mapped_device_project = ["network:floatingip", "network:router_gateway"]
-
-        project_id = (
-            external_port.project_id
-            if external_port.device_owner not in mapped_device_project
-            else self._map_project_id(
+        if external_port.fixed_ips:
+            ip_address = external_port.fixed_ips[0].get("ip_address")
+            subnet_id = external_port.fixed_ips[0].get("subnet_id")
+
+        if subnet_id:
+            try:
+                subnet: OSSubnet = self.connection.get_subnet(subnet_id)
+                if subnet:
+                    subnet_cidr = subnet.cidr
+            except Exception as e:
+                logger.warning(f"Could not get subnet cidr for {subnet_id}: {e}")
+
+        project_id = external_port.project_id
+        mapped_device_projects = ["network:floatingip", "network:router_gateway"]
+        if external_port.device_owner in mapped_device_projects and external_port.device_id:
+            project_id = self._map_project_id(
                 device_owner=external_port.device_owner,
                 device_id=external_port.device_id,
             )
-        )
         
-        network: OSNetwork = self.connection.get_network(name_or_id=external_port.network_id)
-
-        vlan_id = None
+        network: OSNetwork = self.connection.get_network(external_port.network_id)
+        vlan_id = network.provider_segmentation_id if network else None
         
-        try: 
-            if not network.segments: 
-                vlan_id = network.provider_segmentation_id
-            else: 
-                logger.warning(f"Multiple segments mapped for net id {network.id}")
-        except Exception as e: 
-            logger.warning(f"Can not get vlan ID for external port {external_port.id}, net id {network.id}")
-        
-        external_port_ret = ExternalPort(
+        return ExternalPort(
             port_id=external_port.id,
             project_id=project_id,
             network_id=external_port.network_id,
-            network_name=network.name,
+            network_name=network.name if network else None,
             subnet_id=subnet_id,
             subnet_cidr=subnet_cidr,
             ip_address=ip_address,
@@ -151,27 +79,12 @@ class OpenStackExternalPortImporter(ExternalPortImporter):
             vlan_id=vlan_id
         )
 
-        return external_port_ret
-
     def _map_project_id(self, device_owner: str, device_id: str) -> str:
-
-        if device_owner == "network:router_gateway":
-            return self._map_project_id_router(device_id=device_id)
-        elif device_owner == "network:floatingip":
-            return self._map_project_id_floating(device_id=device_id)
-
-        return ""
-
-    def _map_project_id_router(self, device_id: str) -> str:
-        project_id = osi_neutron.get_router_project(
-            self.connection, router_id=device_id
-        )
-
-        return project_id
-
-    def _map_project_id_floating(self, device_id: str) -> str:
-        project_id = osi_neutron.get_floating_ip_project(
-            self.connection, floating_ip_id=device_id
-        )
-
-        return project_id
+        try:
+            if device_owner == "network:router_gateway":
+                return osi_neutron.get_router_project(self.connection, router_id=device_id)
+            elif device_owner == "network:floatingip":
+                return osi_neutron.get_floating_ip_project(self.connection, floating_ip_id=device_id)
+        except Exception as e:
+            logger.warning(f"Could not map project ID for {device_owner}:{device_id}: {e}")
+        return None

osi_dump/importer/flavor/flavor_importer.py

@@ -1,9 +1,9 @@
 from abc import ABC, abstractmethod
-
+from typing import Generator
 from osi_dump.model.flavor import Flavor
 
-
 class FlavorImporter(ABC):
     @abstractmethod
-    def import_flavors(self) -> list[Flavor]:
+    def import_flavors(self) -> Generator[Flavor, None, None]:
         pass
+

osi_dump/importer/flavor/openstack_flavor_importer.py

@@ -1,7 +1,5 @@
 import logging
-
-import concurrent
-
+from typing import Generator
 from openstack.connection import Connection
 from openstack.compute.v2.flavor import Flavor as OSFlavor
 
@@ -10,55 +8,35 @@ from osi_dump.model.flavor import Flavor
 
 logger = logging.getLogger(__name__)
 
-
 class OpenStackFlavorImporter(FlavorImporter):
     def __init__(self, connection: Connection):
         self.connection = connection
 
-    def import_flavors(self) -> list[Flavor]:
-        """Import flavors information from Openstack
-
-        Raises:
-            Exception: Raises exception if fetching flavor failed
-
-        Returns:
-            list[Instance]: _description_
-        """
-
+    def import_flavors(self) -> Generator[Flavor, None, None]:
         logger.info(f"Importing flavors for {self.connection.auth['auth_url']}")
-
         try:
-            osflavors: list[OSFlavor] = list(self.connection.list_flavors())
-        except Exception as e:
-            raise Exception(
-                f"Can not fetch flavors for {self.connection.auth['auth_url']}"
-            ) from e
-
-        flavors: list[Flavor] = []
+            flavor_iterator = self.connection.list_flavors()
 
-        with concurrent.futures.ThreadPoolExecutor() as executor:
-            futures = [
-                executor.submit(self._get_flavor_info, osflavor)
-                for osflavor in osflavors
-            ]
-            for future in concurrent.futures.as_completed(futures):
-                flavors.append(future.result())
+            for osflavor in flavor_iterator:
+                yield self._get_flavor_info(osflavor)
 
-        logger.info(f"Imported flavors for {self.connection.auth['auth_url']}")
+        except Exception as e:
+            logger.error(f"Cannot fetch flavors for {self.connection.auth['auth_url']}: {e}")
+            return 
 
-        return flavors
+        logger.info(f"Finished importing flavors for {self.connection.auth['auth_url']}")
 
     def _get_flavor_info(self, flavor: OSFlavor) -> Flavor:
 
-        ret_flavor = Flavor(
+        swap_val = flavor.swap if flavor.swap else None
+
+        return Flavor(
             flavor_id=flavor.id,
             flavor_name=flavor.name,
             ram=flavor.ram,
             vcpus=flavor.vcpus,
             disk=flavor.disk,
-            swap=flavor.swap,
+            swap=swap_val,
             public=flavor.is_public,
             properties=flavor.extra_specs,
         )
-
-        return ret_flavor

osi_dump/importer/floating_ip/floating_ip_importer.py

@@ -1,9 +1,9 @@
 from abc import ABC, abstractmethod
-
+from typing import Generator
 from osi_dump.model.floating_ip import FloatingIP
 
-
 class FloatingIPImporter(ABC):
     @abstractmethod
-    def import_floating_ips(self) -> list[FloatingIP]:
+    def import_floating_ips(self) -> Generator[FloatingIP, None, None]:
         pass
+