osi-dump 0.1.5__py3-none-any.whl → 0.1.6__py3-none-any.whl

osi_dump/importer/floating_ip/openstack_floating_ip_importer.py

@@ -1,7 +1,5 @@
 import logging
-
-import concurrent
-
+from typing import Generator
 from openstack.connection import Connection
 from openstack.network.v2.floating_ip import FloatingIP as OSFloatingIP
 
@@ -10,49 +8,26 @@ from osi_dump.model.floating_ip import FloatingIP
 
 logger = logging.getLogger(__name__)
 
-
 class OpenStackFloatingIPImporter(FloatingIPImporter):
     def __init__(self, connection: Connection):
         self.connection = connection
 
-    def import_floating_ips(self) -> list[FloatingIP]:
-        """Import instances information from Openstack
-
-        Raises:
-            Exception: Raises exception if fetching server failed
-
-        Returns:
-            list[Instance]: _description_
-        """
-
+    def import_floating_ips(self) -> Generator[FloatingIP, None, None]:
         logger.info(f"Importing floating ips for {self.connection.auth['auth_url']}")
-
         try:
-            osfloating_ips: list[OSFloatingIP] = list(
-                self.connection.list_floating_ips()
-            )
-        except Exception as e:
-            raise Exception(
-                f"Can not fetch floating IPs for {self.connection.auth['auth_url']}"
-            ) from e
-
-        floating_ips: list[FloatingIP] = []
-
-        with concurrent.futures.ThreadPoolExecutor() as executor:
-            futures = [
-                executor.submit(self._get_floating_ip_info, osfloating_ip)
-                for osfloating_ip in osfloating_ips
-            ]
-            for future in concurrent.futures.as_completed(futures):
-                floating_ips.append(future.result())
+            osfloating_ip_iterator = self.connection.list_floating_ips()
+            
+            for floating_ip in osfloating_ip_iterator:
+                yield self._get_floating_ip_info(floating_ip)
 
-        logger.info(f"Imported floating ips for {self.connection.auth['auth_url']}")
+        except Exception as e:
+            logger.error(f"Cannot fetch floating IPs for {self.connection.auth['auth_url']}: {e}")
+            return 
 
-        return floating_ips
+        logger.info(f"Finished importing floating ips for {self.connection.auth['auth_url']}")
 
     def _get_floating_ip_info(self, floating_ip: OSFloatingIP) -> FloatingIP:
-
-        ret_floating_ip = FloatingIP(
+        return FloatingIP(
             floating_ip_id=floating_ip.id,
             project_id=floating_ip.project_id,
             floating_ip_address=floating_ip.floating_ip_address,
@@ -64,5 +39,3 @@ class OpenStackFloatingIPImporter(FloatingIPImporter):
             created_at=floating_ip.created_at,
             updated_at=floating_ip.updated_at,
         )
-
-        return ret_floating_ip

osi_dump/importer/hypervisor/hypervisor_importer.py

@@ -1,9 +1,8 @@
 from abc import ABC, abstractmethod
-
+from typing import Generator
 from osi_dump.model.hypervisor import Hypervisor
 
-
 class HypervisorImporter(ABC):
     @abstractmethod
-    def import_hypervisors(self) -> list[Hypervisor]:
-        pass
+    def import_hypervisors(self) -> Generator[Hypervisor, None, None]:
+        pass

osi_dump/importer/hypervisor/openstack_hypervisor_importer.py

@@ -1,124 +1,84 @@
 import logging
-
-import concurrent
-
+from typing import Generator
 from openstack.connection import Connection
 from openstack.compute.v2.hypervisor import Hypervisor as OSHypervisor
 from openstack.compute.v2.aggregate import Aggregate as OSAggregate
-
 from openstack.placement.v1._proxy import Proxy as PlacementProxy
 from openstack.placement.v1.resource_provider_inventory import ResourceProviderInventory
 
 from osi_dump.importer.hypervisor.hypervisor_importer import HypervisorImporter
 from osi_dump.model.hypervisor import Hypervisor
-
 from osi_dump.api.placement import get_usage
 
 logger = logging.getLogger(__name__)
 
-
 class OpenStackHypervisorImporter(HypervisorImporter):
     def __init__(self, connection: Connection):
         self.connection = connection
 
-    def import_hypervisors(self) -> list[Hypervisor]:
-        """Import hypervisors information from Openstack
-
-        Raises:
-            Exception: Raises exception if fetching hypervisor failed
-
-        Returns:
-            list[Hypervisor]: _description_
-        """
-        aggregates = list(self.connection.list_aggregates())
-
+    def import_hypervisors(self) -> Generator[Hypervisor, None, None]:
+        logger.info("Pre-caching aggregates...")
         try:
-            oshypervisors: list[OSHypervisor] = list(
-                self.connection.compute.hypervisors(details=True, with_servers=True)
-            )
-
+            aggregates = list(self.connection.list_aggregates())
         except Exception as e:
-            raise Exception(
-                f"Can not fetch hypervisor for {self.connection.auth['auth_url']}"
-            ) from e
-
-        hypervisors: list[Hypervisor] = []
-
-        with concurrent.futures.ThreadPoolExecutor() as executor:
-            futures = [
-                executor.submit(self._get_hypervisor_info, hypervisor, aggregates)
-                for hypervisor in oshypervisors
-            ]
-            for future in concurrent.futures.as_completed(futures):
-                hypervisors.append(future.result())
-
-        logger.info(f"Imported hypervisors for {self.connection.auth['auth_url']}")
-
-        return hypervisors
-
-
-    def _normalize_hypervisor_aggregate(self, hypervisors: list[Hypervisor]):
-        
-        aggregate_id_map = {
-
-        }
-
-        aggregates: list[list[dict]] = [
+            logger.warning(f"Could not pre-cache aggregates: {e}")
+            aggregates = []
 
-        ]
-
-        for hypervisor in hypervisors:
-            aggregates.append(hypervisor.aggregates)
-
-    def _swap_element(array, i, j): 
-        array[i], array[j] = array[j], array[i]
+        logger.info(f"Importing hypervisors for {self.connection.auth['auth_url']}")
+        try:
+            hypervisor_iterator = self.connection.compute.hypervisors(details=True, with_servers=True)
+            for hypervisor in hypervisor_iterator:
+                yield self._get_hypervisor_info(hypervisor, aggregates)
+        except Exception as e:
+            logger.error(f"Cannot fetch hypervisors for {self.connection.auth['auth_url']}: {e}")
+            return 
+            
+        logger.info(f"Finished importing hypervisors for {self.connection.auth['auth_url']}")
 
     def _get_hypervisor_info(
         self, hypervisor: OSHypervisor, aggregates: list[OSAggregate]
     ) -> Hypervisor:
-        aggregate_list, az = self._get_aggregates(hypervisor=hypervisor)
+        aggregate_list, az = self._get_aggregates(hypervisor=hypervisor, all_aggregates=aggregates)
 
         placement_proxy: PlacementProxy = self.connection.placement
-
-        rpi: ResourceProviderInventory = list(
-            placement_proxy.resource_provider_inventories(
-                resource_provider=hypervisor.id
+        rpi: list[ResourceProviderInventory] = []
+        usage_data = {}
+        
+        try:
+            rpi = list(
+                placement_proxy.resource_provider_inventories(
+                    resource_provider=hypervisor.id
+                )
             )
-        )
-
-        usage_data = get_usage(self.connection, resource_provider_id=hypervisor.id)
+            usage_data = get_usage(self.connection, resource_provider_id=hypervisor.id)
+        except Exception as e:
+            logger.warning(f"Could not get placement data for hypervisor {hypervisor.name}: {e}")
 
-        vcpu = rpi[0]
-        memory = rpi[1]
-        disk = rpi[2]
+        vcpu = rpi[0] if len(rpi) > 0 else {}
+        memory = rpi[1] if len(rpi) > 1 else {}
+        disk = rpi[2] if len(rpi) > 2 else {}
 
-        ret_hypervisor = Hypervisor(
+        return Hypervisor(
             hypervisor_id=hypervisor.id,
             hypervisor_type=hypervisor.hypervisor_type,
             name=hypervisor.name,
             state=hypervisor.state,
             status=hypervisor.status,
-            local_disk_size=disk["max_unit"],
-            memory_size=memory["max_unit"] + memory["reserved"],
-            vcpus=vcpu["max_unit"],
-            vcpus_usage=usage_data["VCPU"],
-            memory_usage=usage_data["MEMORY_MB"],
-            local_disk_usage=usage_data["DISK_GB"],
+            local_disk_size=disk.get("max_unit", 0),
+            memory_size=memory.get("max_unit", 0) + memory.get("reserved", 0),
+            vcpus=vcpu.get("max_unit", 0),
+            vcpus_usage=usage_data.get("VCPU", 0),
+            memory_usage=usage_data.get("MEMORY_MB", 0),
+            local_disk_usage=usage_data.get("DISK_GB", 0),
             vm_count=len(hypervisor.servers),
             aggregates=aggregate_list,
             availability_zone=az,
         )
 
-        return ret_hypervisor
-
-    def _get_aggregates(self, hypervisor: OSHypervisor):
+    def _get_aggregates(self, hypervisor: OSHypervisor, all_aggregates: list[OSAggregate]):
         aggregates_ret = []
-
-        aggregates: OSAggregate = list(self.connection.list_aggregates())
-
         az = None
-
-        for aggregate in aggregates:
+        for aggregate in all_aggregates:
             if hypervisor.name in aggregate.hosts:
                 aggregates_ret.append(
                     {
@@ -126,12 +86,8 @@ class OpenStackHypervisorImporter(HypervisorImporter):
                         "name": aggregate.name,
                     }
                 )
-
-                if aggregate.availability_zone != None:
+                if aggregate.availability_zone is not None:
                     az = aggregate.availability_zone
-
-        aggregates_ret = [
-            dict(sorted(aggregate.items())) for aggregate in aggregates_ret
-        ]
-
-        return aggregates_ret, az
+        
+        aggregates_ret = sorted(aggregates_ret, key=lambda x: x['name'])
+        return aggregates_ret, az

osi_dump/importer/image/image_importer.py

@@ -1,9 +1,9 @@
 from abc import ABC, abstractmethod
-
+from typing import Generator
 from osi_dump.model.image import Image
 
-
 class ImageImporter(ABC):
     @abstractmethod
-    def import_images(self) -> list[Image]:
+    def import_images(self) -> Generator[Image, None, None]:
         pass
+

osi_dump/importer/image/openstack_image_importer.py

@@ -1,7 +1,5 @@
 import logging
-
-import concurrent
-
+from typing import Generator
 from openstack.connection import Connection
 from openstack.image.v2.image import Image as OSImage
 
@@ -10,64 +8,40 @@ from osi_dump.model.image import Image
 
 logger = logging.getLogger(__name__)
 
-
 class OpenStackImageImporter(ImageImporter):
     def __init__(self, connection: Connection):
         self.connection = connection
 
-    def import_images(self) -> list[Image]:
-        """Import instances information from Openstack
-
-        Raises:
-            Exception: Raises exception if fetching server failed
-
-        Returns:
-            list[Instance]: _description_
-        """
-
+    def import_images(self) -> Generator[Image, None, None]:
         logger.info(f"Importing images for {self.connection.auth['auth_url']}")
-
         try:
+            os_image_iterator = self.connection.list_images(show_all=True)
+            
+            for os_image in os_image_iterator:
+                yield self._get_image_info(os_image)
 
-            os_images: list[OSImage] = list(self.connection.list_images(show_all=True))
         except Exception as e:
-            raise Exception(
-                f"Can not fetch images for {self.connection.auth['auth_url']}"
-            ) from e
+            logger.error(f"Cannot fetch images for {self.connection.auth['auth_url']}: {e}")
+            return
 
-        images: list[OSImage] = []
+        logger.info(f"Finished importing images for {self.connection.auth['auth_url']}")
 
-        with concurrent.futures.ThreadPoolExecutor() as executor:
-            futures = [
-                executor.submit(self._get_image_info, image) for image in os_images
-            ]
-            for future in concurrent.futures.as_completed(futures):
-                images.append(future.result())
-
-        logger.info(f"Imported images for {self.connection.auth['auth_url']}")
-
-        return images
 
     def _get_image_info(self, os_image: OSImage) -> Image:
+        properties = os_image.properties or {}
+        properties.pop("owner_specified.openstack.md5", None)
+        properties.pop("owner_specified.openstack.sha256", None)
+        properties.pop("owner_specified.openstack.object", None)
+        properties.pop("stores", None)
 
-        try:
-            properties: dict = os_image.properties
-
-            properties.pop("owner_specified.openstack.md5", None)
-            properties.pop("owner_specified.openstack.sha256", None)
-            properties.pop("owner_specified.openstack.object", None)
-            properties.pop("stores", None)
-        except Exception as e: 
-            logger.warn(f"properties for {os_image.id} is None")
-
-        image = Image(
+        return Image(
             image_id=os_image.id,
             disk_format=os_image.disk_format,
             min_disk=os_image.min_disk,
             min_ram=os_image.min_ram,
             image_name=os_image.name,
             owner=os_image.owner,
-            properties=os_image.properties,
+            properties=properties,
             protected=os_image.is_protected,
             status=os_image.status,
             os_distro=os_image.os_distro,
@@ -77,5 +51,3 @@ class OpenStackImageImporter(ImageImporter):
             created_at=os_image.created_at,
             updated_at=os_image.updated_at,
         )
-
-        return image

osi_dump/importer/network/network_importer.py

@@ -1,9 +1,8 @@
 from abc import ABC, abstractmethod
-
+from typing import Generator
 from osi_dump.model.network import Network
 
-
 class NetworkImporter(ABC):
     @abstractmethod
-    def import_networks(self) -> list[Network]:
-        pass
+    def import_networks(self) -> Generator[Network, None, None]:
+        pass

osi_dump/importer/network/openstack_network_importer.py

@@ -1,59 +1,33 @@
 import logging
-
-import concurrent
-
+from typing import Generator
 from openstack.network.v2.network import Network as OSNetwork
 from openstack.network.v2.subnet import Subnet as OSSubnet
-
 from openstack.connection import Connection
 
 from osi_dump.importer.network.network_importer import NetworkImporter 
 from osi_dump.model.network import Network 
 
-
 logger = logging.getLogger(__name__)
 
-
 class OpenStackNetworkImporter(NetworkImporter):
     def __init__(self, connection: Connection):
         self.connection = connection
 
-    def import_networks(self) -> list[Network]:
-        """Import networks information from Openstack
-
-        Raises:
-            Exception: Raises exception if fetching networks failed
-
-        Returns:
-            list[Network]: _description_
-        """
-
+    def import_networks(self) -> Generator[Network, None, None]:
+        logger.info(f"Importing networks for {self.connection.auth['auth_url']}")
         try:
-            os_networks: list[OSNetwork] = list(self.connection.list_networks())
-        except Exception as e:
-            raise Exception(
-                f"Can not fetch hypervisor for {self.connection.auth['auth_url']}"
-            ) from e
-
-        networks: list[Network] = []
+            network_iterator = self.connection.list_networks()
 
-        with concurrent.futures.ThreadPoolExecutor() as executor:
-            futures = [
-                executor.submit(self._get_network_info, network)
-                for network in os_networks
-            ]
-            for future in concurrent.futures.as_completed(futures):
-                networks.append(future.result())
-
-        logger.info(f"Imported networks for {self.connection.auth['auth_url']}")
-
-        return networks
+            for network in network_iterator:
+                yield self._get_network_info(network)
 
+        except Exception as e:
+            logger.error(f"Cannot fetch networks for {self.connection.auth['auth_url']}: {e}")
+            return 
 
-    def _get_network_info(
-        self, network: OSNetwork, 
-    ) -> Network:
+        logger.info(f"Finished importing networks for {self.connection.auth['auth_url']}")
 
+    def _get_network_info(self, network: OSNetwork) -> Network:
         subnets = self._get_subnets_info(subnet_ids=network.subnet_ids)
 
         return Network(
@@ -72,23 +46,20 @@ class OpenStackNetworkImporter(NetworkImporter):
             subnets=subnets
         )
     
-
     def _get_subnets_info(self, subnet_ids: list[str]) -> list[dict]: 
         subnets = []
+        if not subnet_ids:
+            return subnets
 
         for subnet_id in subnet_ids: 
-            os_subnet: OSSubnet = self.connection.get_subnet(name_or_id=subnet_id) 
-
-            if not os_subnet: 
-                continue 
+            try:
+                os_subnet: OSSubnet = self.connection.get_subnet(subnet_id) 
+                if os_subnet: 
+                    subnets.append({
+                        "id": os_subnet.id, 
+                        "cidr": os_subnet.cidr
+                    })
+            except Exception as e:
+                logger.warning(f"Could not get info for subnet {subnet_id}: {e}")
                 
-            subnets.append({
-                "id": os_subnet.id, 
-                "cidr": os_subnet.cidr
-            })
-
-        return subnets
-
-
-
-    
+        return subnets

osi_dump/importer/project/openstack_project_importer.py

@@ -1,7 +1,5 @@
 import logging
-
-import concurrent
-
+from typing import Generator
 from openstack.connection import Connection
 from openstack.identity.v3.project import Project as OSProject
 from openstack.load_balancer.v2.load_balancer import LoadBalancer as OSLoadBalancer
@@ -9,101 +7,67 @@ from openstack.load_balancer.v2.load_balancer import LoadBalancer as OSLoadBalan
 from osi_dump.importer.project.project_importer import ProjectImporter
 from osi_dump.model.project import Project
 import osi_dump.api.octavia as octavia_api
-logger = logging.getLogger(__name__)
 
+logger = logging.getLogger(__name__)
 
 class OpenStackProjectImporter(ProjectImporter):
     def __init__(self, connection: Connection):
         self.connection = connection
 
-    def import_projects(self) -> list[Project]:
-        """Import projects information from Openstack
-
-        Raises:
-            Exception: Raises exception if fetching project failed
-
-        Returns:
-            list[Instance]: _description_
-        """
-
-        logger.info(f"Importing projects for {self.connection.auth['auth_url']}")
+    def import_projects(self) -> Generator[Project, None, None]:
+        project_lb_dict = {}
+        logger.info(f"Pre-caching load balancers for project count on {self.connection.auth['auth_url']}")
         try:
-            osload_balancers: list[OSLoadBalancer] = octavia_api.get_load_balancers(
-                connection=self.connection
-            )
+            osload_balancers = octavia_api.get_load_balancers(connection=self.connection)
+            if osload_balancers:
+                for lb in osload_balancers:
+                    project_id = lb.get("project_id")
+                    if project_id:
+                        project_lb_dict[project_id] = project_lb_dict.get(project_id, 0) + 1
         except Exception as e:
-            raise Exception(
-                f"Can not fetch load_balancers for {self.connection.auth['auth_url']} {e}"
-            ) from e
-
-        project_lb_dict = {}
-
-        for osload_balancer in osload_balancers: 
-            if project_lb_dict.get(osload_balancer["project_id"]):
-                project_lb_dict[osload_balancer["project_id"]] += 1 
-            else: 
-                project_lb_dict[osload_balancer["project_id"]] = 1
+            logger.warning(f"Could not pre-cache load balancers for project count: {e}")
 
+        logger.info(f"Importing projects for {self.connection.auth['auth_url']}")
         try:
-            osprojects: list[OSProject] = list(self.connection.identity.projects())
+            project_iterator = self.connection.identity.projects()
+            for project in project_iterator:
+                yield self._get_project_info(project, project_lb_dict)
         except Exception as e:
-            raise Exception(
-                f"Can not fetch projects for {self.connection.auth['auth_url']}"
-            ) from e
-
-        projects: list[Project] = []
-
-        with concurrent.futures.ThreadPoolExecutor() as executor:
-            futures = [
-                executor.submit(self._get_project_info, project, project_lb_dict)
-                for project in osprojects
-            ]
-            for future in concurrent.futures.as_completed(futures):
-                projects.append(future.result())
-
-        logger.info(f"Imported projects for {self.connection.auth['auth_url']}")
-
-        return projects
+            logger.error(f"Cannot fetch projects for {self.connection.auth['auth_url']}: {e}")
+            return 
+            
+        logger.info(f"Finished importing projects for {self.connection.auth['auth_url']}")
 
     def _get_project_info(self, project: OSProject, project_lb_dict: dict) -> Project:
-        
-        usage_instance=None
-        quota_instance=None
-        usage_ram=None
-        quota_ram=None
-        usage_vcpu=None
-        quota_vcpu=None
+        usage_instance, quota_instance = None, None
+        usage_ram, quota_ram = None, None
+        usage_vcpu, quota_vcpu = None, None
+        usage_volume, quota_volume = None, None
+        usage_snapshot, quota_snapshot = None, None
+        usage_storage, quota_storage = None, None
 
         try:
-            compute_quotas = self.connection.compute.get_quota_set(
-                project.id, usage=True
-            )
-
-            usage_instance=compute_quotas.usage["instances"]
-            quota_instance=compute_quotas.instances
-            usage_ram=compute_quotas.usage["ram"]
-            quota_ram=compute_quotas.ram
-            usage_vcpu=compute_quotas.usage["cores"]
-            quota_vcpu=compute_quotas.cores
+            compute_quotas = self.connection.compute.get_quota_set(project.id, usage=True)
+            if compute_quotas and compute_quotas.usage:
+                usage_instance = compute_quotas.usage.get("instances")
+                quota_instance = compute_quotas.instances
+                usage_ram = compute_quotas.usage.get("ram")
+                quota_ram = compute_quotas.ram
+                usage_vcpu = compute_quotas.usage.get("cores")
+                quota_vcpu = compute_quotas.cores
         except Exception as e:
             logger.warning(f"Get compute quotas failed for {project.id} error: {e}")
 
-        usage_volume=None
-        quota_volume=None
-        usage_snapshot=None
-        quota_snapshot=None
-        usage_storage=None
-        quota_storage=None
+        # Lấy thông tin storage quotas
         try:
-            storage_quotas = self.connection.block_storage.get_quota_set(
-                project.id, usage=True
-            )
-            usage_volume=storage_quotas.usage["volumes"]
-            quota_volume=storage_quotas.volumes
-            usage_snapshot=storage_quotas.usage["snapshots"]
-            quota_snapshot=storage_quotas.snapshots
-            usage_storage=storage_quotas.usage["gigabytes"]
-            quota_storage=storage_quotas.gigabytes
+            storage_quotas = self.connection.block_storage.get_quota_set(project.id, usage=True)
+            if storage_quotas and storage_quotas.usage:
+                usage_volume = storage_quotas.usage.get("volumes")
+                quota_volume = storage_quotas.volumes
+                usage_snapshot = storage_quotas.usage.get("snapshots")
+                quota_snapshot = storage_quotas.snapshots
+                usage_storage = storage_quotas.usage.get("gigabytes")
+                quota_storage = storage_quotas.gigabytes
         except Exception as e:
             logger.warning(f"Get storage quotas failed for {project.id} error: {e}")
 
@@ -116,7 +80,7 @@ class OpenStackProjectImporter(ProjectImporter):
 
         lb_count = project_lb_dict.get(project.id)
 
-        project_ret = Project(
+        return Project(
             project_id=project.id,
             project_name=project.name,
             domain_id=project.domain_id,
@@ -136,6 +100,4 @@ class OpenStackProjectImporter(ProjectImporter):
             usage_storage=usage_storage,
             quota_storage=quota_storage,
             load_balancer_count=lb_count
-        )
-
-        return project_ret
+        )