osi-dump 0.1.2__py3-none-any.whl → 0.1.2.2__py3-none-any.whl

osi_dump/importer/flavor/openstack_flavor_importer.py

@@ -0,0 +1,64 @@
+import logging
+
+import concurrent
+
+from openstack.connection import Connection
+from openstack.compute.v2.flavor import Flavor as OSFlavor
+
+from osi_dump.importer.flavor.flavor_importer import FlavorImporter
+from osi_dump.model.flavor import Flavor
+
+logger = logging.getLogger(__name__)
+
+
+class OpenStackFlavorImporter(FlavorImporter):
+    def __init__(self, connection: Connection):
+        self.connection = connection
+
+    def import_flavors(self) -> list[Flavor]:
+        """Import flavors information from Openstack
+
+        Raises:
+            Exception: Raises exception if fetching flavor failed
+
+        Returns:
+            list[Instance]: _description_
+        """
+
+        logger.info(f"Importing flavors for {self.connection.auth['auth_url']}")
+
+        try:
+            osflavors: list[OSFlavor] = list(self.connection.list_flavors())
+        except Exception as e:
+            raise Exception(
+                f"Can not fetch flavors for {self.connection.auth['auth_url']}"
+            ) from e
+
+        flavors: list[Flavor] = []
+
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            futures = [
+                executor.submit(self._get_flavor_info, osflavor)
+                for osflavor in osflavors
+            ]
+            for future in concurrent.futures.as_completed(futures):
+                flavors.append(future.result())
+
+        logger.info(f"Imported flavors for {self.connection.auth['auth_url']}")
+
+        return flavors
+
+    def _get_flavor_info(self, flavor: OSFlavor) -> Flavor:
+
+        ret_flavor = Flavor(
+            flavor_id=flavor.id,
+            flavor_name=flavor.name,
+            ram=flavor.ram,
+            vcpus=flavor.vcpus,
+            disk=flavor.disk,
+            swap=flavor.swap,
+            public=flavor.is_public,
+            properties=flavor.extra_specs,
+        )
+
+        return ret_flavor

osi_dump/importer/hypervisor/openstack_hypervisor_importer.py

@@ -33,7 +33,9 @@ class OpenStackHypervisorImporter(HypervisorImporter):
         logger.info(f"Importing hypervisors for {self.connection.auth['auth_url']}")
 
         try:
-            oshypervisors: list[OSHypervisor] = list(self.connection.list_hypervisors())
+            oshypervisors: list[OSHypervisor] = list(
+                self.connection.compute.hypervisors(details=True)
+            )
         except Exception as e:
             raise Exception(
                 f"Can not fetch hypervisor for {self.connection.auth['auth_url']}"
@@ -63,6 +65,12 @@ class OpenStackHypervisorImporter(HypervisorImporter):
             )
         )
 
+        servers = list(
+            self.connection.compute.servers(
+                details=True, all_project=True, hypervisor_hostname=hypervisor.name
+            )
+        )
+
         usage_data = get_usage(self.connection, resource_provider_id=hypervisor.id)
 
         vcpu = rpi[0]
@@ -81,6 +89,7 @@ class OpenStackHypervisorImporter(HypervisorImporter):
             vcpus_usage=usage_data["VCPU"],
             memory_usage=usage_data["MEMORY_MB"],
             local_disk_usage=usage_data["DISK_GB"],
+            vm_count=len(servers),
         )
 
         return ret_hypervisor

osi_dump/importer/image/image_importer.py

@@ -0,0 +1,9 @@
+from abc import ABC, abstractmethod
+
+from osi_dump.model.image import Image
+
+
+class ImageImporter(ABC):
+    @abstractmethod
+    def import_images(self) -> list[Image]:
+        pass

osi_dump/importer/image/openstack_image_importer.py

@@ -0,0 +1,77 @@
+import logging
+
+import concurrent
+
+from openstack.connection import Connection
+from openstack.image.v2.image import Image as OSImage
+
+from osi_dump.importer.image.image_importer import ImageImporter
+from osi_dump.model.image import Image
+
+logger = logging.getLogger(__name__)
+
+
+class OpenStackImageImporter(ImageImporter):
+    def __init__(self, connection: Connection):
+        self.connection = connection
+
+    def import_images(self) -> list[Image]:
+        """Import instances information from Openstack
+
+        Raises:
+            Exception: Raises exception if fetching server failed
+
+        Returns:
+            list[Instance]: _description_
+        """
+
+        logger.info(f"Importing images for {self.connection.auth['auth_url']}")
+
+        try:
+
+            os_images: list[OSImage] = list(self.connection.list_images(show_all=True))
+        except Exception as e:
+            raise Exception(
+                f"Can not fetch images for {self.connection.auth['auth_url']}"
+            ) from e
+
+        images: list[OSImage] = []
+
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            futures = [
+                executor.submit(self._get_image_info, image) for image in os_images
+            ]
+            for future in concurrent.futures.as_completed(futures):
+                images.append(future.result())
+
+        logger.info(f"Imported images for {self.connection.auth['auth_url']}")
+
+        return images
+
+    def _get_image_info(self, os_image: OSImage) -> Image:
+
+        properties: dict = os_image.properties
+
+        properties.pop("owner_specified.openstack.md5", None)
+        properties.pop("owner_specified.openstack.sha256", None)
+        properties.pop("owner_specified.openstack.object", None)
+        properties.pop("stores", None)
+
+        image = Image(
+            image_id=os_image.id,
+            disk_format=os_image.disk_format,
+            min_disk=os_image.min_disk,
+            min_ram=os_image.min_ram,
+            image_name=os_image.name,
+            owner=os_image.owner,
+            properties=os_image.properties,
+            protected=os_image.is_protected,
+            status=os_image.status,
+            size=os_image.size,
+            virtual_size=os_image.virtual_size,
+            visibility=os_image.visibility,
+            created_at=os_image.created_at,
+            updated_at=os_image.updated_at,
+        )
+
+        return image

osi_dump/importer/instance/openstack_instance_importer.py

@@ -96,6 +96,7 @@ class OpenStackInstanceImporter(InstanceImporter):
             private_v4_ips=private_v4_ips,
             floating_ip=floating_ip,
             status=server.status,
+            hypervisor=server.hypervisor_hostname,
             ram=server.flavor["ram"],
             vcpus=server.flavor["vcpus"],
             created_at=server.created_at,

osi_dump/importer/load_balancer/load_balancer_importer.py

@@ -0,0 +1,9 @@
+from abc import ABC, abstractmethod
+
+from osi_dump.model.load_balancer import LoadBalancer
+
+
+class LoadBalancerImporter(ABC):
+    @abstractmethod
+    def import_load_balancers(self) -> list[LoadBalancer]:
+        pass

osi_dump/importer/load_balancer/openstack_load_balancer_importer.py

@@ -0,0 +1,72 @@
+import logging
+
+import concurrent
+
+from openstack.connection import Connection
+
+from openstack.load_balancer.v2.load_balancer import LoadBalancer as OSLoadBalancer
+
+from osi_dump.importer.load_balancer.load_balancer_importer import (
+    LoadBalancerImporter,
+)
+from osi_dump.model.load_balancer import LoadBalancer
+
+import osi_dump.api.octavia as octavia_api
+
+logger = logging.getLogger(__name__)
+
+
+class OpenStackLoadBalancerImporter(LoadBalancerImporter):
+    def __init__(self, connection: Connection):
+        self.connection = connection
+
+    def import_load_balancers(self) -> list[LoadBalancer]:
+        """Import load_balancers information from Openstack
+
+        Raises:
+            Exception: Raises exception if fetching load_balancer failed
+
+        Returns:
+            list[LoadBalancer]: _description_
+        """
+
+        logger.info(f"Importing load_balancers for {self.connection.auth['auth_url']}")
+
+        try:
+            osload_balancers: list[OSLoadBalancer] = octavia_api.get_load_balancers(
+                connection=self.connection
+            )
+        except Exception as e:
+            raise Exception(
+                f"Can not fetch load_balancers for {self.connection.auth['auth_url']} {e}"
+            ) from e
+
+        load_balancers: list[LoadBalancer] = []
+
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            futures = [
+                executor.submit(self._get_load_balancer_info, load_balancer)
+                for load_balancer in osload_balancers
+            ]
+            for future in concurrent.futures.as_completed(futures):
+                load_balancers.append(future.result())
+
+        logger.info(f"Imported load_balancers for {self.connection.auth['auth_url']}")
+
+        return load_balancers
+
+    def _get_load_balancer_info(self, load_balancer: OSLoadBalancer) -> LoadBalancer:
+
+        amphoraes = octavia_api.get_amphoraes(
+            connection=self.connection, load_balancer_id=load_balancer["id"]
+        )
+
+        load_balancer_ret = LoadBalancer(
+            id=load_balancer["id"],
+            load_balancer_name=load_balancer["name"],
+            status=load_balancer["operating_status"],
+            project_id=load_balancer["project_id"],
+            amphoraes=amphoraes,
+        )
+
+        return load_balancer_ret

osi_dump/importer/project/openstack_project_importer.py

@@ -76,12 +76,12 @@ class OpenStackProjectImporter(ProjectImporter):
             quota_ram=compute_quotas.ram,
             usage_vcpu=compute_quotas.usage["cores"],
             quota_vcpu=compute_quotas.cores,
-            usage_volume=storage_quotas.volumes,
-            quota_volume=storage_quotas.usage["volumes"],
-            usage_snapshot=storage_quotas.snapshots,
-            quota_snapshot=storage_quotas.usage["snapshots"],
-            usage_storage=storage_quotas.gigabytes,
-            quota_storage=storage_quotas.usage["gigabytes"],
+            usage_volume=storage_quotas.usage["volumes"],
+            quota_volume=storage_quotas.volumes,
+            usage_snapshot=storage_quotas.usage["snapshots"],
+            quota_snapshot=storage_quotas.snapshots,
+            usage_storage=storage_quotas.usage["gigabytes"],
+            quota_storage=storage_quotas.gigabytes,
         )
 
         return project_ret

osi_dump/importer/role_assignment/openstack_role_assignment_importer.py

@@ -0,0 +1,100 @@
+import logging
+
+import concurrent
+
+from openstack.connection import Connection
+from openstack.identity.v3.role_assignment import RoleAssignment as OSRoleAssignment
+
+from osi_dump.importer.role_assignment.role_assignment_importer import (
+    RoleAssignmentImporter,
+)
+from osi_dump.model.role_assignment import RoleAssignment
+
+logger = logging.getLogger(__name__)
+
+
+class OpenStackRoleAssignmentImporter(RoleAssignmentImporter):
+    def __init__(self, connection: Connection):
+        self.connection = connection
+
+    def import_role_assignments(self) -> list[RoleAssignment]:
+        """Import role_assignments information from Openstack
+
+        Raises:
+            Exception: Raises exception if fetching role_assignment failed
+
+        Returns:
+            list[RoleAssignment]: _description_
+        """
+
+        logger.info(
+            f"Importing role_assignments for {self.connection.auth['auth_url']}"
+        )
+
+        try:
+            osrole_assignments: list[OSRoleAssignment] = list(
+                self.connection.identity.role_assignments()
+            )
+        except Exception as e:
+            raise Exception(
+                f"Can not fetch role_assignments for {self.connection.auth['auth_url']}"
+            ) from e
+
+        role_assignments: list[RoleAssignment] = []
+
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            futures = [
+                executor.submit(self._get_role_assignment_info, role_assignment)
+                for role_assignment in osrole_assignments
+            ]
+            for future in concurrent.futures.as_completed(futures):
+                role_assignments.append(future.result())
+
+        logger.info(f"Imported role_assignments for {self.connection.auth['auth_url']}")
+
+        return role_assignments
+
+    def _get_role_assignment_info(
+        self, role_assignment: OSRoleAssignment
+    ) -> RoleAssignment:
+
+        user_id = None
+        role_id = None
+
+        try:
+            user_id = role_assignment.user["id"]
+        except Exception as e:
+            logger.warning(f"Can not get user id: {e}")
+
+        try:
+            role_id = role_assignment.role["id"]
+        except Exception as e:
+            logger.warning(f"Can not get role id: {e}")
+
+        user_name = None
+        role_name = None
+
+        try:
+            role_name = self.connection.identity.get_role(
+                role_assignment.role["id"]
+            ).name
+
+        except Exception as e:
+            logger.warning(f"Can not get role name: {e}")
+
+        try:
+            user_name = self.connection.identity.get_user(
+                role_assignment.user["id"]
+            ).name
+        except Exception as e:
+            logger.warning(f"Can not get user name: {e}")
+
+        role_assignment_ret = RoleAssignment(
+            user_id=user_id,
+            user_name=user_name,
+            role_id=role_id,
+            role_name=role_name,
+            scope=role_assignment.scope,
+        )
+
+        return role_assignment_ret

osi_dump/importer/role_assignment/role_assignment_importer.py

@@ -0,0 +1,9 @@
+from abc import ABC, abstractmethod
+
+from osi_dump.model.role_assignment import RoleAssignment
+
+
+class RoleAssignmentImporter(ABC):
+    @abstractmethod
+    def import_role_assignments(self) -> list[RoleAssignment]:
+        pass

osi_dump/importer/router/openstack_router_importer.py

@@ -0,0 +1,86 @@
+import logging
+
+import concurrent
+
+from openstack.connection import Connection
+from openstack.network.v2.router import Router as OSRouter
+
+from osi_dump.importer.router.router_importer import (
+    RouterImporter,
+)
+from osi_dump.model.router import Router
+
+logger = logging.getLogger(__name__)
+
+
+class OpenStackRouterImporter(RouterImporter):
+    def __init__(self, connection: Connection):
+        self.connection = connection
+
+    def import_routers(self) -> list[Router]:
+        """Import routers information from Openstack
+
+        Raises:
+            Exception: Raises exception if fetching router failed
+
+        Returns:
+            list[Router]: _description_
+        """
+
+        logger.info(f"Importing routers for {self.connection.auth['auth_url']}")
+
+        try:
+            osrouters: list[OSRouter] = list(self.connection.network.routers())
+        except Exception as e:
+            raise Exception(
+                f"Can not fetch routers for {self.connection.auth['auth_url']}"
+            ) from e
+
+        routers: list[Router] = []
+
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            futures = [
+                executor.submit(self._get_router_info, router) for router in osrouters
+            ]
+            for future in concurrent.futures.as_completed(futures):
+                routers.append(future.result())
+
+        logger.info(f"Imported routers for {self.connection.auth['auth_url']}")
+
+        return routers
+
+    def _get_router_info(self, router: OSRouter) -> Router:
+        """
+                {"network_id": "49760654-71d8-4967-8fdd-5a35d3ff78ef", "external_fixed_ips": [{"subnet_id":                                   |
+        |                           | "c044a5c0-4b11-4d8d-ae5e-9ff4ce6c1be6", "ip_address": "10.0.2.188"}], "enable_snat": true}
+        """
+
+        external_net_id = None
+
+        try:
+            external_net_id = router.external_gateway_info["network_id"]
+        except Exception as e:
+            logger.warning(f"Could not get external net id for router: {router.id}")
+
+        external_net_ip = None
+
+        try:
+            external_net_ip = router.external_gateway_info["external_fixed_ips"][0][
+                "ip_address"
+            ]
+
+        except Exception as e:
+            logger.warning(f"Could not get external net ip for router {router.id}")
+
+        router_ret = Router(
+            router_id=router.id,
+            name=router.name,
+            external_net_id=external_net_id,
+            external_net_ip=external_net_ip,
+            status=router.status,
+            project_id=router.project_id,
+            created_at=router.created_at,
+            updated_at=router.updated_at,
+        )
+
+        return router_ret

osi_dump/importer/router/router_importer.py

@@ -0,0 +1,9 @@
+from abc import ABC, abstractmethod
+
+from osi_dump.model.router import Router
+
+
+class RouterImporter(ABC):
+    @abstractmethod
+    def import_routers(self) -> list[Router]:
+        pass

osi_dump/model/flavor.py

@@ -0,0 +1,20 @@
+from typing import Optional
+
+from pydantic import BaseModel, ConfigDict, ValidationError
+
+
+class Flavor(BaseModel):
+    model_config = ConfigDict(strict=True)
+
+    flavor_id: str
+
+    flavor_name: str
+
+    properties: Optional[dict]
+
+    ram: int
+    vcpus: int
+    disk: int
+    swap: Optional[int]
+
+    public: bool

osi_dump/model/hypervisor.py

@@ -16,3 +16,5 @@ class Hypervisor(BaseModel):
 
     local_disk_usage: int
     local_disk_size: int
+
+    vm_count: int

osi_dump/model/image.py

@@ -0,0 +1,26 @@
+from typing import Optional
+
+from pydantic import BaseModel, ConfigDict, ValidationError
+
+
+class Image(BaseModel):
+    model_config = ConfigDict(strict=True)
+
+    image_id: str
+
+    disk_format: str
+    min_disk: int
+    min_ram: int
+    image_name: Optional[str]
+    owner: str
+
+    properties: Optional[dict]
+
+    protected: bool
+    status: str
+    size: int
+    virtual_size: Optional[int]
+    visibility: str
+
+    created_at: str
+    updated_at: str

osi_dump/model/instance.py

@@ -16,5 +16,6 @@ class Instance(BaseModel):
     status: str
     ram: int
     vcpus: int
+    hypervisor: Optional[str]
     created_at: str
     updated_at: str

osi_dump/model/load_balancer.py

@@ -0,0 +1,17 @@
+from typing import Optional
+
+from pydantic import BaseModel, ConfigDict, ValidationError
+
+
+class LoadBalancer(BaseModel):
+    model_config = ConfigDict(strict=True)
+
+    id: str
+
+    load_balancer_name: Optional[str]
+
+    status: str
+
+    amphoraes: list[str]
+
+    project_id: Optional[str]

osi_dump/model/role_assignment.py

@@ -0,0 +1,17 @@
+from typing import Optional
+
+from pydantic import BaseModel, ConfigDict, ValidationError
+
+
+class RoleAssignment(BaseModel):
+    model_config = ConfigDict(strict=True)
+
+    user_id: Optional[str]
+
+    user_name: Optional[str]
+
+    role_id: Optional[str]
+
+    role_name: Optional[str]
+
+    scope: Optional[dict]

osi_dump/model/router.py

@@ -0,0 +1,20 @@
+from typing import Optional
+
+from pydantic import BaseModel, ConfigDict, ValidationError
+
+
+class Router(BaseModel):
+    model_config = ConfigDict(strict=True)
+
+    router_id: str
+
+    name: Optional[str]
+
+    external_net_id: Optional[str]
+    external_net_ip: Optional[str]
+
+    status: str
+    project_id: Optional[str]
+
+    created_at: str
+    updated_at: str

osi_dump/util/__init__.py

@@ -4,3 +4,4 @@ from .create_file import create_file
 from .export_data_excel import export_data_excel
 from .excel_sort_sheet import excel_sort_sheet
 from .validate_dir_path import validate_dir_path
+from .panda_excel import expand_list_column

osi_dump/util/panda_excel.py

@@ -0,0 +1,22 @@
+import pandas as pd
+
+from pandas import DataFrame
+
+
+def expand_list_column(df: DataFrame, column: str) -> DataFrame:
+    # Find the maximum length of the list in the column
+    max_len = df[column].apply(len).max()
+
+    # Create a DataFrame from the list column
+    expanded_df = pd.DataFrame(
+        df[column].apply(lambda x: x + [None] * (max_len - len(x))).tolist(),
+        index=df.index,
+    )
+
+    # Rename the columns
+    expanded_df.columns = [f"{column}_{i+1}" for i in range(expanded_df.shape[1])]
+
+    # Drop the original column and join the expanded columns
+    df = df.drop(column, axis=1).join(expanded_df)
+
+    return df

{osi_dump-0.1.2.dist-info → osi_dump-0.1.2.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: osi-dump
-Version: 0.1.2
+Version: 0.1.2.2
 Summary: OpenStack instances information dump tool
 Author: TVKain
 License: Apache-2.0