osi-dump 0.1.4__py3-none-any.whl → 0.1.5__py3-none-any.whl

osi_dump/batch_handler/role_assignment_batch_handler.py

@@ -31,7 +31,6 @@ class RoleAssignmentBatchHandler:
     def process(self):
         for importer, exporter in self._importer_exporter_list:
             try:
-                # importer to exporter
                 exporter.export_role_assignments(importer=importer)
             except Exception as e:
                 logger.warning(e)

osi_dump/exporter/role_assignment/excel_role_assignment_exporter.py

@@ -12,22 +12,14 @@ class ExcelRoleAssignmentExporter(RoleAssignmentExporter):
         self.output_file = output_file
 
     def export_role_assignments(self, importer: RoleAssignmentImporter):
-        # Sheet 1: User Effective Roles
-        df_effective = pd.json_normalize(
-            (role.model_dump() for role in importer.calculate_effective_roles())
-        )
-        if not df_effective.empty:
-            df_effective.sort_values(by=['user_name', 'role_name'], inplace=True)
-            self._export_to_sheet(df_effective, f"{self.sheet_name_prefix}-effective")
-
-        # Sheet 2: Raw User Roles
+        # Sheet 1: Raw User Role Assignments
         df_user = pd.json_normalize(
             (role.model_dump() for role in importer.get_user_roles())
         )
         if not df_user.empty:
             self._export_to_sheet(df_user, f"{self.sheet_name_prefix}-user")
             
-        # Sheet 3: Raw Group Roles
+        # Sheet 2: Raw Group Role Assignments
         df_group = pd.json_normalize(
             (role.model_dump() for role in importer.get_group_roles())
         )

osi_dump/importer/instance/openstack_instance_importer.py

@@ -1,10 +1,7 @@
 import logging
-
-import concurrent
-
+from typing import Generator
 from openstack.connection import Connection
 from openstack.compute.v2.server import Server
-
 from openstack.compute.v2.flavor import Flavor as OSFlavor
 
 from osi_dump.importer.instance.instance_importer import InstanceImporter
@@ -17,63 +14,45 @@ class OpenStackInstanceImporter(InstanceImporter):
     def __init__(self, connection: Connection):
         self.connection = connection
 
-    def import_instances(self) -> list[Instance]:
-        """Import instances information from Openstack
-
-        Raises:
-            Exception: Raises exception if fetching server failed
-
-        Returns:
-            list[Instance]: _description_
-        """
+    def import_instances(self) -> Generator[Instance, None, None]:
+        """Import instances information from Openstack as a generator."""
 
         logger.info(f"Importing instances for {self.connection.auth['auth_url']}")
-
         try:
-            servers: list[Server] = list(
-                self.connection.compute.servers(details=True, all_projects=True)
-            )
-        except Exception as e:
-            raise Exception(
-                f"Can not fetch instances for {self.connection.auth['auth_url']}: {e}"
-            ) from e
-
-        instances: list[Instance] = []
-
-        with concurrent.futures.ThreadPoolExecutor() as executor:
-            futures = [
-                executor.submit(self._get_instance_info, server) for server in servers
-            ]
-            for future in concurrent.futures.as_completed(futures):
-                instances.append(future.result())
+            server_iterator = self.connection.compute.servers(details=True, all_projects=True)
+            
+            for server in server_iterator:
+                yield self._get_instance_info(server)
 
-        logger.info(f"Imported instances for {self.connection.auth['auth_url']}")
-
-        return instances
+        except Exception as e:
+            logger.error(f"Cannot fetch instances for {self.connection.auth['auth_url']}: {e}")
+            return 
+        
+        logger.info(f"Finished importing instances for {self.connection.auth['auth_url']}")
 
     def _get_instance_info(self, server: Server) -> Instance:
-
         project_name = None
         project_id = None
+        project = None 
         try:
             project = self.connection.identity.get_project(server.project_id)
             project_name = project.name
             project_id = project.id
         except Exception as e:
-            logger.warn(
+            logger.warning(
                 f"Unable to obtain project name for instance: {server.name}: {e}"
             )
 
         domain_name = None
-        try:
-            domain = self.connection.identity.get_domain(project.domain_id)
-            domain_name = domain.name
-        except Exception as e:
-            logger.warning(
-                f"Unable to obtain domain name for instance {server.name}: {e}"
-            )
+        if project: 
+            try:
+                domain = self.connection.identity.get_domain(project.domain_id)
+                domain_name = domain.name
+            except Exception as e:
+                logger.warning(
+                    f"Unable to obtain domain name for instance {server.name}: {e}"
+                )
 
-        # Lấy thông tin IPv4 private
         private_v4_ips = []
         floating_ip = None
 
@@ -91,28 +70,25 @@ class OpenStackInstanceImporter(InstanceImporter):
 
         vgpus = None
         vgpu_type = None
-
         vgpu_metadata_property = "pci_passthrough:alias"
 
         try:
             flavor: OSFlavor = self.connection.get_flavor(
                 name_or_id=server.flavor["id"]
             )
-
-            vgpu_prop: str = flavor.extra_specs[vgpu_metadata_property]
-
-            vgpu_props = vgpu_prop.split(":")
-
-            vgpu_type = vgpu_props[0]
-            vgpus = int(vgpu_props[1])
-
-        except Exception as e:
+            if flavor and flavor.extra_specs:
+                vgpu_prop: str = flavor.extra_specs.get(vgpu_metadata_property)
+                if vgpu_prop:
+                    vgpu_props = vgpu_prop.split(":")
+                    vgpu_type = vgpu_props[0]
+                    vgpus = int(vgpu_props[1])
+        except Exception:
             pass
         
-        image_id = server.image["id"]
-        flavor_id = server.flavor["id"]
+        image_id = server.image.get("id") if server.image else None
+        flavor_id = server.flavor.get("id") if server.flavor else None
         
-        instance = Instance(
+        return Instance(
             instance_id=server.id,
             instance_name=server.name,
             project_id=project_id,
@@ -131,6 +107,4 @@ class OpenStackInstanceImporter(InstanceImporter):
             vgpu_type=vgpu_type,
             image_id=image_id, 
             flavor_id=flavor_id
-        )
-
-        return instance
+        )

osi_dump/importer/role_assignment/openstack_role_assignment_importer.py

@@ -4,7 +4,7 @@ from openstack.identity.v3.role_assignment import RoleAssignment as OSRoleAssign
 from typing import Generator
 
 from .role_assignment_importer import RoleAssignmentImporter
-from osi_dump.model.role_assignment import UserRoleAssignment, GroupRoleAssignment, EffectiveUserRole
+from osi_dump.model.role_assignment import UserRoleAssignment, GroupRoleAssignment
 from osi_dump.api.keystone import get_users
 
 logger = logging.getLogger(__name__)
@@ -15,15 +15,14 @@ class OpenStackRoleAssignmentImporter(RoleAssignmentImporter):
         self.users = {}
         self.roles = {}
         self.groups = {}
-        self.group_members = {}  # group_id -> [user_id_1, user_id_2]
-        self._raw_assignments = None # Cache 
+        self._raw_assignments = None 
 
-    def _fetch_all_data(self):
-        """pre-cache for metadata."""
+    def _fetch_metadata(self):
+        """cache metadata"""
         if self._raw_assignments is not None:
-            return
+            return 
 
-        logger.info(f"Pre-caching all role assignment metadata for {self.connection.auth['auth_url']}")
+        logger.info(f"Pre-caching metadata for role assignments on {self.connection.auth['auth_url']}")
         
         try:
             os_users = get_users(self.connection)
@@ -39,14 +38,7 @@ class OpenStackRoleAssignmentImporter(RoleAssignmentImporter):
             
         try:
             os_groups = list(self.connection.identity.groups())
-            for group in os_groups:
-                self.groups[group.id] = group.name
-                try:
-                    members = list(self.connection.identity.group_users(group))
-                    self.group_members[group.id] = [user.id for user in members]
-                except Exception as e:
-                    logger.warning(f"Could not fetch members for group {group.name}: {e}")
-                    self.group_members[group.id] = []
+            for group in os_groups: self.groups[group.id] = group.name
         except Exception as e:
             logger.error(f"Could not fetch groups: {e}")
 
@@ -57,7 +49,7 @@ class OpenStackRoleAssignmentImporter(RoleAssignmentImporter):
             self._raw_assignments = []
 
     def get_user_roles(self) -> Generator[UserRoleAssignment, None, None]:
-        self._fetch_all_data()
+        self._fetch_metadata()
         for assignment in self._raw_assignments:
             if assignment.user:
                 user_id = assignment.user.get('id')
@@ -75,7 +67,7 @@ class OpenStackRoleAssignmentImporter(RoleAssignmentImporter):
                 )
 
     def get_group_roles(self) -> Generator[GroupRoleAssignment, None, None]:
-        self._fetch_all_data()
+        self._fetch_metadata()
         for assignment in self._raw_assignments:
             if assignment.group:
                 group_id = assignment.group.get('id')
@@ -86,32 +78,4 @@ class OpenStackRoleAssignmentImporter(RoleAssignmentImporter):
                     role_id=role_id,
                     role_name=self.roles.get(role_id),
                     scope=assignment.scope
-                )
-
-    def calculate_effective_roles(self) -> Generator[EffectiveUserRole, None, None]:
-        # generator
-        # direct role
-        for user_role in self.get_user_roles():
-            yield EffectiveUserRole(
-                user_id=user_role.user_id,
-                user_name=user_role.user_name,
-                role_id=user_role.role_id,
-                role_name=user_role.role_name,
-                scope=user_role.scope,
-                inherited_from_group='[Direct]'
-            )
-        
-        # inherited_from_group
-        for group_role in self.get_group_roles():
-            group_id = group_role.group_id
-            if group_id in self.group_members:
-                for user_id in self.group_members[group_id]:
-                    user_info = self.users.get(user_id, {})
-                    yield EffectiveUserRole(
-                        user_id=user_id,
-                        user_name=user_info.get('name'),
-                        role_id=group_role.role_id,
-                        role_name=group_role.role_name,
-                        scope=group_role.scope,
-                        inherited_from_group=group_role.group_name
-                    )
+                )

osi_dump/importer/role_assignment/role_assignment_importer.py

@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
 from typing import Generator
-from osi_dump.model.role_assignment import UserRoleAssignment, GroupRoleAssignment, EffectiveUserRole
+from osi_dump.model.role_assignment import UserRoleAssignment, GroupRoleAssignment
 
 class RoleAssignmentImporter(ABC):
     @abstractmethod
@@ -10,7 +10,7 @@ class RoleAssignmentImporter(ABC):
     @abstractmethod
     def get_group_roles(self) -> Generator[GroupRoleAssignment, None, None]:
         pass
-
-    @abstractmethod
-    def calculate_effective_roles(self) -> Generator[EffectiveUserRole, None, None]:
-        pass
+    
+#    @abstractmethod
+#    def calculate_effective_roles(self) -> Generator[EffectiveUserRole, None, None]:
+#        pass

osi_dump/model/role_assignment.py

@@ -20,11 +20,11 @@ class GroupRoleAssignment(BaseModel):
     role_name: Optional[str]
     scope: dict
 
-class EffectiveUserRole(BaseModel):
-    model_config = ConfigDict(strict=True)
-    user_id: str
-    user_name: Optional[str]
-    role_id: str
-    role_name: Optional[str]
-    scope: dict
-    inherited_from_group: Optional[str]
+#class EffectiveUserRole(BaseModel):
+#    model_config = ConfigDict(strict=True)
+#    user_id: str
+#    user_name: Optional[str]
+#    role_id: str
+#    role_name: Optional[str]
+#    scope: dict
+#    inherited_from_group: Optional[str]

{osi_dump-0.1.4.dist-info → osi_dump-0.1.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: osi-dump
-Version: 0.1.4
+Version: 0.1.5
 Summary: OpenStack information dump tool
 Author: TVKain
 License: Apache-2.0

{osi_dump-0.1.4.dist-info → osi_dump-0.1.5.dist-info}/RECORD

@@ -16,7 +16,7 @@ osi_dump/batch_handler/instance_batch_handler.py,sha256=9Q6T7qLGgxVWGMWX9Re7rW0U
 osi_dump/batch_handler/load_balancer_batch_handler.py,sha256=57M8umBGPGkzJfES8Ge7NyW2PPkURTIH-jo5TfNKvhE,1855
 osi_dump/batch_handler/network_batch_handler.py,sha256=k-WLnz5UCiX70fGjLywRV_ufEMySSP8UZbdlI0W1DSM,1665
 osi_dump/batch_handler/project_batch_handler.py,sha256=8tpF3_yJMXXvOogX2iN1C4xx5_IKt_xIP71ZFwOzNPI,1677
-osi_dump/batch_handler/role_assignment_batch_handler.py,sha256=LBaPY7rD34sJSXznCx8_R4W4VZMOP3x45oHZrsKjYFM,1787
+osi_dump/batch_handler/role_assignment_batch_handler.py,sha256=Ms1BkmoBqAeVlLOlb6lR-gj9rymI-xSdXIRiGSxK5T0,1747
 osi_dump/batch_handler/router_batch_handler.py,sha256=cjccPBF7nBzLMxaMgPYa7yNU6YLWz6Wcsbur7o9Jwa4,1636
 osi_dump/batch_handler/security_group_batch_handler.py,sha256=iI5s-Lup9masy1s0TNtdKTCYpbQ1Hznwqc3edfRsPyo,1747
 osi_dump/batch_handler/volume_batch_handler.py,sha256=Lqd5CHsdRZzr7dxCy0jiXKatTHFdqU4H7AdO0OFHiYU,1614
@@ -49,7 +49,7 @@ osi_dump/exporter/project/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJW
 osi_dump/exporter/project/excel_project_exporter.py,sha256=mr0QKYxkTt8ilFLyIjBZ9SFIFBOKV5hEaq5D0enam6A,946
 osi_dump/exporter/project/project_exporter.py,sha256=iD0TfkhNCGzu6GXDoK5amabPWNwbio8n16I68ezNWfc,165
 osi_dump/exporter/role_assignment/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-osi_dump/exporter/role_assignment/excel_role_assignment_exporter.py,sha256=XPYcINXWZ1SLiYmo2DG3DSMd286rkxNigWvddHawNsc,1888
+osi_dump/exporter/role_assignment/excel_role_assignment_exporter.py,sha256=nj4wCKHsQ39FtVLik93vs47f4YN78xPSTKGICJimp2Y,1524
 osi_dump/exporter/role_assignment/role_assignment_exporter.py,sha256=7PJuwTTkSolSWalbKY5iydniPGPUS5eoB3kf1mMFlNQ,277
 osi_dump/exporter/router/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 osi_dump/exporter/router/excel_router_exporter.py,sha256=rckGg5SMCK6Y1gbXjasWpkOnkBEWSj6c1jrtt0Pd-wU,930
@@ -76,7 +76,7 @@ osi_dump/importer/image/image_importer.py,sha256=-S4zjWvnEONK_G_D8cJaK7txDgICgmd
 osi_dump/importer/image/openstack_image_importer.py,sha256=jf0bE6SNV8vMInGsrMNWrtAt1U7cJqbRLMEPNpGQLGg,2666
 osi_dump/importer/instance/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 osi_dump/importer/instance/instance_importer.py,sha256=PaTGNBzMtIht624LGTumlXEZVdcLS2WkXvzeAtDNAHA,246
-osi_dump/importer/instance/openstack_instance_importer.py,sha256=PWckeP7o1QWdgGvfJ4sn2E7seEtK-J-wTf68gwv_f4Q,4385
+osi_dump/importer/instance/openstack_instance_importer.py,sha256=aQIbIL_ftgW-sMYXBK65ZQ1cFsoFMN3ICX9m67pAo24,4100
 osi_dump/importer/load_balancer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 osi_dump/importer/load_balancer/load_balancer_importer.py,sha256=8QPbe0DmaLEOcRcr82IcCuB1gRc0ZG-vIuSXiO_QfhQ,227
 osi_dump/importer/load_balancer/openstack_load_balancer_importer.py,sha256=U1DyAc_ca0fXR1n27sMk14R_X_vI2rQESOGBEpdlR4I,4146
@@ -87,8 +87,8 @@ osi_dump/importer/project/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJW
 osi_dump/importer/project/openstack_project_importer.py,sha256=wlPPGEi1uOBDCU0eoxsof6KjIs2lVy3laXiRvt53QyQ,5105
 osi_dump/importer/project/project_importer.py,sha256=DYQFTIPsROt0BYgPWI8NNRwyvnjmaTNThXoCB415xe8,200
 osi_dump/importer/role_assignment/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-osi_dump/importer/role_assignment/openstack_role_assignment_importer.py,sha256=uIdnvebqXcH8E_039ltQ-7LpCcqWAs-7dgZVLf0eJdw,5094
-osi_dump/importer/role_assignment/role_assignment_importer.py,sha256=5KerMFUQ8cXHXnn1RQORydNihL_21vkn8WFTpiYXtO0,555
+osi_dump/importer/role_assignment/openstack_role_assignment_importer.py,sha256=ezUFhMIMNvRH-4Wbz8PtBkmmA5AqEvDO-9hm01mlQBo,3388
+osi_dump/importer/role_assignment/role_assignment_importer.py,sha256=fNHhTufpopzUo-0j3HlHVa2pQSCxAxcS68Z16P7TW9w,543
 osi_dump/importer/router/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 osi_dump/importer/router/openstack_router_importer.py,sha256=cV-R4qUFUYlAIVLFxxozlD6howOfjZ-O4vgU4mL5UhE,2868
 osi_dump/importer/router/router_importer.py,sha256=4tEbkMPErSrFiW648fSRwnd80DtSKbz70et-gzzRJMA,195
@@ -108,7 +108,7 @@ osi_dump/model/instance.py,sha256=UjJagGoD86sSEkAC4vS0j4VFoHTGZvns-TV45KynOtU,66
 osi_dump/model/load_balancer.py,sha256=uPYqzoh2vbPeAhKPHdM2TGhh5-ZpGC0u2fhX6IZxdKk,597
 osi_dump/model/network.py,sha256=HJmRIrVPCu4FLEPVY3Q0W05CE42qFhiEpMId2gclSRs,565
 osi_dump/model/project.py,sha256=nTVswIdfhLH-vvtvrcHvxc31Wol_jPCKopGVMhEN2XE,782
-osi_dump/model/role_assignment.py,sha256=X_-GeBLwTlwjzOlLKIMPrazCQBsnBLuUDGocaCOmH1g,803
+osi_dump/model/role_assignment.py,sha256=tv9h4HqNrXI-WV8deBm8d-onpy6-wJtHy6V6Hu7FmTY,811
 osi_dump/model/router.py,sha256=CORkRw5AuU_A8eGY1t2FJSFX0l0nExzjZTGl3I0hGLs,440
 osi_dump/model/security_group.py,sha256=MJjmgSM8WOuGW2h4PxMv6PwlwLrZzRiToR-5JFUDbBc,642
 osi_dump/model/volume.py,sha256=cHbIVmG1Cs08AkQGE167Ih3hYCIWRjdKHSwD3SDb4js,448
@@ -123,8 +123,8 @@ osi_dump/util/extract_hostname.py,sha256=NdpF2tYt-Gl1Wx4nVZA0FkBzh-IbPfid_dLGQrA
 osi_dump/util/openstack_util.py,sha256=m8Xcp2KNNf3jzo-J7BYj_SdnoNymRN-1DJeQcOwygDg,1347
 osi_dump/util/panda_excel.py,sha256=db53jqLsBhSE1oM14GbeKJZpF7RAUA7bHBjDh9Be8H4,760
 osi_dump/util/validate_dir_path.py,sha256=JHjZBsQIT5ubw8rnsHbw2hkKDBFzYMjmjQ_oJT8EtJc,527
-osi_dump-0.1.4.dist-info/METADATA,sha256=gLvhzT2bIRdlKkndUGA3iUcQ1TT-laKX3HO1hXPXRaQ,719
-osi_dump-0.1.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-osi_dump-0.1.4.dist-info/entry_points.txt,sha256=ozm5sIBtXzLv6_FiUe26v1BgA3_xUReGLPhKQKZ56wQ,46
-osi_dump-0.1.4.dist-info/top_level.txt,sha256=OtAAwmJfcoPvlw_Cemo_H1aXIGV_7w0O2941KQt6faQ,9
-osi_dump-0.1.4.dist-info/RECORD,,
+osi_dump-0.1.5.dist-info/METADATA,sha256=K1Em-pKBquQWRPhuIMAqMfuqJJZwzkJrqKYFFjA9PVc,719
+osi_dump-0.1.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+osi_dump-0.1.5.dist-info/entry_points.txt,sha256=ozm5sIBtXzLv6_FiUe26v1BgA3_xUReGLPhKQKZ56wQ,46
+osi_dump-0.1.5.dist-info/top_level.txt,sha256=OtAAwmJfcoPvlw_Cemo_H1aXIGV_7w0O2941KQt6faQ,9
+osi_dump-0.1.5.dist-info/RECORD,,