osi-dump 0.1.3.3.4.3__py3-none-any.whl → 0.1.4__py3-none-any.whl

osi_dump/importer/project/openstack_project_importer.py

@@ -1,141 +1,141 @@
-import logging
-
-import concurrent
-
-from openstack.connection import Connection
-from openstack.identity.v3.project import Project as OSProject
-from openstack.load_balancer.v2.load_balancer import LoadBalancer as OSLoadBalancer
-
-from osi_dump.importer.project.project_importer import ProjectImporter
-from osi_dump.model.project import Project
-import osi_dump.api.octavia as octavia_api
-logger = logging.getLogger(__name__)
-
-
-class OpenStackProjectImporter(ProjectImporter):
-    def __init__(self, connection: Connection):
-        self.connection = connection
-
-    def import_projects(self) -> list[Project]:
-        """Import projects information from Openstack
-
-        Raises:
-            Exception: Raises exception if fetching project failed
-
-        Returns:
-            list[Instance]: _description_
-        """
-
-        logger.info(f"Importing projects for {self.connection.auth['auth_url']}")
-        try:
-            osload_balancers: list[OSLoadBalancer] = octavia_api.get_load_balancers(
-                connection=self.connection
-            )
-        except Exception as e:
-            raise Exception(
-                f"Can not fetch load_balancers for {self.connection.auth['auth_url']} {e}"
-            ) from e
-
-        project_lb_dict = {}
-
-        for osload_balancer in osload_balancers: 
-            if project_lb_dict.get(osload_balancer["project_id"]):
-                project_lb_dict[osload_balancer["project_id"]] += 1 
-            else: 
-                project_lb_dict[osload_balancer["project_id"]] = 1
-
-        try:
-            osprojects: list[OSProject] = list(self.connection.identity.projects())
-        except Exception as e:
-            raise Exception(
-                f"Can not fetch projects for {self.connection.auth['auth_url']}"
-            ) from e
-
-        projects: list[Project] = []
-
-        with concurrent.futures.ThreadPoolExecutor() as executor:
-            futures = [
-                executor.submit(self._get_project_info, project, project_lb_dict)
-                for project in osprojects
-            ]
-            for future in concurrent.futures.as_completed(futures):
-                projects.append(future.result())
-
-        logger.info(f"Imported projects for {self.connection.auth['auth_url']}")
-
-        return projects
-
-    def _get_project_info(self, project: OSProject, project_lb_dict: dict) -> Project:
-        
-        usage_instance=None
-        quota_instance=None
-        usage_ram=None
-        quota_ram=None
-        usage_vcpu=None
-        quota_vcpu=None
-
-        try:
-            compute_quotas = self.connection.compute.get_quota_set(
-                project.id, usage=True
-            )
-
-            usage_instance=compute_quotas.usage["instances"]
-            quota_instance=compute_quotas.instances
-            usage_ram=compute_quotas.usage["ram"]
-            quota_ram=compute_quotas.ram
-            usage_vcpu=compute_quotas.usage["cores"]
-            quota_vcpu=compute_quotas.cores
-        except Exception as e:
-            logger.warning(f"Get compute quotas failed for {project.id} error: {e}")
-
-        usage_volume=None
-        quota_volume=None
-        usage_snapshot=None
-        quota_snapshot=None
-        usage_storage=None
-        quota_storage=None
-        try:
-            storage_quotas = self.connection.block_storage.get_quota_set(
-                project.id, usage=True
-            )
-            usage_volume=storage_quotas.usage["volumes"]
-            quota_volume=storage_quotas.volumes
-            usage_snapshot=storage_quotas.usage["snapshots"]
-            quota_snapshot=storage_quotas.snapshots
-            usage_storage=storage_quotas.usage["gigabytes"]
-            quota_storage=storage_quotas.gigabytes
-        except Exception as e:
-            logger.warning(f"Get storage quotas failed for {project.id} error: {e}")
-
-        domain_name = None
-        try:
-            domain = self.connection.identity.get_domain(project.domain_id)
-            domain_name = domain.name
-        except Exception as e:
-            logger.warning(f"Get domain failed for {project.domain_id} error: {e}")
-
-        lb_count = project_lb_dict.get(project.id)
-
-        project_ret = Project(
-            project_id=project.id,
-            project_name=project.name,
-            domain_id=project.domain_id,
-            domain_name=domain_name,
-            enabled=project.is_enabled,
-            parent_id=project.parent_id,
-            usage_instance=usage_instance, 
-            quota_instance=quota_instance,
-            usage_ram=usage_ram,
-            quota_ram=quota_ram, 
-            usage_vcpu=usage_vcpu,
-            quota_vcpu=quota_vcpu,
-            usage_volume=usage_volume,
-            quota_volume=quota_volume,
-            usage_snapshot=usage_snapshot,
-            quota_snapshot=quota_snapshot,
-            usage_storage=usage_storage,
-            quota_storage=quota_storage,
-            load_balancer_count=lb_count
-        )
-
-        return project_ret
+import logging
+
+import concurrent
+
+from openstack.connection import Connection
+from openstack.identity.v3.project import Project as OSProject
+from openstack.load_balancer.v2.load_balancer import LoadBalancer as OSLoadBalancer
+
+from osi_dump.importer.project.project_importer import ProjectImporter
+from osi_dump.model.project import Project
+import osi_dump.api.octavia as octavia_api
+logger = logging.getLogger(__name__)
+
+
+class OpenStackProjectImporter(ProjectImporter):
+    def __init__(self, connection: Connection):
+        self.connection = connection
+
+    def import_projects(self) -> list[Project]:
+        """Import projects information from Openstack
+
+        Raises:
+            Exception: Raises exception if fetching project failed
+
+        Returns:
+            list[Instance]: _description_
+        """
+
+        logger.info(f"Importing projects for {self.connection.auth['auth_url']}")
+        try:
+            osload_balancers: list[OSLoadBalancer] = octavia_api.get_load_balancers(
+                connection=self.connection
+            )
+        except Exception as e:
+            raise Exception(
+                f"Can not fetch load_balancers for {self.connection.auth['auth_url']} {e}"
+            ) from e
+
+        project_lb_dict = {}
+
+        for osload_balancer in osload_balancers: 
+            if project_lb_dict.get(osload_balancer["project_id"]):
+                project_lb_dict[osload_balancer["project_id"]] += 1 
+            else: 
+                project_lb_dict[osload_balancer["project_id"]] = 1
+
+        try:
+            osprojects: list[OSProject] = list(self.connection.identity.projects())
+        except Exception as e:
+            raise Exception(
+                f"Can not fetch projects for {self.connection.auth['auth_url']}"
+            ) from e
+
+        projects: list[Project] = []
+
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            futures = [
+                executor.submit(self._get_project_info, project, project_lb_dict)
+                for project in osprojects
+            ]
+            for future in concurrent.futures.as_completed(futures):
+                projects.append(future.result())
+
+        logger.info(f"Imported projects for {self.connection.auth['auth_url']}")
+
+        return projects
+
+    def _get_project_info(self, project: OSProject, project_lb_dict: dict) -> Project:
+        
+        usage_instance=None
+        quota_instance=None
+        usage_ram=None
+        quota_ram=None
+        usage_vcpu=None
+        quota_vcpu=None
+
+        try:
+            compute_quotas = self.connection.compute.get_quota_set(
+                project.id, usage=True
+            )
+
+            usage_instance=compute_quotas.usage["instances"]
+            quota_instance=compute_quotas.instances
+            usage_ram=compute_quotas.usage["ram"]
+            quota_ram=compute_quotas.ram
+            usage_vcpu=compute_quotas.usage["cores"]
+            quota_vcpu=compute_quotas.cores
+        except Exception as e:
+            logger.warning(f"Get compute quotas failed for {project.id} error: {e}")
+
+        usage_volume=None
+        quota_volume=None
+        usage_snapshot=None
+        quota_snapshot=None
+        usage_storage=None
+        quota_storage=None
+        try:
+            storage_quotas = self.connection.block_storage.get_quota_set(
+                project.id, usage=True
+            )
+            usage_volume=storage_quotas.usage["volumes"]
+            quota_volume=storage_quotas.volumes
+            usage_snapshot=storage_quotas.usage["snapshots"]
+            quota_snapshot=storage_quotas.snapshots
+            usage_storage=storage_quotas.usage["gigabytes"]
+            quota_storage=storage_quotas.gigabytes
+        except Exception as e:
+            logger.warning(f"Get storage quotas failed for {project.id} error: {e}")
+
+        domain_name = None
+        try:
+            domain = self.connection.identity.get_domain(project.domain_id)
+            domain_name = domain.name
+        except Exception as e:
+            logger.warning(f"Get domain failed for {project.domain_id} error: {e}")
+
+        lb_count = project_lb_dict.get(project.id)
+
+        project_ret = Project(
+            project_id=project.id,
+            project_name=project.name,
+            domain_id=project.domain_id,
+            domain_name=domain_name,
+            enabled=project.is_enabled,
+            parent_id=project.parent_id,
+            usage_instance=usage_instance, 
+            quota_instance=quota_instance,
+            usage_ram=usage_ram,
+            quota_ram=quota_ram, 
+            usage_vcpu=usage_vcpu,
+            quota_vcpu=quota_vcpu,
+            usage_volume=usage_volume,
+            quota_volume=quota_volume,
+            usage_snapshot=usage_snapshot,
+            quota_snapshot=quota_snapshot,
+            usage_storage=usage_storage,
+            quota_storage=quota_storage,
+            load_balancer_count=lb_count
+        )
+
+        return project_ret

osi_dump/importer/project/project_importer.py

@@ -1,9 +1,9 @@
-from abc import ABC, abstractmethod
-
-from osi_dump.model.project import Project
-
-
-class ProjectImporter(ABC):
-    @abstractmethod
-    def import_projects(self) -> list[Project]:
-        pass
+from abc import ABC, abstractmethod
+
+from osi_dump.model.project import Project
+
+
+class ProjectImporter(ABC):
+    @abstractmethod
+    def import_projects(self) -> list[Project]:
+        pass

osi_dump/importer/role_assignment/openstack_role_assignment_importer.py

@@ -1,147 +1,117 @@
-import logging
-
-import concurrent
-
-from openstack.connection import Connection
-from openstack.identity.v3.role_assignment import RoleAssignment as OSRoleAssignment
-from openstack.identity.v3.user import User as OSUser
-
-
-from osi_dump.importer.role_assignment.role_assignment_importer import (
-    RoleAssignmentImporter,
-)
-from osi_dump.model.role_assignment import RoleAssignment
-
-from osi_dump.api.keystone import get_role_assignments, get_users
-
-logger = logging.getLogger(__name__)
-
-
-class OpenStackRoleAssignmentImporter(RoleAssignmentImporter):
-    def __init__(self, connection: Connection):
-        self.connection = connection
-
-        self.users: dict[str, dict] = {}
-        self.roles = {}
-
-    def _get_users(self):
-        os_users = get_users(self.connection)
-
-        for os_user in os_users:
-            self.users[os_user["id"]] = os_user
-
-    def _get_roles(self):
-        os_roles = self.connection.identity.roles()
-
-        for os_role in os_roles:
-            self.roles[os_role.id] = os_role.name
-
-    def import_role_assignments(self) -> list[RoleAssignment]:
-        """Import role_assignments information from Openstack
-
-        Raises:
-            Exception: Raises exception if fetching role_assignment failed
-
-        Returns:
-            list[RoleAssignment]: _description_
-        """
-
-        logger.info(
-            f"Importing role_assignments for {self.connection.auth['auth_url']}"
-        )
-
-        try:
-            self._get_users()
-        except Exception as e:
-            logger.info(f"Getting user list failed {e}")
-
-        try:
-            self._get_roles()
-        except Exception as e:
-            logger.info(f"Getting role list failed {e}")
-
-        try:
-            osrole_assignments: list[OSRoleAssignment] = list(
-                self.connection.identity.role_assignments()
-            )
-
-        except Exception as e:
-            raise Exception(
-                f"Can not fetch role_assignments for {self.connection.auth['auth_url']} {e}"
-            ) from e
-
-        role_assignments: list[RoleAssignment] = []
-
-        with concurrent.futures.ThreadPoolExecutor() as executor:
-            futures = [
-                executor.submit(self._get_role_assignment_info, role_assignment)
-                for role_assignment in osrole_assignments
-            ]
-            for future in concurrent.futures.as_completed(futures):
-                role_assignments.append(future.result())
-
-        logger.info(f"Imported role_assignments for {self.connection.auth['auth_url']}")
-
-        return role_assignments
-
-    def _get_role_assignment_info(
-        self, role_assignment: OSRoleAssignment
-    ) -> RoleAssignment:
-
-        user_id = None
-        role_id = None
-
-        try:
-            user_id: str = role_assignment["user"]["id"]
-        except Exception as e:
-            logger.warning(f"Can not get user id: {e}")
-
-        try:
-            role_id = role_assignment["role"]["id"]
-        except Exception as e:
-            logger.warning(f"Can not get role id: {e}")
-
-        user_name = None
-        role_name = None
-
-        password_expires_at = None
-        options = None
-
-        try:
-            user_name = self.users[user_id]["name"]
-        except Exception as e:
-            logger.warning(f"Can not get user name: {e}")
-
-        try:
-            role_name = self.roles[role_id]
-        except Exception as e:
-            logger.warning(f"Can not get role name: {e}")
-
-        try:
-            password_expires_at = self.users[user_id]["password_expires_at"]
-        except Exception as e: 
-            logger.warning(f"Can not get password expires at: {e}")
-
-
-        try:
-            options = self.users[user_id]["options"]
-        except Exception as e:
-            logger.warning(f"Can not get option")
-
-        try:
-            enabled = self.users[user_id]["enabled"]
-        except Exception as e:
-            logger.warning(f"Can not get enabled")
-
-        role_assignment_ret = RoleAssignment(
-            user_id=user_id,
-            user_name=user_name,
-            role_id=role_id,
-            role_name=role_name,
-            enabled=enabled,
-            scope=role_assignment["scope"],
-            password_expires_at=password_expires_at,
-            options=options
-        )
-
-        return role_assignment_ret
+import logging
+from openstack.connection import Connection
+from openstack.identity.v3.role_assignment import RoleAssignment as OSRoleAssignment
+from typing import Generator
+
+from .role_assignment_importer import RoleAssignmentImporter
+from osi_dump.model.role_assignment import UserRoleAssignment, GroupRoleAssignment, EffectiveUserRole
+from osi_dump.api.keystone import get_users
+
+logger = logging.getLogger(__name__)
+
+class OpenStackRoleAssignmentImporter(RoleAssignmentImporter):
+    def __init__(self, connection: Connection):
+        self.connection = connection
+        self.users = {}
+        self.roles = {}
+        self.groups = {}
+        self.group_members = {}  # group_id -> [user_id_1, user_id_2]
+        self._raw_assignments = None # Cache 
+
+    def _fetch_all_data(self):
+        """pre-cache for metadata."""
+        if self._raw_assignments is not None:
+            return
+
+        logger.info(f"Pre-caching all role assignment metadata for {self.connection.auth['auth_url']}")
+        
+        try:
+            os_users = get_users(self.connection)
+            for os_user in os_users: self.users[os_user["id"]] = os_user
+        except Exception as e:
+            logger.error(f"Could not fetch users: {e}")
+
+        try:
+            os_roles = self.connection.identity.roles()
+            for os_role in os_roles: self.roles[os_role.id] = os_role.name
+        except Exception as e:
+            logger.error(f"Could not fetch roles: {e}")
+            
+        try:
+            os_groups = list(self.connection.identity.groups())
+            for group in os_groups:
+                self.groups[group.id] = group.name
+                try:
+                    members = list(self.connection.identity.group_users(group))
+                    self.group_members[group.id] = [user.id for user in members]
+                except Exception as e:
+                    logger.warning(f"Could not fetch members for group {group.name}: {e}")
+                    self.group_members[group.id] = []
+        except Exception as e:
+            logger.error(f"Could not fetch groups: {e}")
+
+        try:
+            self._raw_assignments = list(self.connection.identity.role_assignments())
+        except Exception as e:
+            logger.error(f"Could not fetch role assignments: {e}")
+            self._raw_assignments = []
+
+    def get_user_roles(self) -> Generator[UserRoleAssignment, None, None]:
+        self._fetch_all_data()
+        for assignment in self._raw_assignments:
+            if assignment.user:
+                user_id = assignment.user.get('id')
+                user_info = self.users.get(user_id, {})
+                role_id = assignment.role.get('id') if assignment.role else None
+                yield UserRoleAssignment(
+                    user_id=user_id,
+                    user_name=user_info.get('name'),
+                    role_id=role_id,
+                    role_name=self.roles.get(role_id),
+                    scope=assignment.scope,
+                    enabled=user_info.get('enabled'),
+                    password_expires_at=user_info.get('password_expires_at'),
+                    options=user_info.get('options')
+                )
+
+    def get_group_roles(self) -> Generator[GroupRoleAssignment, None, None]:
+        self._fetch_all_data()
+        for assignment in self._raw_assignments:
+            if assignment.group:
+                group_id = assignment.group.get('id')
+                role_id = assignment.role.get('id') if assignment.role else None
+                yield GroupRoleAssignment(
+                    group_id=group_id,
+                    group_name=self.groups.get(group_id),
+                    role_id=role_id,
+                    role_name=self.roles.get(role_id),
+                    scope=assignment.scope
+                )
+
+    def calculate_effective_roles(self) -> Generator[EffectiveUserRole, None, None]:
+        # generator
+        # direct role
+        for user_role in self.get_user_roles():
+            yield EffectiveUserRole(
+                user_id=user_role.user_id,
+                user_name=user_role.user_name,
+                role_id=user_role.role_id,
+                role_name=user_role.role_name,
+                scope=user_role.scope,
+                inherited_from_group='[Direct]'
+            )
+        
+        # inherited_from_group
+        for group_role in self.get_group_roles():
+            group_id = group_role.group_id
+            if group_id in self.group_members:
+                for user_id in self.group_members[group_id]:
+                    user_info = self.users.get(user_id, {})
+                    yield EffectiveUserRole(
+                        user_id=user_id,
+                        user_name=user_info.get('name'),
+                        role_id=group_role.role_id,
+                        role_name=group_role.role_name,
+                        scope=group_role.scope,
+                        inherited_from_group=group_role.group_name
+                    )

osi_dump/importer/role_assignment/role_assignment_importer.py

@@ -1,9 +1,16 @@
-from abc import ABC, abstractmethod
-
-from osi_dump.model.role_assignment import RoleAssignment
-
-
-class RoleAssignmentImporter(ABC):
-    @abstractmethod
-    def import_role_assignments(self) -> list[RoleAssignment]:
-        pass
+from abc import ABC, abstractmethod
+from typing import Generator
+from osi_dump.model.role_assignment import UserRoleAssignment, GroupRoleAssignment, EffectiveUserRole
+
+class RoleAssignmentImporter(ABC):
+    @abstractmethod
+    def get_user_roles(self) -> Generator[UserRoleAssignment, None, None]:
+        pass
+
+    @abstractmethod
+    def get_group_roles(self) -> Generator[GroupRoleAssignment, None, None]:
+        pass
+
+    @abstractmethod
+    def calculate_effective_roles(self) -> Generator[EffectiveUserRole, None, None]:
+        pass