cartography 0.104.0rc2__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/keycloak/identityproviders.py

@@ -0,0 +1,94 @@
+import logging
+from typing import Any
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.keycloak.util import get_paginated
+from cartography.models.keycloak.identityprovider import KeycloakIdentityProviderSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    base_url: str,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    identityproviders = get(
+        api_session,
+        base_url,
+        common_job_parameters["REALM"],
+    )
+    idps_transformed = transform(identityproviders)
+    load_identityproviders(
+        neo4j_session,
+        idps_transformed,
+        common_job_parameters["REALM"],
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+    realm: str,
+) -> list[dict[str, Any]]:
+    result: list[dict[str, Any]] = []
+    url = f"{base_url}/admin/realms/{realm}/identity-provider/instances"
+    for idp in get_paginated(api_session, url, params={"briefRepresentation": False}):
+        # Get members
+        members_url = f"{base_url}/admin/realms/{realm}/users"
+        idp["_members"] = list(
+            get_paginated(
+                api_session,
+                members_url,
+                params={"idpAlias": idp["alias"], "briefRepresentation": True},
+            )
+        )
+        result.append(idp)
+    return result
+
+
+def transform(idps: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    for idp in idps:
+        idp["_member_ids"] = [member["id"] for member in idp["_members"]]
+        idp.pop("_members", None)
+    return idps
+
+
+@timeit
+def load_identityproviders(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    realm: str,
+    update_tag: int,
+) -> None:
+    logger.info(
+        "Loading %d Keycloak IdentityProviders (%s) into Neo4j.", len(data), realm
+    )
+    load(
+        neo4j_session,
+        KeycloakIdentityProviderSchema(),
+        data,
+        LASTUPDATED=update_tag,
+        REALM=realm,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(
+        KeycloakIdentityProviderSchema(), common_job_parameters
+    ).run(neo4j_session)

cartography/intel/keycloak/organizations.py

@@ -0,0 +1,163 @@
+import logging
+from typing import Any
+from typing import Tuple
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.keycloak.util import get_paginated
+from cartography.models.keycloak.organization import KeycloakOrganizationSchema
+from cartography.models.keycloak.organizationdomain import (
+    KeycloakOrganizationDomainSchema,
+)
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    base_url: str,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    organizations = get(
+        api_session,
+        base_url,
+        common_job_parameters["REALM"],
+    )
+    transformed_orgs, transformed_domains = transform(organizations)
+    load_organizations(
+        neo4j_session,
+        transformed_orgs,
+        common_job_parameters["REALM"],
+        common_job_parameters["UPDATE_TAG"],
+    )
+    load_org_domains(
+        neo4j_session,
+        transformed_domains,
+        common_job_parameters["REALM"],
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+def transform(
+    organizations: list[dict[str, Any]],
+) -> Tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    transformed_orgs = []
+    transformed_domains = {}
+    for org in organizations:
+        # Transform members to a list of IDs
+        org["_managed_members"] = []
+        org["_unmanaged_members"] = []
+        for member in org.get("_members", []):
+            if member.get("membershipType") == "UNMANAGED":
+                org["_unmanaged_members"].append(member["id"])
+            else:
+                org["_managed_members"].append(member["id"])
+        org.pop("_members", None)
+        # Transform identity providers to a list of IDs
+        org["_idp_ids"] = [
+            idp["internalId"] for idp in org.get("_identity_providers", [])
+        ]
+        org.pop("_identity_providers", None)
+        # Extract domains
+        domains = org.get("domains", [])
+        for domain in domains:
+            domain_id = f"{org['id']}-{domain['name']}"
+            transformed_domains[domain_id] = {
+                "id": domain_id,
+                "verified": domain.get("verified", False),
+                "name": domain["name"],
+                "organization_id": org["id"],
+            }
+        org.pop("domains", None)
+        transformed_orgs.append(org)
+    return transformed_orgs, list(transformed_domains.values())
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+    realm: str,
+) -> list[dict[str, Any]]:
+    result: list[dict[str, Any]] = []
+    url = f"{base_url}/admin/realms/{realm}/organizations"
+    for org in get_paginated(api_session, url):
+        # Get members
+        members_url = (
+            f"{base_url}/admin/realms/{realm}/organizations/{org['id']}/members"
+        )
+        org["_members"] = list(
+            get_paginated(
+                api_session,
+                members_url,
+                params={"briefRepresentation": True},
+            )
+        )
+        # Get Identity Providers
+        idp_url = f"{base_url}/admin/realms/{realm}/organizations/{org['id']}/identity-providers"
+        org["_identity_providers"] = list(
+            get_paginated(
+                api_session,
+                idp_url,
+                params={"briefRepresentation": True},
+            )
+        )
+        result.append(org)
+    return result
+
+
+@timeit
+def load_organizations(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    realm: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Keycloak Organizations (%s) into Neo4j.", len(data), realm)
+    load(
+        neo4j_session,
+        KeycloakOrganizationSchema(),
+        data,
+        LASTUPDATED=update_tag,
+        REALM=realm,
+    )
+
+
+@timeit
+def load_org_domains(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    realm: str,
+    update_tag: int,
+) -> None:
+    logger.info(
+        "Loading %d Keycloak Organization Domains (%s) into Neo4j.", len(data), realm
+    )
+    load(
+        neo4j_session,
+        KeycloakOrganizationDomainSchema(),
+        data,
+        LASTUPDATED=update_tag,
+        REALM=realm,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(
+        KeycloakOrganizationDomainSchema(), common_job_parameters
+    ).run(neo4j_session)
+    GraphJob.from_node_schema(KeycloakOrganizationSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/keycloak/realms.py

@@ -0,0 +1,61 @@
+import logging
+from typing import Any
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.keycloak.realm import KeycloakRealmSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    base_url: str,
+    common_job_parameters: dict[str, Any],
+) -> list[dict]:
+    realms = get(api_session, base_url)
+    load_realms(neo4j_session, realms, common_job_parameters["UPDATE_TAG"])
+    cleanup(neo4j_session, common_job_parameters)
+    return realms
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+) -> list[dict[str, Any]]:
+    req = api_session.get(f"{base_url}/admin/realms", timeout=_TIMEOUT)
+    req.raise_for_status()
+    return req.json()
+
+
+@timeit
+def load_realms(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Keycloak Realms into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        KeycloakRealmSchema(),
+        data,
+        LASTUPDATED=update_tag,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(KeycloakRealmSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/keycloak/roles.py

@@ -0,0 +1,202 @@
+import logging
+from typing import Any
+from urllib.parse import quote
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.keycloak.util import get_paginated
+from cartography.models.keycloak.role import KeycloakRoleSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    base_url: str,
+    common_job_parameters: dict[str, Any],
+    client_ids: list[str],
+    scope_ids: list[str],
+) -> None:
+    roles = get(api_session, base_url, common_job_parameters["REALM"], client_ids)
+    scope_role_mapping = get_mapping(
+        api_session,
+        base_url,
+        common_job_parameters["REALM"],
+        scope_ids,
+    )
+    transformed_roles = transform(roles, scope_role_mapping)
+    load_roles(
+        neo4j_session,
+        transformed_roles,
+        common_job_parameters["REALM"],
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    api_session: requests.Session, base_url: str, realm: str, client_ids: list[str]
+) -> list[dict[str, Any]]:
+    roles_by_id: dict[str, dict[str, Any]] = {}
+
+    # Get roles at the REALM level
+    url = f"{base_url}/admin/realms/{realm}/roles"
+    for role in get_paginated(api_session, url, params={"briefRepresentation": False}):
+        if role.get("composite", False):
+            # If the role is composite, we need to get its composites
+            composite_roles = get_paginated(
+                api_session,
+                f"{base_url}/admin/realms/{realm}/roles-by-id/{role['id']}/composites",
+            )
+            role["_composite_roles"] = [
+                composite_role["id"] for composite_role in composite_roles
+            ]
+        # Get the direct members
+        direct_members = get_paginated(
+            api_session,
+            f"{base_url}/admin/realms/{realm}/roles/{quote(role['name'])}/users",
+        )
+        role["_direct_members"] = [member["id"] for member in direct_members]
+        roles_by_id[role["id"]] = role
+
+    # Get roles for each client
+    for client_id in client_ids:
+        url = f"{base_url}/admin/realms/{realm}/clients/{client_id}/roles"
+        for role in get_paginated(
+            api_session, url, params={"briefRepresentation": False}
+        ):
+            # If the role is composite, we need to get its composites
+            if role.get("composite", False):
+                composite_roles = get_paginated(
+                    api_session,
+                    f"{base_url}/admin/realms/{realm}/roles-by-id/{role['id']}/composites",
+                )
+                role["_composite_roles"] = [
+                    composite_role["id"] for composite_role in composite_roles
+                ]
+            # Get the direct members
+            direct_members = get_paginated(
+                api_session,
+                f"{base_url}/admin/realms/{realm}/clients/{client_id}/roles/{quote(role['name'])}/users",
+            )
+            role["_direct_members"] = [member["id"] for member in direct_members]
+            roles_by_id[role["id"]] = role
+    return list(roles_by_id.values())
+
+
+@timeit
+def get_mapping(
+    api_session: requests.Session,
+    base_url: str,
+    realm: str,
+    scope_ids: list[str],
+) -> dict[str, Any]:
+    result: dict[str, Any] = {}
+    for scope_id in scope_ids:
+        mappings_url = (
+            f"{base_url}/admin/realms/{realm}/client-scopes/{scope_id}/scope-mappings"
+        )
+        req = api_session.get(
+            mappings_url,
+            timeout=_TIMEOUT,
+        )
+        req.raise_for_status()
+        result[scope_id] = req.json()
+    return result
+
+
+def transform(
+    roles: list[dict[str, Any]], scope_role_mapping: dict[str, Any]
+) -> list[dict[str, Any]]:
+    transformed_roles = []
+
+    # Transform the mapping of scopes to roles
+    scopes_by_roles: dict[str, list[str]] = {}
+    for scope_id, mapping in scope_role_mapping.items():
+        for client_details in mapping.get("clientMappings", {}).values():
+            for client_mapping in client_details.get("mappings", []):
+                scopes_by_roles.setdefault(client_mapping["id"], []).append(scope_id)
+        for realm_mapping in mapping.get("realmMappings", []):
+            scopes_by_roles.setdefault(realm_mapping["id"], []).append(scope_id)
+
+    for role in roles:
+        role["_scope_ids"] = scopes_by_roles.get(role["id"], None)
+        transformed_roles.append(role)
+
+    return _order_dicts_dfs(
+        transformed_roles, children_key="_composite_roles", ignore_unknown_children=True
+    )
+
+
+@timeit
+def load_roles(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    realm: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Keycloak Roles (%s) into Neo4j.", len(data), realm)
+    load(
+        neo4j_session,
+        KeycloakRoleSchema(),
+        data,
+        LASTUPDATED=update_tag,
+        REALM=realm,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(KeycloakRoleSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+def _order_dicts_dfs(
+    nodes: list[dict[str, Any]],
+    children_key: str = "children",
+    ignore_unknown_children: bool = False,
+) -> list[dict[str, Any]]:
+    by_id = {n["id"]: n for n in nodes}
+
+    WHITE, GRAY, BLACK = 0, 1, 2  # unvisited, in stack, done
+    state: dict[str, int] = {}
+    ordered: list[dict[str, Any]] = []
+
+    def visit(node_id: str, path: list[str]) -> None:
+        s = state.get(node_id, WHITE)
+        if s == GRAY:
+            cycle = " -> ".join(path + [node_id])
+            raise ValueError(f"Cycle detected: {cycle}")
+        if s == BLACK:
+            return
+
+        state[node_id] = GRAY
+        node = by_id[node_id]
+
+        for child_id in node.get(children_key, []):
+            if child_id not in by_id:
+                if ignore_unknown_children:
+                    continue
+                raise KeyError(f"Unknown child id: {child_id}")
+            visit(child_id, path + [node_id])
+
+        state[node_id] = BLACK
+        ordered.append(node)  # post-order: children before parent
+
+    for nid in by_id:
+        if state.get(nid, WHITE) == WHITE:
+            visit(nid, [])
+
+    return ordered

cartography/intel/keycloak/scopes.py

@@ -0,0 +1,73 @@
+import logging
+from typing import Any
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.keycloak.util import get_paginated
+from cartography.models.keycloak.scope import KeycloakScopeSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    base_url: str,
+    common_job_parameters: dict[str, Any],
+) -> list[dict[str, Any]]:
+    scopes = get(
+        api_session,
+        base_url,
+        common_job_parameters["REALM"],
+    )
+    load_scopes(
+        neo4j_session,
+        scopes,
+        common_job_parameters["REALM"],
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+    return scopes
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+    realm: str,
+) -> list[dict[str, Any]]:
+    url = f"{base_url}/admin/realms/{realm}/client-scopes"
+    return list(get_paginated(api_session, url, params={"briefRepresentation": False}))
+
+
+@timeit
+def load_scopes(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    realm: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Keycloak Scopes (%s) into Neo4j.", len(data), realm)
+    load(
+        neo4j_session,
+        KeycloakScopeSchema(),
+        data,
+        LASTUPDATED=update_tag,
+        REALM=realm,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(KeycloakScopeSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/keycloak/users.py

@@ -0,0 +1,70 @@
+import logging
+from typing import Any
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.keycloak.util import get_paginated
+from cartography.models.keycloak.user import KeycloakUserSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    base_url: str,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    users = get(
+        api_session,
+        base_url,
+        common_job_parameters["REALM"],
+    )
+    load_users(
+        neo4j_session,
+        users,
+        common_job_parameters["REALM"],
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+    realm: str,
+) -> list[dict[str, Any]]:
+    url = f"{base_url}/admin/realms/{realm}/users"
+    return list(get_paginated(api_session, url))
+
+
+@timeit
+def load_users(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    realm: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Keycloak Users (%s) into Neo4j.", len(data), realm)
+    load(
+        neo4j_session,
+        KeycloakUserSchema(),
+        data,
+        LASTUPDATED=update_tag,
+        REALM=realm,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(KeycloakUserSchema(), common_job_parameters).run(
+        neo4j_session
+    )