cartography 0.93.0rc1__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/keycloak/clients.py

@@ -0,0 +1,187 @@
+import logging
+from typing import Any
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.client.core.tx import load_matchlinks
+from cartography.graph.job import GraphJob
+from cartography.intel.keycloak.util import get_paginated
+from cartography.models.keycloak.client import KeycloakClientSchema
+from cartography.models.keycloak.client import KeycloakClientToFlowMatchLink
+from cartography.models.keycloak.user import KeycloakUserSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    base_url: str,
+    common_job_parameters: dict[str, Any],
+    realm_default_flows: dict[str, Any],
+) -> list[dict]:
+    clients = get(
+        api_session,
+        base_url,
+        common_job_parameters["REALM"],
+    )
+    transformed_clients, service_accounts, flows_binding = transform(
+        clients, realm_default_flows
+    )
+    load_service_accounts(
+        neo4j_session,
+        service_accounts,
+        common_job_parameters["REALM"],
+        common_job_parameters["UPDATE_TAG"],
+    )
+    load_clients(
+        neo4j_session,
+        transformed_clients,
+        common_job_parameters["REALM"],
+        common_job_parameters["UPDATE_TAG"],
+    )
+    load_flow_bindings(
+        neo4j_session,
+        flows_binding,
+        common_job_parameters["REALM_ID"],
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+    return clients
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+    realm: str,
+) -> list[dict[str, Any]]:
+    result: list[dict[str, Any]] = []
+    url = f"{base_url}/admin/realms/{realm}/clients"
+    for client in get_paginated(
+        api_session, url, params={"briefRepresentation": False}
+    ):
+        # Check if the client has a service account user
+        if "service_account" in client.get("defaultClientScopes", []):
+            # Get service account user for each client
+            service_account_url = f"{base_url}/admin/realms/{realm}/clients/{client['id']}/service-account-user"
+            sa_req = api_session.get(
+                service_account_url,
+                timeout=_TIMEOUT,
+                params={"briefRepresentation": False},
+            )
+            sa_req.raise_for_status()
+            client["service_account_user"] = sa_req.json()
+        result.append(client)
+    return result
+
+
+def transform(
+    clients: list[dict[str, Any]], default_flows: dict[str, Any]
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]:
+    transformed_clients = []
+    service_accounts = []
+    flow_bindings = []
+    for client in clients:
+        sa = client.get("service_account_user")
+        if sa:
+            service_accounts.append(sa)
+            client["_service_account_user_id"] = sa["id"]
+            client.pop("service_account_user", None)
+
+        for flow_name, default_flow_id in default_flows.items():
+            flow_binding = {
+                "client_id": client["id"],
+                "flow_name": flow_name,
+                "flow_id": default_flow_id,
+                "default_flow": True,
+            }
+            if client.get("authenticationFlowBindingOverrides", {}).get(flow_name):
+                flow_binding["flow_id"] = client["authenticationFlowBindingOverrides"][
+                    flow_name
+                ]
+                flow_binding["default_flow"] = False
+            flow_bindings.append(flow_binding)
+
+        transformed_clients.append(client)
+    return transformed_clients, service_accounts, flow_bindings
+
+
+@timeit
+def load_clients(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    realm: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Keycloak Clients (%s) into Neo4j.", len(data), realm)
+    load(
+        neo4j_session,
+        KeycloakClientSchema(),
+        data,
+        LASTUPDATED=update_tag,
+        REALM=realm,
+    )
+
+
+@timeit
+def load_service_accounts(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    realm: str,
+    update_tag: int,
+) -> None:
+    logger.info(
+        "Loading %d Keycloak Service Accounts (%s) into Neo4j.", len(data), realm
+    )
+    load(
+        neo4j_session,
+        KeycloakUserSchema(),
+        data,
+        LASTUPDATED=update_tag,
+        REALM=realm,
+    )
+
+
+@timeit
+def load_flow_bindings(
+    neo4j_session: neo4j.Session,
+    biddings: list[dict[str, Any]],
+    realm_id: str,
+    update_tag: int,
+) -> None:
+    load_matchlinks(
+        neo4j_session,
+        KeycloakClientToFlowMatchLink(),
+        biddings,
+        LASTUPDATED=update_tag,
+        _sub_resource_label="KeycloakRealm",
+        _sub_resource_id=realm_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(KeycloakClientSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    # It's OK to cleanup users here as it will not clean the regular users (which have the same UPDATE_TAG)
+    GraphJob.from_node_schema(KeycloakUserSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_matchlink(
+        KeycloakClientToFlowMatchLink(),
+        sub_resource_label="KeycloakRealm",
+        sub_resource_id=common_job_parameters["REALM_ID"],
+        update_tag=common_job_parameters["UPDATE_TAG"],
+    ).run(
+        neo4j_session,
+    )

cartography/intel/keycloak/groups.py

@@ -0,0 +1,126 @@
+import logging
+from typing import Any
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.keycloak.util import get_paginated
+from cartography.models.keycloak.group import KeycloakGroupSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    base_url: str,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    groups = get(
+        api_session,
+        base_url,
+        common_job_parameters["REALM"],
+    )
+    transformed_groups = transform(groups)
+    load_groups(
+        neo4j_session,
+        transformed_groups,
+        common_job_parameters["REALM"],
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def _get_subgroups(
+    api_session: requests.Session,
+    base_url: str,
+    realm: str,
+    group_id: str,
+) -> list[dict[str, Any]]:
+    result: list[dict[str, Any]] = []
+    url = f"{base_url}/admin/realms/{realm}/groups/{group_id}/children"
+    for group in get_paginated(
+        api_session,
+        url,
+    ):
+        group["_members"] = _get_members(api_session, base_url, realm, group["id"])
+        result.append(group)
+        if group.get("subGroupCount", 0) > 0:
+            result.extend(_get_subgroups(api_session, base_url, realm, group["id"]))
+    return result
+
+
+@timeit
+def _get_members(
+    api_session: requests.Session,
+    base_url: str,
+    realm: str,
+    group_id: str,
+) -> list[dict[str, Any]]:
+    url = f"{base_url}/admin/realms/{realm}/groups/{group_id}/members"
+    return list(get_paginated(api_session, url))
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+    realm: str,
+) -> list[dict[str, Any]]:
+    result: list[dict[str, Any]] = []
+
+    url = f"{base_url}/admin/realms/{realm}/groups"
+    for group in get_paginated(api_session, url, params={"briefRepresentation": False}):
+        group["_members"] = _get_members(api_session, base_url, realm, group["id"])
+        result.append(group)
+        if group.get("subGroupCount", 0) > 0:
+            result.extend(_get_subgroups(api_session, base_url, realm, group["id"]))
+    return result
+
+
+def transform(groups: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    for group in groups:
+        # Transform members to a list of IDs for easier relationship handling
+        group["_member_ids"] = [m["id"] for m in group["_members"]]
+        group.pop("_members")
+        # Transform roles to a list of role names for easier relationship handling
+        group["_roles"] = []
+        for role_name in group.get("realmRoles", []):
+            group["_roles"].append(role_name)
+            group.pop("realmRoles", None)
+        for roles in group.get("clientRoles", {}).values():
+            for role_name in roles:
+                group["_roles"].append(role_name)
+        group.pop("clientRoles", None)
+    return groups
+
+
+@timeit
+def load_groups(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    realm: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Keycloak Groups (%s) into Neo4j.", len(data), realm)
+    load(
+        neo4j_session,
+        KeycloakGroupSchema(),
+        data,
+        LASTUPDATED=update_tag,
+        REALM=realm,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(KeycloakGroupSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/keycloak/identityproviders.py

@@ -0,0 +1,94 @@
+import logging
+from typing import Any
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.keycloak.util import get_paginated
+from cartography.models.keycloak.identityprovider import KeycloakIdentityProviderSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    base_url: str,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    identityproviders = get(
+        api_session,
+        base_url,
+        common_job_parameters["REALM"],
+    )
+    idps_transformed = transform(identityproviders)
+    load_identityproviders(
+        neo4j_session,
+        idps_transformed,
+        common_job_parameters["REALM"],
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+    realm: str,
+) -> list[dict[str, Any]]:
+    result: list[dict[str, Any]] = []
+    url = f"{base_url}/admin/realms/{realm}/identity-provider/instances"
+    for idp in get_paginated(api_session, url, params={"briefRepresentation": False}):
+        # Get members
+        members_url = f"{base_url}/admin/realms/{realm}/users"
+        idp["_members"] = list(
+            get_paginated(
+                api_session,
+                members_url,
+                params={"idpAlias": idp["alias"], "briefRepresentation": True},
+            )
+        )
+        result.append(idp)
+    return result
+
+
+def transform(idps: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    for idp in idps:
+        idp["_member_ids"] = [member["id"] for member in idp["_members"]]
+        idp.pop("_members", None)
+    return idps
+
+
+@timeit
+def load_identityproviders(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    realm: str,
+    update_tag: int,
+) -> None:
+    logger.info(
+        "Loading %d Keycloak IdentityProviders (%s) into Neo4j.", len(data), realm
+    )
+    load(
+        neo4j_session,
+        KeycloakIdentityProviderSchema(),
+        data,
+        LASTUPDATED=update_tag,
+        REALM=realm,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(
+        KeycloakIdentityProviderSchema(), common_job_parameters
+    ).run(neo4j_session)

cartography/intel/keycloak/organizations.py

@@ -0,0 +1,163 @@
+import logging
+from typing import Any
+from typing import Tuple
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.keycloak.util import get_paginated
+from cartography.models.keycloak.organization import KeycloakOrganizationSchema
+from cartography.models.keycloak.organizationdomain import (
+    KeycloakOrganizationDomainSchema,
+)
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    base_url: str,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    organizations = get(
+        api_session,
+        base_url,
+        common_job_parameters["REALM"],
+    )
+    transformed_orgs, transformed_domains = transform(organizations)
+    load_organizations(
+        neo4j_session,
+        transformed_orgs,
+        common_job_parameters["REALM"],
+        common_job_parameters["UPDATE_TAG"],
+    )
+    load_org_domains(
+        neo4j_session,
+        transformed_domains,
+        common_job_parameters["REALM"],
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+def transform(
+    organizations: list[dict[str, Any]],
+) -> Tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    transformed_orgs = []
+    transformed_domains = {}
+    for org in organizations:
+        # Transform members to a list of IDs
+        org["_managed_members"] = []
+        org["_unmanaged_members"] = []
+        for member in org.get("_members", []):
+            if member.get("membershipType") == "UNMANAGED":
+                org["_unmanaged_members"].append(member["id"])
+            else:
+                org["_managed_members"].append(member["id"])
+        org.pop("_members", None)
+        # Transform identity providers to a list of IDs
+        org["_idp_ids"] = [
+            idp["internalId"] for idp in org.get("_identity_providers", [])
+        ]
+        org.pop("_identity_providers", None)
+        # Extract domains
+        domains = org.get("domains", [])
+        for domain in domains:
+            domain_id = f"{org['id']}-{domain['name']}"
+            transformed_domains[domain_id] = {
+                "id": domain_id,
+                "verified": domain.get("verified", False),
+                "name": domain["name"],
+                "organization_id": org["id"],
+            }
+        org.pop("domains", None)
+        transformed_orgs.append(org)
+    return transformed_orgs, list(transformed_domains.values())
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+    realm: str,
+) -> list[dict[str, Any]]:
+    result: list[dict[str, Any]] = []
+    url = f"{base_url}/admin/realms/{realm}/organizations"
+    for org in get_paginated(api_session, url):
+        # Get members
+        members_url = (
+            f"{base_url}/admin/realms/{realm}/organizations/{org['id']}/members"
+        )
+        org["_members"] = list(
+            get_paginated(
+                api_session,
+                members_url,
+                params={"briefRepresentation": True},
+            )
+        )
+        # Get Identity Providers
+        idp_url = f"{base_url}/admin/realms/{realm}/organizations/{org['id']}/identity-providers"
+        org["_identity_providers"] = list(
+            get_paginated(
+                api_session,
+                idp_url,
+                params={"briefRepresentation": True},
+            )
+        )
+        result.append(org)
+    return result
+
+
+@timeit
+def load_organizations(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    realm: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Keycloak Organizations (%s) into Neo4j.", len(data), realm)
+    load(
+        neo4j_session,
+        KeycloakOrganizationSchema(),
+        data,
+        LASTUPDATED=update_tag,
+        REALM=realm,
+    )
+
+
+@timeit
+def load_org_domains(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    realm: str,
+    update_tag: int,
+) -> None:
+    logger.info(
+        "Loading %d Keycloak Organization Domains (%s) into Neo4j.", len(data), realm
+    )
+    load(
+        neo4j_session,
+        KeycloakOrganizationDomainSchema(),
+        data,
+        LASTUPDATED=update_tag,
+        REALM=realm,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(
+        KeycloakOrganizationDomainSchema(), common_job_parameters
+    ).run(neo4j_session)
+    GraphJob.from_node_schema(KeycloakOrganizationSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/keycloak/realms.py

@@ -0,0 +1,61 @@
+import logging
+from typing import Any
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.keycloak.realm import KeycloakRealmSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    base_url: str,
+    common_job_parameters: dict[str, Any],
+) -> list[dict]:
+    realms = get(api_session, base_url)
+    load_realms(neo4j_session, realms, common_job_parameters["UPDATE_TAG"])
+    cleanup(neo4j_session, common_job_parameters)
+    return realms
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+) -> list[dict[str, Any]]:
+    req = api_session.get(f"{base_url}/admin/realms", timeout=_TIMEOUT)
+    req.raise_for_status()
+    return req.json()
+
+
+@timeit
+def load_realms(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Keycloak Realms into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        KeycloakRealmSchema(),
+        data,
+        LASTUPDATED=update_tag,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(KeycloakRealmSchema(), common_job_parameters).run(
+        neo4j_session
+    )