cartography 0.104.0rc2__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/gsuite/groups.py

@@ -0,0 +1,291 @@
+import logging
+from typing import Any
+
+import neo4j
+from googleapiclient.discovery import Resource
+from googleapiclient.errors import HttpError
+
+from cartography.client.core.tx import load
+from cartography.client.core.tx import load_matchlinks
+from cartography.graph.job import GraphJob
+from cartography.models.gsuite.group import GSuiteGroupSchema
+from cartography.models.gsuite.group import GSuiteGroupToGroupMemberRel
+from cartography.models.gsuite.group import GSuiteGroupToGroupOwnerRel
+from cartography.models.gsuite.tenant import GSuiteTenantSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+GOOGLE_API_NUM_RETRIES = 5
+
+
+@timeit
+def get_all_groups(
+    admin: Resource, customer_id: str = "my_customer"
+) -> list[dict[str, Any]]:
+    """
+    Return list of Google Groups in your organization
+    Returns empty list if we are unable to enumerate the groups for any reasons
+
+    googleapiclient.discovery.build('admin', 'directory_v1', credentials=credentials, cache_discovery=False)
+
+    :param admin: google's apiclient discovery resource object.  From googleapiclient.discovery.build
+    See https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.discovery-module.html#build.
+    :return: list of Google groups in domain
+    """
+    request = admin.groups().list(
+        customer=customer_id,
+        maxResults=20,
+        orderBy="email",
+    )
+    response_objects = []
+    while request is not None:
+        try:
+            resp = request.execute(num_retries=GOOGLE_API_NUM_RETRIES)
+            response_objects.extend(resp.get("groups", []))
+            request = admin.groups().list_next(request, resp)
+        except HttpError as e:
+            if (
+                e.resp.status == 403
+                and "Request had insufficient authentication scopes" in str(e)
+            ):
+                logger.error(
+                    "Missing required GSuite scopes. If using the gcloud CLI, "
+                    "run: gcloud auth application-default login --scopes="
+                    '"https://www.googleapis.com/auth/admin.directory.user.readonly,'
+                    "https://www.googleapis.com/auth/admin.directory.group.readonly,"
+                    "https://www.googleapis.com/auth/admin.directory.group.member.readonly,"
+                    'https://www.googleapis.com/auth/cloud-platform"'
+                )
+            raise
+    return response_objects
+
+
+@timeit
+def get_members_for_groups(
+    admin: Resource, groups_email: list[str]
+) -> dict[str, list[dict[str, Any]]]:
+    """Get all members for given groups emails
+
+    Args:
+        admin (Resource): google's apiclient discovery resource object.  From googleapiclient.discovery.build
+        See https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.discovery-module.html#build.
+        groups_email (list[str]): List of group email addresses to get members for
+
+
+    :return: list of dictionaries representing Users or Groups grouped by group email
+    """
+    results: dict[str, list[dict]] = {}
+    for group_email in groups_email:
+        request = admin.members().list(
+            groupKey=group_email,
+            maxResults=500,
+        )
+        members: list[dict] = []
+        while request is not None:
+            resp = request.execute(num_retries=GOOGLE_API_NUM_RETRIES)
+            members = members + resp.get("members", [])
+            request = admin.members().list_next(request, resp)
+        results[group_email] = members
+
+    return results
+
+
+@timeit
+def transform_groups(
+    groups: list[dict], group_memberships: dict[str, list[dict[str, Any]]]
+) -> tuple[list[dict], list[dict], list[dict]]:
+    """Strips list of API response objects to return list of group objects only and lists of subgroup relationships
+
+    :param groups: Raw groups from Google API
+    :param group_memberships: Group memberships data
+    :return: tuple of (groups, group_member_relationships, group_owner_relationships)
+    """
+    transformed_groups: list[dict] = []
+    group_member_relationships: list[dict] = []
+    group_owner_relationships: list[dict] = []
+
+    for group in groups:
+        group_id = group["id"]
+        group_email = group["email"]
+        group["member_ids"] = []
+        group["owner_ids"] = []
+
+        for member in group_memberships.get(group_email, []):
+            if member["type"] == "GROUP":
+                # Create group-to-group relationships
+                relationship_data = {
+                    "parent_group_id": group_id,
+                    "subgroup_id": member.get("id"),
+                    "role": member.get("role"),
+                }
+
+                if member.get("role") == "OWNER":
+                    group_owner_relationships.append(relationship_data)
+                else:
+                    group_member_relationships.append(relationship_data)
+                continue
+
+            # Handle user memberships
+            if member.get("role") == "OWNER":
+                group["owner_ids"].append(member.get("id"))
+            group["member_ids"].append(member.get("id"))
+
+        transformed_groups.append(group)
+
+    return transformed_groups, group_member_relationships, group_owner_relationships
+
+
+@timeit
+def load_gsuite_groups(
+    neo4j_session: neo4j.Session,
+    groups: list[dict],
+    customer_id: str,
+    gsuite_update_tag: int,
+) -> None:
+    """
+    Load GSuite groups using the modern data model
+    """
+    logger.info("Ingesting %d gsuite groups", len(groups))
+
+    # Load tenant first if it doesn't exist
+    tenant_data = [{"id": customer_id}]
+    load(
+        neo4j_session,
+        GSuiteTenantSchema(),
+        tenant_data,
+        lastupdated=gsuite_update_tag,
+    )
+
+    # Load groups with relationship to tenant
+    load(
+        neo4j_session,
+        GSuiteGroupSchema(),
+        groups,
+        lastupdated=gsuite_update_tag,
+        CUSTOMER_ID=customer_id,
+    )
+
+
+@timeit
+def load_gsuite_group_to_group_relationships(
+    neo4j_session: neo4j.Session,
+    group_member_relationships: list[dict],
+    group_owner_relationships: list[dict],
+    customer_id: str,
+    gsuite_update_tag: int,
+) -> None:
+    """
+    Load GSuite group-to-group relationships using MatchLinks
+    """
+    logger.info(
+        "Ingesting %d group member relationships", len(group_member_relationships)
+    )
+    logger.info(
+        "Ingesting %d group owner relationships", len(group_owner_relationships)
+    )
+
+    # Load group member relationships (Group -> Group MEMBER)
+    if group_member_relationships:
+        load_matchlinks(
+            neo4j_session,
+            GSuiteGroupToGroupMemberRel(),
+            group_member_relationships,
+            lastupdated=gsuite_update_tag,
+            _sub_resource_label="GSuiteTenant",
+            _sub_resource_id=customer_id,
+        )
+
+    # Load group owner relationships (Group -> Group OWNER)
+    if group_owner_relationships:
+        load_matchlinks(
+            neo4j_session,
+            GSuiteGroupToGroupOwnerRel(),
+            group_owner_relationships,
+            lastupdated=gsuite_update_tag,
+            _sub_resource_label="GSuiteTenant",
+            _sub_resource_id=customer_id,
+        )
+
+
+@timeit
+def cleanup_gsuite_groups(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+    customer_id: str,
+    gsuite_update_tag: int,
+) -> None:
+    """
+    Clean up GSuite groups and group-to-group relationships using the modern data model
+    """
+    logger.debug("Running GSuite groups cleanup job")
+
+    # Cleanup group nodes
+    GraphJob.from_node_schema(GSuiteGroupSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+    # Cleanup group-to-group member relationships
+    GraphJob.from_matchlink(
+        GSuiteGroupToGroupMemberRel(),
+        "GSuiteTenant",
+        customer_id,
+        gsuite_update_tag,
+    ).run(neo4j_session)
+
+    # Cleanup group-to-group owner relationships
+    GraphJob.from_matchlink(
+        GSuiteGroupToGroupOwnerRel(),
+        "GSuiteTenant",
+        customer_id,
+        gsuite_update_tag,
+    ).run(neo4j_session)
+
+
+@timeit
+def sync_gsuite_groups(
+    neo4j_session: neo4j.Session,
+    admin: Resource,
+    gsuite_update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    """
+    GET GSuite group objects using the google admin api resource, load the data into Neo4j and clean up stale nodes.
+
+    :param neo4j_session: The Neo4j session
+    :param admin: Google admin resource object created by `googleapiclient.discovery.build()`.
+    See https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.discovery-module.html#build.
+    :param gsuite_update_tag: The timestamp value to set our new Neo4j nodes with
+    :param common_job_parameters: Parameters to carry to the Neo4j jobs
+    :return: Nothing
+    """
+    logger.debug("Syncing GSuite Groups")
+
+    customer_id = common_job_parameters.get(
+        "CUSTOMER_ID", "my_customer"
+    )  # Default to "my_customer" for backward compatibility
+
+    # 1. GET - Fetch data from API
+    resp_objs = get_all_groups(admin, customer_id)
+    group_members = get_members_for_groups(admin, [resp["email"] for resp in resp_objs])
+
+    # 2. TRANSFORM - Shape data for ingestion
+    groups, group_member_relationships, group_owner_relationships = transform_groups(
+        resp_objs, group_members
+    )
+
+    # 3. LOAD - Ingest to Neo4j using data model
+    load_gsuite_groups(neo4j_session, groups, customer_id, gsuite_update_tag)
+
+    # Load group-to-group relationships after groups are loaded
+    load_gsuite_group_to_group_relationships(
+        neo4j_session,
+        group_member_relationships,
+        group_owner_relationships,
+        customer_id,
+        gsuite_update_tag,
+    )
+
+    # 4. CLEANUP - Remove stale data
+    cleanup_params = {**common_job_parameters, "CUSTOMER_ID": customer_id}
+    cleanup_gsuite_groups(neo4j_session, cleanup_params, customer_id, gsuite_update_tag)

cartography/intel/gsuite/users.py

@@ -0,0 +1,142 @@
+import logging
+from collections import defaultdict
+from typing import Any
+
+import neo4j
+from googleapiclient.discovery import Resource
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.gsuite.tenant import GSuiteTenantSchema
+from cartography.models.gsuite.user import GSuiteUserSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+GOOGLE_API_NUM_RETRIES = 5
+
+
+@timeit
+def get_all_users(admin: Resource) -> list[dict]:
+    """
+    Return list of Google Users in your organization
+    Returns empty list if we are unable to enumerate the users for any reasons
+    https://developers.google.com/admin-sdk/directory/v1/guides/manage-users
+
+    :param admin: apiclient discovery resource object
+    :return: list of Google users in domain
+    see https://developers.google.com/admin-sdk/directory/v1/guides/manage-users#get_all_domain_users
+    """
+    request = admin.users().list(
+        customer="my_customer",
+        maxResults=500,
+        orderBy="email",
+    )
+    response_objects = []
+    while request is not None:
+        resp = request.execute(num_retries=GOOGLE_API_NUM_RETRIES)
+        response_objects.append(resp)
+        request = admin.users().list_next(request, resp)
+    return response_objects
+
+
+@timeit
+def transform_users(response_objects: list[dict]) -> dict[str, list[dict[str, Any]]]:
+    """Transform list of API response objects to return list of user objects with flattened structure grouped by customerId
+    :param response_objects: Raw API response objects
+    :return: list of dictionary objects for data model consumption
+    """
+    results = defaultdict(list)
+    for response_object in response_objects:
+        for user in response_object["users"]:
+            # Flatten the nested name structure
+            transformed_user = user.copy()
+            if "name" in user and isinstance(user["name"], dict):
+                transformed_user["name"] = user["name"].get("fullName")
+                transformed_user["family_name"] = user["name"].get("familyName")
+                transformed_user["given_name"] = user["name"].get("givenName")
+            results[transformed_user["customerId"]].append(transformed_user)
+    return results
+
+
+@timeit
+def load_gsuite_users(
+    neo4j_session: neo4j.Session,
+    users_by_customer: dict[str, list[dict]],
+    gsuite_update_tag: int,
+) -> None:
+    """
+    Load GSuite users using the modern data model
+    """
+    logger.info("Ingesting %s gsuite tenants", len(users_by_customer))
+    tenant_data = [{"id": customer_id} for customer_id in users_by_customer.keys()]
+    load(
+        neo4j_session,
+        GSuiteTenantSchema(),
+        tenant_data,
+        lastupdated=gsuite_update_tag,
+    )
+
+    for customer_id, users in users_by_customer.items():
+        logger.info(
+            "Ingesting %s gsuite users for customer %s", len(users), customer_id
+        )
+        # Load users with relationship to tenant
+        load(
+            neo4j_session,
+            GSuiteUserSchema(),
+            users,
+            lastupdated=gsuite_update_tag,
+            CUSTOMER_ID=customer_id,
+        )
+
+
+@timeit
+def cleanup_gsuite_users(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    """
+    Clean up GSuite users using the modern data model
+    """
+    logger.debug("Running GSuite users cleanup job")
+    GraphJob.from_node_schema(GSuiteUserSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync_gsuite_users(
+    neo4j_session: neo4j.Session,
+    admin: Resource,
+    gsuite_update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> list[str]:
+    """
+    GET GSuite user objects using the google admin api resource, load the data into Neo4j and clean up stale nodes.
+
+    :param neo4j_session: The Neo4j session
+    :param admin: Google admin resource object created by `googleapiclient.discovery.build()`.
+    See https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.discovery-module.html#build.
+    :param gsuite_update_tag: The timestamp value to set our new Neo4j nodes with
+    :param common_job_parameters: Parameters to carry to the Neo4j jobs
+    :return: list of customer IDs
+    """
+    logger.debug("Syncing GSuite Users")
+
+    # 1. GET - Fetch data from API
+    resp_objs = get_all_users(admin)
+
+    # 2. TRANSFORM - Shape data for ingestion
+    users_by_customers = transform_users(resp_objs)
+
+    # 3. LOAD - Ingest to Neo4j using data model
+    load_gsuite_users(neo4j_session, users_by_customers, gsuite_update_tag)
+
+    # 4. CLEANUP - Remove stale data
+    for customer_id in users_by_customers.keys():
+        cleanup_params = {**common_job_parameters, "CUSTOMER_ID": customer_id}
+        cleanup_gsuite_users(neo4j_session, cleanup_params)
+
+    # Return the list of customer IDs
+    return list(users_by_customers.keys())

cartography/intel/jamf/computers.py

@@ -4,6 +4,7 @@ from typing import List
 
 import neo4j
 
+from cartography.client.core.tx import run_write_query
 from cartography.intel.jamf.util import call_jamf_api
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
@@ -35,7 +36,12 @@ def load_computer_groups(
     g.lastupdated = $UpdateTag
     """
     groups = data.get("computer_groups")
-    neo4j_session.run(ingest_groups, JsonData=groups, UpdateTag=update_tag)
+    run_write_query(
+        neo4j_session,
+        ingest_groups,
+        JsonData=groups,
+        UpdateTag=update_tag,
+    )
 
 
 @timeit

cartography/intel/keycloak/__init__.py

@@ -0,0 +1,153 @@
+import logging
+
+import neo4j
+import requests
+
+import cartography.intel.keycloak.authenticationexecutions
+import cartography.intel.keycloak.authenticationflows
+import cartography.intel.keycloak.clients
+import cartography.intel.keycloak.groups
+import cartography.intel.keycloak.identityproviders
+import cartography.intel.keycloak.organizations
+import cartography.intel.keycloak.realms
+import cartography.intel.keycloak.roles
+import cartography.intel.keycloak.scopes
+import cartography.intel.keycloak.users
+from cartography.config import Config
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def start_keycloak_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of Keycloak data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+    if (
+        not config.keycloak_client_id
+        or not config.keycloak_client_secret
+        or not config.keycloak_url
+        or not config.keycloak_realm
+    ):
+        logger.info(
+            "Keycloak import is not configured - skipping this module. "
+            "See docs to configure.",
+        )
+        return
+
+    # Create requests sessions
+    with requests.session() as api_session:
+        payload = {
+            "grant_type": "client_credentials",
+            "client_id": config.keycloak_client_id,
+            "client_secret": config.keycloak_client_secret,
+        }
+        req = api_session.post(
+            f"{config.keycloak_url}/realms/{config.keycloak_realm}/protocol/openid-connect/token",
+            data=payload,
+            timeout=_TIMEOUT,
+        )
+        req.raise_for_status()
+        api_session.headers.update(
+            {"Authorization": f'Bearer {req.json()["access_token"]}'}
+        )
+
+        common_job_parameters = {
+            "UPDATE_TAG": config.update_tag,
+        }
+
+        for realm in cartography.intel.keycloak.realms.sync(
+            neo4j_session, api_session, config.keycloak_url, common_job_parameters
+        ):
+            realm_scopped_job_parameters = {
+                "UPDATE_TAG": config.update_tag,
+                "REALM": realm["realm"],
+                "REALM_ID": realm["id"],
+            }
+            cartography.intel.keycloak.users.sync(
+                neo4j_session,
+                api_session,
+                config.keycloak_url,
+                realm_scopped_job_parameters,
+            )
+            cartography.intel.keycloak.identityproviders.sync(
+                neo4j_session,
+                api_session,
+                config.keycloak_url,
+                realm_scopped_job_parameters,
+            )
+            scopes = cartography.intel.keycloak.scopes.sync(
+                neo4j_session,
+                api_session,
+                config.keycloak_url,
+                realm_scopped_job_parameters,
+            )
+            scope_ids = [s["id"] for s in scopes]
+            flows = cartography.intel.keycloak.authenticationflows.sync(
+                neo4j_session,
+                api_session,
+                config.keycloak_url,
+                realm_scopped_job_parameters,
+            )
+            flow_aliases_to_id = {f["alias"]: f["id"] for f in flows}
+            cartography.intel.keycloak.authenticationexecutions.sync(
+                neo4j_session,
+                api_session,
+                config.keycloak_url,
+                realm_scopped_job_parameters,
+                list(flow_aliases_to_id.keys()),
+            )
+            realm_default_flows = {
+                "browser": flow_aliases_to_id.get(realm.get("browserFlow")),
+                "registration": flow_aliases_to_id.get(realm.get("registrationFlow")),
+                "direct_grant": flow_aliases_to_id.get(realm.get("directGrantFlow")),
+                "reset_credentials": flow_aliases_to_id.get(
+                    realm.get("resetCredentialsFlow")
+                ),
+                "client_authentication": flow_aliases_to_id.get(
+                    realm.get("clientAuthenticationFlow")
+                ),
+                "docker_authentication": flow_aliases_to_id.get(
+                    realm.get("dockerAuthenticationFlow")
+                ),
+                "first_broker_login": flow_aliases_to_id.get(
+                    realm.get("firstBrokerLoginFlow")
+                ),
+            }
+
+            clients = cartography.intel.keycloak.clients.sync(
+                neo4j_session,
+                api_session,
+                config.keycloak_url,
+                realm_scopped_job_parameters,
+                realm_default_flows,
+            )
+            client_ids = [c["id"] for c in clients]
+            cartography.intel.keycloak.roles.sync(
+                neo4j_session,
+                api_session,
+                config.keycloak_url,
+                realm_scopped_job_parameters,
+                client_ids,
+                scope_ids,
+            )
+            cartography.intel.keycloak.groups.sync(
+                neo4j_session,
+                api_session,
+                config.keycloak_url,
+                realm_scopped_job_parameters,
+            )
+
+            # Organizations if they are enabled
+            if realm.get("organizationsEnabled", False):
+                cartography.intel.keycloak.organizations.sync(
+                    neo4j_session,
+                    api_session,
+                    config.keycloak_url,
+                    realm_scopped_job_parameters,
+                )