cartography 0.93.0rc1__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/gsuite/groups.py

@@ -0,0 +1,291 @@
+import logging
+from typing import Any
+
+import neo4j
+from googleapiclient.discovery import Resource
+from googleapiclient.errors import HttpError
+
+from cartography.client.core.tx import load
+from cartography.client.core.tx import load_matchlinks
+from cartography.graph.job import GraphJob
+from cartography.models.gsuite.group import GSuiteGroupSchema
+from cartography.models.gsuite.group import GSuiteGroupToGroupMemberRel
+from cartography.models.gsuite.group import GSuiteGroupToGroupOwnerRel
+from cartography.models.gsuite.tenant import GSuiteTenantSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+GOOGLE_API_NUM_RETRIES = 5
+
+
+@timeit
+def get_all_groups(
+    admin: Resource, customer_id: str = "my_customer"
+) -> list[dict[str, Any]]:
+    """
+    Return list of Google Groups in your organization
+    Returns empty list if we are unable to enumerate the groups for any reasons
+
+    googleapiclient.discovery.build('admin', 'directory_v1', credentials=credentials, cache_discovery=False)
+
+    :param admin: google's apiclient discovery resource object.  From googleapiclient.discovery.build
+    See https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.discovery-module.html#build.
+    :return: list of Google groups in domain
+    """
+    request = admin.groups().list(
+        customer=customer_id,
+        maxResults=20,
+        orderBy="email",
+    )
+    response_objects = []
+    while request is not None:
+        try:
+            resp = request.execute(num_retries=GOOGLE_API_NUM_RETRIES)
+            response_objects.extend(resp.get("groups", []))
+            request = admin.groups().list_next(request, resp)
+        except HttpError as e:
+            if (
+                e.resp.status == 403
+                and "Request had insufficient authentication scopes" in str(e)
+            ):
+                logger.error(
+                    "Missing required GSuite scopes. If using the gcloud CLI, "
+                    "run: gcloud auth application-default login --scopes="
+                    '"https://www.googleapis.com/auth/admin.directory.user.readonly,'
+                    "https://www.googleapis.com/auth/admin.directory.group.readonly,"
+                    "https://www.googleapis.com/auth/admin.directory.group.member.readonly,"
+                    'https://www.googleapis.com/auth/cloud-platform"'
+                )
+            raise
+    return response_objects
+
+
+@timeit
+def get_members_for_groups(
+    admin: Resource, groups_email: list[str]
+) -> dict[str, list[dict[str, Any]]]:
+    """Get all members for given groups emails
+
+    Args:
+        admin (Resource): google's apiclient discovery resource object.  From googleapiclient.discovery.build
+        See https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.discovery-module.html#build.
+        groups_email (list[str]): List of group email addresses to get members for
+
+
+    :return: list of dictionaries representing Users or Groups grouped by group email
+    """
+    results: dict[str, list[dict]] = {}
+    for group_email in groups_email:
+        request = admin.members().list(
+            groupKey=group_email,
+            maxResults=500,
+        )
+        members: list[dict] = []
+        while request is not None:
+            resp = request.execute(num_retries=GOOGLE_API_NUM_RETRIES)
+            members = members + resp.get("members", [])
+            request = admin.members().list_next(request, resp)
+        results[group_email] = members
+
+    return results
+
+
+@timeit
+def transform_groups(
+    groups: list[dict], group_memberships: dict[str, list[dict[str, Any]]]
+) -> tuple[list[dict], list[dict], list[dict]]:
+    """Strips list of API response objects to return list of group objects only and lists of subgroup relationships
+
+    :param groups: Raw groups from Google API
+    :param group_memberships: Group memberships data
+    :return: tuple of (groups, group_member_relationships, group_owner_relationships)
+    """
+    transformed_groups: list[dict] = []
+    group_member_relationships: list[dict] = []
+    group_owner_relationships: list[dict] = []
+
+    for group in groups:
+        group_id = group["id"]
+        group_email = group["email"]
+        group["member_ids"] = []
+        group["owner_ids"] = []
+
+        for member in group_memberships.get(group_email, []):
+            if member["type"] == "GROUP":
+                # Create group-to-group relationships
+                relationship_data = {
+                    "parent_group_id": group_id,
+                    "subgroup_id": member.get("id"),
+                    "role": member.get("role"),
+                }
+
+                if member.get("role") == "OWNER":
+                    group_owner_relationships.append(relationship_data)
+                else:
+                    group_member_relationships.append(relationship_data)
+                continue
+
+            # Handle user memberships
+            if member.get("role") == "OWNER":
+                group["owner_ids"].append(member.get("id"))
+            group["member_ids"].append(member.get("id"))
+
+        transformed_groups.append(group)
+
+    return transformed_groups, group_member_relationships, group_owner_relationships
+
+
+@timeit
+def load_gsuite_groups(
+    neo4j_session: neo4j.Session,
+    groups: list[dict],
+    customer_id: str,
+    gsuite_update_tag: int,
+) -> None:
+    """
+    Load GSuite groups using the modern data model
+    """
+    logger.info("Ingesting %d gsuite groups", len(groups))
+
+    # Load tenant first if it doesn't exist
+    tenant_data = [{"id": customer_id}]
+    load(
+        neo4j_session,
+        GSuiteTenantSchema(),
+        tenant_data,
+        lastupdated=gsuite_update_tag,
+    )
+
+    # Load groups with relationship to tenant
+    load(
+        neo4j_session,
+        GSuiteGroupSchema(),
+        groups,
+        lastupdated=gsuite_update_tag,
+        CUSTOMER_ID=customer_id,
+    )
+
+
+@timeit
+def load_gsuite_group_to_group_relationships(
+    neo4j_session: neo4j.Session,
+    group_member_relationships: list[dict],
+    group_owner_relationships: list[dict],
+    customer_id: str,
+    gsuite_update_tag: int,
+) -> None:
+    """
+    Load GSuite group-to-group relationships using MatchLinks
+    """
+    logger.info(
+        "Ingesting %d group member relationships", len(group_member_relationships)
+    )
+    logger.info(
+        "Ingesting %d group owner relationships", len(group_owner_relationships)
+    )
+
+    # Load group member relationships (Group -> Group MEMBER)
+    if group_member_relationships:
+        load_matchlinks(
+            neo4j_session,
+            GSuiteGroupToGroupMemberRel(),
+            group_member_relationships,
+            lastupdated=gsuite_update_tag,
+            _sub_resource_label="GSuiteTenant",
+            _sub_resource_id=customer_id,
+        )
+
+    # Load group owner relationships (Group -> Group OWNER)
+    if group_owner_relationships:
+        load_matchlinks(
+            neo4j_session,
+            GSuiteGroupToGroupOwnerRel(),
+            group_owner_relationships,
+            lastupdated=gsuite_update_tag,
+            _sub_resource_label="GSuiteTenant",
+            _sub_resource_id=customer_id,
+        )
+
+
+@timeit
+def cleanup_gsuite_groups(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+    customer_id: str,
+    gsuite_update_tag: int,
+) -> None:
+    """
+    Clean up GSuite groups and group-to-group relationships using the modern data model
+    """
+    logger.debug("Running GSuite groups cleanup job")
+
+    # Cleanup group nodes
+    GraphJob.from_node_schema(GSuiteGroupSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+    # Cleanup group-to-group member relationships
+    GraphJob.from_matchlink(
+        GSuiteGroupToGroupMemberRel(),
+        "GSuiteTenant",
+        customer_id,
+        gsuite_update_tag,
+    ).run(neo4j_session)
+
+    # Cleanup group-to-group owner relationships
+    GraphJob.from_matchlink(
+        GSuiteGroupToGroupOwnerRel(),
+        "GSuiteTenant",
+        customer_id,
+        gsuite_update_tag,
+    ).run(neo4j_session)
+
+
+@timeit
+def sync_gsuite_groups(
+    neo4j_session: neo4j.Session,
+    admin: Resource,
+    gsuite_update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    """
+    GET GSuite group objects using the google admin api resource, load the data into Neo4j and clean up stale nodes.
+
+    :param neo4j_session: The Neo4j session
+    :param admin: Google admin resource object created by `googleapiclient.discovery.build()`.
+    See https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.discovery-module.html#build.
+    :param gsuite_update_tag: The timestamp value to set our new Neo4j nodes with
+    :param common_job_parameters: Parameters to carry to the Neo4j jobs
+    :return: Nothing
+    """
+    logger.debug("Syncing GSuite Groups")
+
+    customer_id = common_job_parameters.get(
+        "CUSTOMER_ID", "my_customer"
+    )  # Default to "my_customer" for backward compatibility
+
+    # 1. GET - Fetch data from API
+    resp_objs = get_all_groups(admin, customer_id)
+    group_members = get_members_for_groups(admin, [resp["email"] for resp in resp_objs])
+
+    # 2. TRANSFORM - Shape data for ingestion
+    groups, group_member_relationships, group_owner_relationships = transform_groups(
+        resp_objs, group_members
+    )
+
+    # 3. LOAD - Ingest to Neo4j using data model
+    load_gsuite_groups(neo4j_session, groups, customer_id, gsuite_update_tag)
+
+    # Load group-to-group relationships after groups are loaded
+    load_gsuite_group_to_group_relationships(
+        neo4j_session,
+        group_member_relationships,
+        group_owner_relationships,
+        customer_id,
+        gsuite_update_tag,
+    )
+
+    # 4. CLEANUP - Remove stale data
+    cleanup_params = {**common_job_parameters, "CUSTOMER_ID": customer_id}
+    cleanup_gsuite_groups(neo4j_session, cleanup_params, customer_id, gsuite_update_tag)

cartography/intel/gsuite/users.py

@@ -0,0 +1,142 @@
+import logging
+from collections import defaultdict
+from typing import Any
+
+import neo4j
+from googleapiclient.discovery import Resource
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.gsuite.tenant import GSuiteTenantSchema
+from cartography.models.gsuite.user import GSuiteUserSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+GOOGLE_API_NUM_RETRIES = 5
+
+
+@timeit
+def get_all_users(admin: Resource) -> list[dict]:
+    """
+    Return list of Google Users in your organization
+    Returns empty list if we are unable to enumerate the users for any reasons
+    https://developers.google.com/admin-sdk/directory/v1/guides/manage-users
+
+    :param admin: apiclient discovery resource object
+    :return: list of Google users in domain
+    see https://developers.google.com/admin-sdk/directory/v1/guides/manage-users#get_all_domain_users
+    """
+    request = admin.users().list(
+        customer="my_customer",
+        maxResults=500,
+        orderBy="email",
+    )
+    response_objects = []
+    while request is not None:
+        resp = request.execute(num_retries=GOOGLE_API_NUM_RETRIES)
+        response_objects.append(resp)
+        request = admin.users().list_next(request, resp)
+    return response_objects
+
+
+@timeit
+def transform_users(response_objects: list[dict]) -> dict[str, list[dict[str, Any]]]:
+    """Transform list of API response objects to return list of user objects with flattened structure grouped by customerId
+    :param response_objects: Raw API response objects
+    :return: list of dictionary objects for data model consumption
+    """
+    results = defaultdict(list)
+    for response_object in response_objects:
+        for user in response_object["users"]:
+            # Flatten the nested name structure
+            transformed_user = user.copy()
+            if "name" in user and isinstance(user["name"], dict):
+                transformed_user["name"] = user["name"].get("fullName")
+                transformed_user["family_name"] = user["name"].get("familyName")
+                transformed_user["given_name"] = user["name"].get("givenName")
+            results[transformed_user["customerId"]].append(transformed_user)
+    return results
+
+
+@timeit
+def load_gsuite_users(
+    neo4j_session: neo4j.Session,
+    users_by_customer: dict[str, list[dict]],
+    gsuite_update_tag: int,
+) -> None:
+    """
+    Load GSuite users using the modern data model
+    """
+    logger.info("Ingesting %s gsuite tenants", len(users_by_customer))
+    tenant_data = [{"id": customer_id} for customer_id in users_by_customer.keys()]
+    load(
+        neo4j_session,
+        GSuiteTenantSchema(),
+        tenant_data,
+        lastupdated=gsuite_update_tag,
+    )
+
+    for customer_id, users in users_by_customer.items():
+        logger.info(
+            "Ingesting %s gsuite users for customer %s", len(users), customer_id
+        )
+        # Load users with relationship to tenant
+        load(
+            neo4j_session,
+            GSuiteUserSchema(),
+            users,
+            lastupdated=gsuite_update_tag,
+            CUSTOMER_ID=customer_id,
+        )
+
+
+@timeit
+def cleanup_gsuite_users(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    """
+    Clean up GSuite users using the modern data model
+    """
+    logger.debug("Running GSuite users cleanup job")
+    GraphJob.from_node_schema(GSuiteUserSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync_gsuite_users(
+    neo4j_session: neo4j.Session,
+    admin: Resource,
+    gsuite_update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> list[str]:
+    """
+    GET GSuite user objects using the google admin api resource, load the data into Neo4j and clean up stale nodes.
+
+    :param neo4j_session: The Neo4j session
+    :param admin: Google admin resource object created by `googleapiclient.discovery.build()`.
+    See https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.discovery-module.html#build.
+    :param gsuite_update_tag: The timestamp value to set our new Neo4j nodes with
+    :param common_job_parameters: Parameters to carry to the Neo4j jobs
+    :return: list of customer IDs
+    """
+    logger.debug("Syncing GSuite Users")
+
+    # 1. GET - Fetch data from API
+    resp_objs = get_all_users(admin)
+
+    # 2. TRANSFORM - Shape data for ingestion
+    users_by_customers = transform_users(resp_objs)
+
+    # 3. LOAD - Ingest to Neo4j using data model
+    load_gsuite_users(neo4j_session, users_by_customers, gsuite_update_tag)
+
+    # 4. CLEANUP - Remove stale data
+    for customer_id in users_by_customers.keys():
+        cleanup_params = {**common_job_parameters, "CUSTOMER_ID": customer_id}
+        cleanup_gsuite_users(neo4j_session, cleanup_params)
+
+    # Return the list of customer IDs
+    return list(users_by_customers.keys())

cartography/intel/jamf/__init__.py

@@ -1,13 +1,31 @@
+import logging
+
 import neo4j
 
 from cartography.config import Config
 from cartography.intel.jamf import computers
 from cartography.util import timeit
 
+logger = logging.getLogger(__name__)
+
 
 @timeit
 def start_jamf_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+
+    if not config.jamf_base_uri or not config.jamf_user or not config.jamf_password:
+        # If the config is not set, we don't want to run this module
+        logger.info(
+            "Jamf import is not configured - skipping this module. See docs to configure."
+        )
+        return
+
     common_job_parameters = {
         "UPDATE_TAG": config.update_tag,
     }
-    computers.sync(neo4j_session, config.jamf_base_uri, config.jamf_user, config.jamf_password, common_job_parameters)
+    computers.sync(
+        neo4j_session,
+        config.jamf_base_uri,
+        config.jamf_user,
+        config.jamf_password,
+        common_job_parameters,
+    )

cartography/intel/jamf/computers.py

@@ -4,21 +4,29 @@ from typing import List
 
 import neo4j
 
+from cartography.client.core.tx import run_write_query
 from cartography.intel.jamf.util import call_jamf_api
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
-
 logger = logging.getLogger(__name__)
 
 
 @timeit
-def get_computer_groups(jamf_base_uri: str, jamf_user: str, jamf_password: str) -> List[Dict]:
+def get_computer_groups(
+    jamf_base_uri: str,
+    jamf_user: str,
+    jamf_password: str,
+) -> List[Dict]:
     return call_jamf_api("/computergroups", jamf_base_uri, jamf_user, jamf_password)
 
 
 @timeit
-def load_computer_groups(data: Dict, neo4j_session: neo4j.Session, update_tag: int) -> None:
+def load_computer_groups(
+    data: Dict,
+    neo4j_session: neo4j.Session,
+    update_tag: int,
+) -> None:
     ingest_groups = """
     UNWIND $JsonData as group
     MERGE (g:JamfComputerGroup{id: group.id})
@@ -28,17 +36,29 @@ def load_computer_groups(data: Dict, neo4j_session: neo4j.Session, update_tag: i
     g.lastupdated = $UpdateTag
     """
     groups = data.get("computer_groups")
-    neo4j_session.run(ingest_groups, JsonData=groups, UpdateTag=update_tag)
+    run_write_query(
+        neo4j_session,
+        ingest_groups,
+        JsonData=groups,
+        UpdateTag=update_tag,
+    )
 
 
 @timeit
 def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
-    run_cleanup_job('jamf_import_computers_cleanup.json', neo4j_session, common_job_parameters)
+    run_cleanup_job(
+        "jamf_import_computers_cleanup.json",
+        neo4j_session,
+        common_job_parameters,
+    )
 
 
 @timeit
 def sync_computer_groups(
-    neo4j_session: neo4j.Session, update_tag: int, jamf_base_uri: str, jamf_user: str,
+    neo4j_session: neo4j.Session,
+    update_tag: int,
+    jamf_base_uri: str,
+    jamf_user: str,
     jamf_password: str,
 ) -> None:
     groups = get_computer_groups(jamf_base_uri, jamf_user, jamf_password)
@@ -47,7 +67,16 @@ def sync_computer_groups(
 
 @timeit
 def sync(
-    neo4j_session: neo4j.Session, jamf_base_uri: str, jamf_user: str, jamf_password: str,
+    neo4j_session: neo4j.Session,
+    jamf_base_uri: str,
+    jamf_user: str,
+    jamf_password: str,
     common_job_parameters: Dict,
 ) -> None:
-    sync_computer_groups(neo4j_session, common_job_parameters['UPDATE_TAG'], jamf_base_uri, jamf_user, jamf_password)
+    sync_computer_groups(
+        neo4j_session,
+        common_job_parameters["UPDATE_TAG"],
+        jamf_base_uri,
+        jamf_user,
+        jamf_password,
+    )

cartography/intel/jamf/util.py

@@ -12,14 +12,19 @@ _TIMEOUT = (60, 60)
 
 
 @timeit
-def call_jamf_api(api_and_parameters: str, jamf_base_uri: str, jamf_user: str, jamf_password: str) -> List[Dict]:
+def call_jamf_api(
+    api_and_parameters: str,
+    jamf_base_uri: str,
+    jamf_user: str,
+    jamf_password: str,
+) -> List[Dict]:
     uri = jamf_base_uri + api_and_parameters
     jamf_auth = requests.auth.HTTPBasicAuth(jamf_user, jamf_password)
     try:
         response = requests.get(
             uri,
             auth=jamf_auth,
-            headers={'Accept': 'application/json'},
+            headers={"Accept": "application/json"},
             timeout=_TIMEOUT,
         )
     except requests.exceptions.Timeout:

cartography/intel/kandji/__init__.py

@@ -19,10 +19,13 @@ def start_kandji_ingestion(neo4j_session: neo4j.Session, config: Config) -> None
 
     :return: None
     """
-    if config.kandji_base_uri is None or config.kandji_token is None or config.kandji_tenant_id is None:
+    if (
+        config.kandji_base_uri is None
+        or config.kandji_token is None
+        or config.kandji_tenant_id is None
+    ):
         logger.warning(
-            'Required parameter(s) missing. Skipping sync.',
-            'See docs to configure.',
+            "Required parameter missing. Skipping sync. " "See docs to configure.",
         )
         return
 

cartography/intel/kandji/devices.py

@@ -12,7 +12,6 @@ from cartography.models.kandji.device import KandjiDeviceSchema
 from cartography.models.kandji.tenant import KandjiTenantSchema
 from cartography.util import timeit
 
-
 logger = logging.getLogger(__name__)
 _TIMEOUT = (60, 60)
 
@@ -21,14 +20,40 @@ _TIMEOUT = (60, 60)
 def get(kandji_base_uri: str, kandji_token: str) -> List[Dict[str, Any]]:
     api_endpoint = f"{kandji_base_uri}/api/v1/devices"
     headers = {
-        'Accept': 'application/json',
-        'Authorization': f'Bearer {kandji_token}',
+        "Accept": "application/json",
+        "Authorization": f"Bearer {kandji_token}",
+    }
+
+    offset = 0
+    limit = 300
+    params: dict[str, str | int] = {
+        "sort": "serial_number",
+        "limit": limit,
+        "offset": offset,
     }
 
+    devices: List[Dict[str, Any]] = []
     session = Session()
-    req = session.get(api_endpoint, headers=headers, timeout=_TIMEOUT)
-    req.raise_for_status()
-    return req.json()
+    while True:
+        logger.debug("Kandji device offset: %s", offset)
+
+        params["offset"] = offset
+        response = session.get(
+            api_endpoint, headers=headers, timeout=_TIMEOUT, params=params
+        )
+        response.raise_for_status()
+
+        result = response.json()
+        # If no more result, we are done
+        if len(result) == 0:
+            break
+
+        devices.extend(result)
+
+        offset += limit
+
+    logger.debug("Kandji device count: %d", len(devices))
+    return devices
 
 
 @timeit
@@ -36,7 +61,7 @@ def transform(api_result: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
     result: List[Dict[str, Any]] = []
     for device in api_result:
         n_device = device
-        n_device['id'] = device['device_id']
+        n_device["id"] = device["device_id"]
         result.append(n_device)
     return result
 
@@ -54,7 +79,7 @@ def load_devices(
     load(
         neo4j_session,
         KandjiTenantSchema(),
-        [{'id': tenant_id}],
+        [{"id": tenant_id}],
         lastupdated=update_tag,
     )
 
@@ -67,8 +92,13 @@ def load_devices(
     )
 
 
-def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
-    GraphJob.from_node_schema(KandjiDeviceSchema(), common_job_parameters).run(neo4j_session)
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    GraphJob.from_node_schema(KandjiDeviceSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
 
 
 @timeit