cartography 0.102.0rc2__py3-none-any.whl → 0.103.0__py3-none-any.whl

cartography/intel/duo/tokens.py

@@ -21,9 +21,9 @@ def sync(
     neo4j_session: neo4j.Session,
     common_job_parameters: Dict[str, Any],
 ) -> None:
-    '''
+    """
     Sync
-    '''
+    """
     data = _get(client)
     transformed_data = _transform(data)
     _load(neo4j_session, transformed_data, common_job_parameters)
@@ -32,34 +32,34 @@ def sync(
 
 @timeit
 def _get(client: duo_client.Admin) -> List[Dict[str, Any]]:
-    '''
+    """
     Fetch all data
     https://duo.com/docs/adminapi#endpoints
-    '''
-    logger.info(f'Fetching data for {Schema.label}')
+    """
+    logger.info(f"Fetching data for {Schema.label}")
     return client.get_tokens()
 
 
 @timeit
 def _transform(data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
-    '''
+    """
     Reformat the data before loading
-    '''
-    logger.info(f'Transforming {len(data)} items for {Schema.label}')
+    """
+    logger.info(f"Transforming {len(data)} items for {Schema.label}")
     transformed_data = []
     for datum in data:
         transformed_datum = {
-            'admins': json.dumps(datum['admins']),
-            'serial': datum['serial'],
-            'token_id': datum['token_id'],
-            'totp_step': datum['totp_step'],
-            'type': datum['type'],
+            "admins": json.dumps(datum["admins"]),
+            "serial": datum["serial"],
+            "token_id": datum["token_id"],
+            "totp_step": datum["totp_step"],
+            "type": datum["type"],
         }
         transformed_data.append(transformed_datum)
-        for user in datum['users']:
+        for user in datum["users"]:
             match_datum = {
                 **transformed_datum,
-                'user_id': user['user_id'],
+                "user_id": user["user_id"],
             }
             transformed_data.append(match_datum)
     return transformed_data
@@ -71,22 +71,25 @@ def _load(
     data: List[Dict[str, Any]],
     common_job_parameters: Dict[str, Any],
 ) -> None:
-    '''
+    """
     Load the data into the database
-    '''
-    logger.info(f'Loading {len(data)} items for {Schema.label}')
+    """
+    logger.info(f"Loading {len(data)} items for {Schema.label}")
     load(
         neo4j_session,
         Schema(),
         data,
-        DUO_API_HOSTNAME=common_job_parameters['DUO_API_HOSTNAME'],
-        lastupdated=common_job_parameters['UPDATE_TAG'],
+        DUO_API_HOSTNAME=common_job_parameters["DUO_API_HOSTNAME"],
+        lastupdated=common_job_parameters["UPDATE_TAG"],
     )
 
 
 @timeit
-def _cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
-    '''
+def _cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    """
     Cleanup nodes
-    '''
+    """
     GraphJob.from_node_schema(Schema(), common_job_parameters).run(neo4j_session)

cartography/intel/duo/users.py

@@ -21,9 +21,9 @@ def sync_duo_users(
     neo4j_session: neo4j.Session,
     common_job_parameters: Dict[str, Any],
 ) -> None:
-    '''
+    """
     Sync Duo Users
-    '''
+    """
     users = _get_users(client)
     transformed_users = _transform_users(users)
     _load_users(neo4j_session, transformed_users, common_job_parameters)
@@ -32,77 +32,78 @@ def sync_duo_users(
 
 @timeit
 def _get_users(client: duo_client.Admin) -> List[Dict[str, Any]]:
-    '''
+    """
     Fetch all users data
     https://duo.com/docs/adminapi#users
-    '''
+    """
     logger.info("Fetching Duo users")
     return client.get_users()
 
 
 @timeit
 def _transform_users(users: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
-    '''
+    """
     Reformat the data before loading
-    '''
-    logger.info(f'Transforming {len(users)} duo users')
+    """
+    logger.info(f"Transforming {len(users)} duo users")
     transformed_users = []
     for user in users:
         transformed_user = {
-            'alias1': user['alias1'],
-            'alias2': user['alias2'],
-            'alias3': user['alias3'],
-            'alias4': user['alias4'],
-            'created': user['created'],
-            'email': user['email'],
-            'firstname': user['firstname'],
-            'is_enrolled': user['is_enrolled'],
-            'last_directory_sync': user['last_directory_sync'],
-            'last_login': user['last_login'],
-            'lastname': user['lastname'],
-            'notes': user['notes'],
-            'phones': [
-                dumps({
-                    **phone,
-                    'number': None,
-                })
-                for phone in user['phones']
+            "alias1": user["alias1"],
+            "alias2": user["alias2"],
+            "alias3": user["alias3"],
+            "alias4": user["alias4"],
+            "created": user["created"],
+            "email": user["email"],
+            "firstname": user["firstname"],
+            "is_enrolled": user["is_enrolled"],
+            "last_directory_sync": user["last_directory_sync"],
+            "last_login": user["last_login"],
+            "lastname": user["lastname"],
+            "notes": user["notes"],
+            "phones": [
+                dumps(
+                    {
+                        **phone,
+                        "number": None,
+                    },
+                )
+                for phone in user["phones"]
             ],
-            'realname': user['realname'],
-            'status': user['status'],
-            'tokens': [dumps(token) for token in user['tokens']],
-            'u2ftokens': [dumps(u2ftoken) for u2ftoken in user['u2ftokens']],
-            'user_id': user['user_id'],
-            'username': user['username'],
-            'webauthncredentials': [
+            "realname": user["realname"],
+            "status": user["status"],
+            "tokens": [dumps(token) for token in user["tokens"]],
+            "u2ftokens": [dumps(u2ftoken) for u2ftoken in user["u2ftokens"]],
+            "user_id": user["user_id"],
+            "username": user["username"],
+            "webauthncredentials": [
                 dumps(webauthncredential)
-                for webauthncredential
-                in user['webauthncredentials']
+                for webauthncredential in user["webauthncredentials"]
             ],
         }
         transformed_users.append(transformed_user)
-        for group in user['groups']:
+        for group in user["groups"]:
             match_user = {
                 **transformed_user,
-                'group_id': group['group_id'],
+                "group_id": group["group_id"],
             }
             transformed_users.append(match_user)
-        for phone in user['phones']:
+        for phone in user["phones"]:
             match_user = {
                 **transformed_user,
-                'phone_id': phone['phone_id'],
+                "phone_id": phone["phone_id"],
             }
             transformed_users.append(match_user)
-        for token in user['tokens']:
+        for token in user["tokens"]:
             match_user = {
                 **transformed_user,
-                'token_id': token['token_id'],
+                "token_id": token["token_id"],
             }
             transformed_users.append(match_user)
-        for webauthncredential in user['webauthncredentials']:
+        for webauthncredential in user["webauthncredentials"]:
             match_user = {
                 **transformed_user,
-                'webauthnkey': webauthncredential['webauthnkey'],
+                "webauthnkey": webauthncredential["webauthnkey"],
             }
             transformed_users.append(match_user)
     return transformed_users
@@ -114,22 +115,25 @@ def _load_users(
     users: List[Dict[str, Any]],
     common_job_parameters: Dict[str, Any],
 ) -> None:
-    '''
+    """
     Load the users into the database
-    '''
-    logger.info(f'Loading {len(users)} duo users')
+    """
+    logger.info(f"Loading {len(users)} duo users")
     load(
         neo4j_session,
         DuoUserSchema(),
         users,
-        DUO_API_HOSTNAME=common_job_parameters['DUO_API_HOSTNAME'],
-        lastupdated=common_job_parameters['UPDATE_TAG'],
+        DUO_API_HOSTNAME=common_job_parameters["DUO_API_HOSTNAME"],
+        lastupdated=common_job_parameters["UPDATE_TAG"],
     )
 
 
 @timeit
-def _cleanup_users(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
-    '''
+def _cleanup_users(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    """
     Cleanup endpoints
-    '''
+    """
     GraphJob.from_node_schema(DuoUserSchema(), common_job_parameters).run(neo4j_session)

cartography/intel/duo/web_authn_credentials.py

@@ -8,7 +8,9 @@ import neo4j
 
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
-from cartography.models.duo.web_authn_credential import DuoWebAuthnCredentialSchema as Schema
+from cartography.models.duo.web_authn_credential import (
+    DuoWebAuthnCredentialSchema as Schema,
+)
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -20,9 +22,9 @@ def sync(
     neo4j_session: neo4j.Session,
     common_job_parameters: Dict[str, Any],
 ) -> None:
-    '''
+    """
     Sync
-    '''
+    """
     data = _get(client)
     transformed_data = _transform(data)
     _load(neo4j_session, transformed_data, common_job_parameters)
@@ -31,37 +33,37 @@ def sync(
 
 @timeit
 def _get(client: duo_client.Admin) -> List[Dict[str, Any]]:
-    '''
+    """
     Fetch all data
     https://duo.com/docs/adminapi#endpoints
-    '''
-    logger.info(f'Fetching data for {Schema.label}')
+    """
+    logger.info(f"Fetching data for {Schema.label}")
     return client.get_webauthncredentials()
 
 
 @timeit
 def _transform(data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
-    '''
+    """
     Reformat the data before loading
-    '''
-    logger.info(f'Transforming {len(data)} items for {Schema.label}')
+    """
+    logger.info(f"Transforming {len(data)} items for {Schema.label}")
     transformed_data = []
     for datum in data:
         transformed_datum = {
             # The admin property may be null if the cred is attached to a user
-            'admin': datum.get('admin'),
-            'credential_name': datum['credential_name'],
-            'date_added': datum['date_added'],
-            'label': datum['label'],
-            'user': datum['user'],
-            'webauthnkey': datum['webauthnkey'],
+            "admin": datum.get("admin"),
+            "credential_name": datum["credential_name"],
+            "date_added": datum["date_added"],
+            "label": datum["label"],
+            "user": datum["user"],
+            "webauthnkey": datum["webauthnkey"],
         }
         transformed_data.append(transformed_datum)
         # The user property may be null if the cred is attached to an admin
-        if datum.get('user'):
+        if datum.get("user"):
             match_datum = {
                 **transformed_datum,
-                'user_id': datum['user']['user_id'],
+                "user_id": datum["user"]["user_id"],
             }
             transformed_data.append(match_datum)
     return transformed_data
@@ -73,22 +75,25 @@ def _load(
     data: List[Dict[str, Any]],
     common_job_parameters: Dict[str, Any],
 ) -> None:
-    '''
+    """
     Load the data into the database
-    '''
-    logger.info(f'Loading {len(data)} items for {Schema.label}')
+    """
+    logger.info(f"Loading {len(data)} items for {Schema.label}")
     load(
         neo4j_session,
         Schema(),
         data,
-        DUO_API_HOSTNAME=common_job_parameters['DUO_API_HOSTNAME'],
-        lastupdated=common_job_parameters['UPDATE_TAG'],
+        DUO_API_HOSTNAME=common_job_parameters["DUO_API_HOSTNAME"],
+        lastupdated=common_job_parameters["UPDATE_TAG"],
     )
 
 
 @timeit
-def _cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
-    '''
+def _cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    """
     Cleanup nodes
-    '''
+    """
     GraphJob.from_node_schema(Schema(), common_job_parameters).run(neo4j_session)

cartography/intel/entra/__init__.py

@@ -4,6 +4,7 @@ import logging
 import neo4j
 
 from cartography.config import Config
+from cartography.intel.entra.ou import sync_entra_ous
 from cartography.intel.entra.users import sync_entra_users
 from cartography.util import timeit
 
@@ -19,10 +20,14 @@ def start_entra_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
     :return: None
     """
 
-    if not config.entra_tenant_id or not config.entra_client_id or not config.entra_client_secret:
+    if (
+        not config.entra_tenant_id
+        or not config.entra_client_id
+        or not config.entra_client_secret
+    ):
         logger.info(
-            'Entra import is not configured - skipping this module. '
-            'See docs to configure.',
+            "Entra import is not configured - skipping this module. "
+            "See docs to configure.",
         )
         return
 
@@ -31,13 +36,26 @@ def start_entra_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
         "TENANT_ID": config.entra_tenant_id,
     }
 
-    asyncio.run(
-        sync_entra_users(
+    async def main() -> None:
+        # Run user sync
+        await sync_entra_users(
             neo4j_session,
             config.entra_tenant_id,
             config.entra_client_id,
             config.entra_client_secret,
             config.update_tag,
             common_job_parameters,
-        ),
-    )
+        )
+
+        # Run OU sync
+        await sync_entra_ous(
+            neo4j_session,
+            config.entra_tenant_id,
+            config.entra_client_id,
+            config.entra_client_secret,
+            config.update_tag,
+            common_job_parameters,
+        )
+
+    # Execute both syncs in sequence
+    asyncio.run(main())

cartography/intel/entra/ou.py

@@ -0,0 +1,112 @@
+# cartography/intel/entra/ou.py
+import logging
+from typing import Any
+
+import neo4j
+from azure.identity import ClientSecretCredential
+from msgraph import GraphServiceClient
+from msgraph.generated.models.administrative_unit import AdministrativeUnit
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.entra.users import load_tenant
+from cartography.models.entra.ou import EntraOUSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+async def get_entra_ous(client: GraphServiceClient) -> list[AdministrativeUnit]:
+    """
+    Get all OUs from Microsoft Graph API with pagination support
+    """
+    all_units: list[AdministrativeUnit] = []
+    request = client.directory.administrative_units.request()
+
+    while request:
+        response = await request.get()
+        all_units.extend(response.value)
+        request = response.odata_next_link if response.odata_next_link else None
+
+    return all_units
+
+
+def transform_ous(
+    units: list[AdministrativeUnit], tenant_id: str
+) -> list[dict[str, Any]]:
+    """
+    Transform the API response into the format expected by our schema
+    """
+    result: list[dict[str, Any]] = []
+    for unit in units:
+        transformed_unit = {
+            "id": unit.id,
+            "display_name": unit.display_name,
+            "description": unit.description,
+            "visibility": unit.visibility,
+            "membership_type": unit.membership_type,
+            "is_member_management_restricted": unit.is_member_management_restricted,
+            "deleted_date_time": unit.deleted_date_time,
+            "tenant_id": tenant_id,
+        }
+        result.append(transformed_unit)
+    return result
+
+
+@timeit
+def load_ous(
+    neo4j_session: neo4j.Session,
+    units: list[dict[str, Any]],
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    logger.info(f"Loading {len(units)} Entra OUs")
+    load(
+        neo4j_session,
+        EntraOUSchema(),
+        units,
+        lastupdated=update_tag,
+        TENANT_ID=common_job_parameters["TENANT_ID"],
+        UPDATE_TAG=common_job_parameters["UPDATE_TAG"],
+    )
+
+
+def cleanup_ous(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(EntraOUSchema(), common_job_parameters).run(neo4j_session)
+
+
+@timeit
+async def sync_entra_ous(
+    neo4j_session: neo4j.Session,
+    tenant_id: str,
+    client_id: str,
+    client_secret: str,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    """
+    Sync Entra OUs
+    """
+    # Initialize Graph client
+    credential = ClientSecretCredential(
+        tenant_id=tenant_id,
+        client_id=client_id,
+        client_secret=client_secret,
+    )
+    client = GraphServiceClient(
+        credential, scopes=["https://graph.microsoft.com/.default"]
+    )
+
+    # Get OUs
+    units = await get_entra_ous(client)
+    transformed_units = transform_ous(units, tenant_id)
+
+    # Load data
+    load_tenant(neo4j_session, {"id": tenant_id}, update_tag)
+    load_ous(neo4j_session, transformed_units, update_tag, common_job_parameters)
+
+    # Cleanup stale data
+    cleanup_ous(neo4j_session, common_job_parameters)