cartography 0.106.0rc1__py3-none-any.whl → 0.107.0rc1__py3-none-any.whl

cartography/intel/aws/efs.py

@@ -9,6 +9,7 @@ import neo4j
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
 from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.efs.access_point import EfsAccessPointSchema
 from cartography.models.aws.efs.file_system import EfsFileSystemSchema
 from cartography.models.aws.efs.mount_target import EfsMountTargetSchema
 from cartography.util import aws_handle_regions
@@ -44,6 +45,7 @@ def transform_efs_file_systems(
         transformed_file_system = {
             "FileSystemId": file_system["FileSystemId"],
             "FileSystemArn": file_system["FileSystemArn"],
+            "Region": region,
             "OwnerId": file_system.get("OwnerId"),
             "CreationToken": file_system.get("CreationToken"),
             "CreationTime": file_system.get("CreationTime"),
@@ -87,6 +89,49 @@ def get_efs_mount_targets(
     return mountTargets
 
 
+@timeit
+@aws_handle_regions
+def get_efs_access_points(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "efs", region_name=region, config=get_botocore_config()
+    )
+
+    paginator = client.get_paginator("describe_access_points")
+    accessPoints = []
+    for page in paginator.paginate():
+        accessPoints.extend(page.get("AccessPoints", []))
+
+    return accessPoints
+
+
+def transform_efs_access_points(
+    accessPoints: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    """
+    Transform Efs Access Points data for ingestion
+    """
+    transformed = []
+    for ap in accessPoints:
+        transformed.append(
+            {
+                "AccessPointArn": ap["AccessPointArn"],
+                "AccessPointId": ap["AccessPointId"],
+                "Region": region,
+                "FileSystemId": ap["FileSystemId"],
+                "Name": ap.get("Name"),
+                "LifeCycleState": ap.get("LifeCycleState"),
+                "OwnerId": ap.get("OwnerId"),
+                "Uid": ap.get("PosixUser", {}).get("Uid"),
+                "Gid": ap.get("PosixUser", {}).get("Gid"),
+                "RootDirectoryPath": ap.get("RootDirectory", {}).get("Path"),
+            }
+        )
+
+    return transformed
+
+
 @timeit
 def load_efs_mount_targets(
     neo4j_session: neo4j.Session,
@@ -129,6 +174,27 @@ def load_efs_file_systems(
     )
 
 
+@timeit
+def load_efs_access_points(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Efs {len(data)} access points for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        EfsAccessPointSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
 @timeit
 def cleanup(
     neo4j_session: neo4j.Session,
@@ -141,6 +207,9 @@ def cleanup(
     GraphJob.from_node_schema(EfsFileSystemSchema(), common_job_parameters).run(
         neo4j_session
     )
+    GraphJob.from_node_schema(EfsAccessPointSchema(), common_job_parameters).run(
+        neo4j_session
+    )
 
 
 @timeit
@@ -178,4 +247,15 @@ def sync(
             update_tag,
         )
 
+        accessPoints = get_efs_access_points(boto3_session, region)
+        accessPoints_transformed = transform_efs_access_points(accessPoints, region)
+
+        load_efs_access_points(
+            neo4j_session,
+            accessPoints_transformed,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
     cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/inspector.py

@@ -16,8 +16,6 @@ from cartography.util import aws_handle_regions
 from cartography.util import aws_paginate
 from cartography.util import batch
 from cartography.util import timeit
-from cartography.util import to_asynchronous
-from cartography.util import to_synchronous
 
 logger = logging.getLogger(__name__)
 
@@ -329,10 +327,9 @@ def sync(
         member_accounts = get_member_accounts(boto3_session, region)
         # the current host account may not be considered a "member", but we still fetch its findings
         member_accounts.append(current_aws_account_id)
-
-        async def async_ingest_findings_for_account(account_id: str) -> None:
-            await to_asynchronous(
-                _sync_findings_for_account,
+        logger.info(f"Member accounts to be synced: {member_accounts}")
+        for account_id in member_accounts:
+            _sync_findings_for_account(
                 neo4j_session,
                 boto3_session,
                 region,
@@ -341,11 +338,4 @@ def sync(
                 current_aws_account_id,
             )
 
-        to_synchronous(
-            *[
-                async_ingest_findings_for_account(account_id)
-                for account_id in member_accounts
-            ]
-        )
-
     cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/resources.py

@@ -6,7 +6,9 @@ from cartography.intel.aws.ec2.route_tables import sync_route_tables
 from . import acm
 from . import apigateway
 from . import cloudtrail
+from . import cloudtrail_management_events
 from . import cloudwatch
+from . import codebuild
 from . import config
 from . import dynamodb
 from . import ecr
@@ -106,6 +108,8 @@ RESOURCE_FUNCTIONS: Dict[str, Callable[..., None]] = {
     "config": config.sync,
     "identitycenter": identitycenter.sync_identity_center_instances,
     "cloudtrail": cloudtrail.sync,
+    "cloudtrail_management_events": cloudtrail_management_events.sync,
     "cloudwatch": cloudwatch.sync,
     "efs": efs.sync,
+    "codebuild": codebuild.sync,
 }

cartography/intel/aws/sns.py

@@ -1,4 +1,5 @@
 import logging
+from typing import Any
 from typing import Dict
 from typing import List
 from typing import Optional
@@ -9,6 +10,7 @@ import neo4j
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
 from cartography.models.aws.sns.topic import SNSTopicSchema
+from cartography.models.aws.sns.topic_subscription import SNSTopicSubscriptionSchema
 from cartography.stats import get_stats_client
 from cartography.util import aws_handle_regions
 from cartography.util import merge_module_sync_metadata
@@ -108,6 +110,48 @@ def load_sns_topics(
     )
 
 
+@timeit
+@aws_handle_regions
+def get_subscriptions(
+    boto3_session: boto3.session.Session, region: str
+) -> List[Dict[str, Any]]:
+    """
+    Get all SNS Topics Subscriptions for a region.
+    """
+    client = boto3_session.client("sns", region_name=region)
+    paginator = client.get_paginator("list_subscriptions")
+    subscriptions = []
+    for page in paginator.paginate():
+        subscriptions.extend(page.get("Subscriptions", []))
+
+    return subscriptions
+
+
+@timeit
+def load_sns_topic_subscription(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    aws_account_id: str,
+    update_tag: int,
+) -> None:
+    """
+    Load SNS Topic Subscription information into the graph
+    """
+    logger.info(
+        f"Loading {len(data)} SNS topic subscription for region {region} into graph."
+    )
+
+    load(
+        neo4j_session,
+        SNSTopicSubscriptionSchema(),
+        data,
+        lastupdated=update_tag,
+        Region=region,
+        AWS_ID=aws_account_id,
+    )
+
+
 @timeit
 def cleanup_sns(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
     """
@@ -117,6 +161,11 @@ def cleanup_sns(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> No
     cleanup_job = GraphJob.from_node_schema(SNSTopicSchema(), common_job_parameters)
     cleanup_job.run(neo4j_session)
 
+    cleanup_job = GraphJob.from_node_schema(
+        SNSTopicSubscriptionSchema(), common_job_parameters
+    )
+    cleanup_job.run(neo4j_session)
+
 
 @timeit
 def sync(
@@ -128,7 +177,7 @@ def sync(
     common_job_parameters: Dict,
 ) -> None:
     """
-    Sync SNS Topics for all regions
+    Sync SNS Topics and Subscriptions for all regions
     """
     for region in regions:
         logger.info(
@@ -153,9 +202,20 @@ def sync(
             update_tag,
         )
 
+        # Get and load subscriptions
+        subscriptions = get_subscriptions(boto3_session, region)
+
+        load_sns_topic_subscription(
+            neo4j_session,
+            subscriptions,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    # Cleanup and metadata update (outside region loop)
     cleanup_sns(neo4j_session, common_job_parameters)
 
-    # Record that we've synced this module
     merge_module_sync_metadata(
         neo4j_session,
         group_type="AWSAccount",

cartography/intel/entra/users.py

@@ -15,6 +15,51 @@ from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
+# NOTE:
+# Microsoft Graph imposes limits on the length of the $select clause as well as
+# the number of properties that can be selected in a single request.  In
+# practice we have seen 400 Bad Request responses that bubble up as
+# `Microsoft.SharePoint.Client.InvalidClientQueryException` once that limit is
+# breached (Graph internally rewrites the next-link using a SharePoint style
+# `id in (…)` filter which is then rejected).
+#
+# To avoid tripping this bug we only request a *core* subset of user attributes
+# that are most commonly used in downstream analysis.  The transform() function
+# tolerates missing attributes (the generated MS Graph SDK simply returns
+# `None` for properties that are not present in the payload), so fetching fewer
+# fields is safe – we merely get more `null` values in the graph.
+#
+# If you need additional attributes in the future, append them here but keep the
+# total character count of the comma-separated list comfortably below 500 and
+# stay within the official v1.0 contract (beta-only fields cause similar
+# failures). 20–25 fields is a good rule-of-thumb.
+#
+# References:
+#   • https://learn.microsoft.com/graph/query-parameters#select-parameter
+#   • https://learn.microsoft.com/graph/api/user-list?view=graph-rest-1.0
+#
+USER_SELECT_FIELDS = [
+    "id",
+    "userPrincipalName",
+    "displayName",
+    "givenName",
+    "surname",
+    "mail",
+    "mobilePhone",
+    "businessPhones",
+    "jobTitle",
+    "department",
+    "officeLocation",
+    "city",
+    "country",
+    "companyName",
+    "preferredLanguage",
+    "employeeId",
+    "employeeType",
+    "accountEnabled",
+    "ageGroup",
+]
+
 
 @timeit
 async def get_tenant(client: GraphServiceClient) -> Organization:
@@ -27,14 +72,20 @@ async def get_tenant(client: GraphServiceClient) -> Organization:
 
 @timeit
 async def get_users(client: GraphServiceClient) -> list[User]:
+    """Fetch all users with their manager reference in as few requests as possible.
+
+    We leverage `$expand=manager($select=id)` so the manager's *id* is hydrated
+    alongside every user record.  This avoids making a second round-trip per
+    user – vastly reducing latency and eliminating the noisy 404s that occur
+    when a user has no manager assigned.
     """
-    Get all users from Microsoft Graph API with pagination support
-    """
+
     all_users: list[User] = []
     request_configuration = client.users.UsersRequestBuilderGetRequestConfiguration(
         query_parameters=client.users.UsersRequestBuilderGetQueryParameters(
-            # Request more items per page to reduce number of API calls
             top=999,
+            select=USER_SELECT_FIELDS,
+            expand=["manager($select=id)"],
         ),
     )
 
@@ -43,18 +94,32 @@ async def get_users(client: GraphServiceClient) -> list[User]:
         all_users.extend(page.value)
         if not page.odata_next_link:
             break
-        page = await client.users.with_url(page.odata_next_link).get()
+
+        try:
+            page = await client.users.with_url(page.odata_next_link).get()
+        except Exception as e:
+            logger.error(
+                "Failed to fetch next page of Entra ID users – stopping pagination early: %s",
+                e,
+            )
+            break
 
     return all_users
 
 
 @timeit
+# The manager reference is now embedded in the user objects courtesy of the
+# `$expand` we added above, so we no longer need a separate `manager_map`.
 def transform_users(users: list[User]) -> list[dict[str, Any]]:
-    """
-    Transform the API response into the format expected by our schema
-    """
+    """Convert MS Graph SDK `User` models into dicts matching our schema."""
+
     result: list[dict[str, Any]] = []
     for user in users:
+        manager_id: str | None = None
+        if getattr(user, "manager", None) is not None:
+            # The SDK materialises `manager` as a DirectoryObject (or subclass)
+            manager_id = getattr(user.manager, "id", None)
+
         transformed_user = {
             "id": user.id,
             "user_principal_name": user.user_principal_name,
@@ -62,47 +127,24 @@ def transform_users(users: list[User]) -> list[dict[str, Any]]:
             "given_name": user.given_name,
             "surname": user.surname,
             "mail": user.mail,
-            "other_mails": user.other_mails,
-            "preferred_language": user.preferred_language,
-            "preferred_name": user.preferred_name,
-            "state": user.state,
-            "usage_location": user.usage_location,
-            "user_type": user.user_type,
-            "show_in_address_list": user.show_in_address_list,
-            "sign_in_sessions_valid_from_date_time": user.sign_in_sessions_valid_from_date_time,
-            "security_identifier": user.on_premises_security_identifier,
-            "account_enabled": user.account_enabled,
-            "age_group": user.age_group,
+            "mobile_phone": user.mobile_phone,
             "business_phones": user.business_phones,
+            "job_title": user.job_title,
+            "department": user.department,
+            "office_location": user.office_location,
             "city": user.city,
-            "company_name": user.company_name,
-            "consent_provided_for_minor": user.consent_provided_for_minor,
+            "state": user.state,
             "country": user.country,
-            "created_date_time": user.created_date_time,
-            "creation_type": user.creation_type,
-            "deleted_date_time": user.deleted_date_time,
-            "department": user.department,
+            "company_name": user.company_name,
+            "preferred_language": user.preferred_language,
             "employee_id": user.employee_id,
             "employee_type": user.employee_type,
-            "external_user_state": user.external_user_state,
-            "external_user_state_change_date_time": user.external_user_state_change_date_time,
-            "hire_date": user.hire_date,
-            "is_management_restricted": user.is_management_restricted,
-            "is_resource_account": user.is_resource_account,
-            "job_title": user.job_title,
-            "last_password_change_date_time": user.last_password_change_date_time,
-            "mail_nickname": user.mail_nickname,
-            "office_location": user.office_location,
-            "on_premises_distinguished_name": user.on_premises_distinguished_name,
-            "on_premises_domain_name": user.on_premises_domain_name,
-            "on_premises_immutable_id": user.on_premises_immutable_id,
-            "on_premises_last_sync_date_time": user.on_premises_last_sync_date_time,
-            "on_premises_sam_account_name": user.on_premises_sam_account_name,
-            "on_premises_security_identifier": user.on_premises_security_identifier,
-            "on_premises_sync_enabled": user.on_premises_sync_enabled,
-            "on_premises_user_principal_name": user.on_premises_user_principal_name,
+            "account_enabled": user.account_enabled,
+            "age_group": user.age_group,
+            "manager_id": manager_id,
         }
         result.append(transformed_user)
+
     return result
 
 
@@ -198,7 +240,7 @@ async def sync_entra_users(
         credential, scopes=["https://graph.microsoft.com/.default"]
     )
 
-    # Get tenant information
+    # Fetch tenant and users (with manager reference already populated by `$expand`)
     tenant = await get_tenant(client)
     users = await get_users(client)
 

cartography/intel/scaleway/__init__.py

@@ -0,0 +1,127 @@
+import logging
+
+import neo4j
+import scaleway
+
+import cartography.intel.scaleway.iam.apikeys
+import cartography.intel.scaleway.iam.applications
+import cartography.intel.scaleway.iam.groups
+import cartography.intel.scaleway.iam.users
+import cartography.intel.scaleway.instances.flexibleips
+import cartography.intel.scaleway.instances.instances
+import cartography.intel.scaleway.projects
+import cartography.intel.scaleway.storage.snapshots
+import cartography.intel.scaleway.storage.volumes
+from cartography.config import Config
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def start_scaleway_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of Scaleway data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+
+    if (
+        not config.scaleway_access_key
+        or not config.scaleway_secret_key
+        or not config.scaleway_org
+    ):
+        logger.info(
+            "Tailscale import is not configured - skipping this module. "
+            "See docs to configure.",
+        )
+        return
+
+    # Create client
+    client = scaleway.Client(
+        access_key=config.scaleway_access_key,
+        secret_key=config.scaleway_secret_key,
+    )
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+        "ORG_ID": config.scaleway_org,
+    }
+
+    # Organization level
+    projects = cartography.intel.scaleway.projects.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        update_tag=config.update_tag,
+    )
+    projects_id = [project["id"] for project in projects]
+    cartography.intel.scaleway.iam.users.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        update_tag=config.update_tag,
+    )
+    cartography.intel.scaleway.iam.applications.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        update_tag=config.update_tag,
+    )
+    cartography.intel.scaleway.iam.groups.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        update_tag=config.update_tag,
+    )
+    cartography.intel.scaleway.iam.apikeys.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        update_tag=config.update_tag,
+    )
+
+    # Storage
+    cartography.intel.scaleway.storage.volumes.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        projects_id=projects_id,
+        update_tag=config.update_tag,
+    )
+    cartography.intel.scaleway.storage.snapshots.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        projects_id=projects_id,
+        update_tag=config.update_tag,
+    )
+
+    # Instances
+    # DISABLED due to https://github.com/scaleway/scaleway-sdk-python/issues/1040
+    """
+    cartography.intel.scaleway.instances.flexibleips.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        projects_id=projects_id,
+        update_tag=config.update_tag,
+    )
+    """
+    cartography.intel.scaleway.instances.instances.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        projects_id=projects_id,
+        update_tag=config.update_tag,
+    )

cartography/intel/scaleway/iam/apikeys.py

@@ -0,0 +1,71 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.iam.v1alpha1 import APIKey
+from scaleway.iam.v1alpha1 import IamV1Alpha1API
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.iam.apikey import ScalewayApiKeySchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    apikeys = get(client, org_id)
+    formatted_apikeys = transform_apikeys(apikeys)
+    load_apikeys(neo4j_session, formatted_apikeys, org_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[APIKey]:
+    api = IamV1Alpha1API(client)
+    return api.list_api_keys_all(organization_id=org_id)
+
+
+def transform_apikeys(apikeys: list[APIKey]) -> list[dict[str, Any]]:
+    formatted_apikeys = []
+    for apikey in apikeys:
+        formatted_apikeys.append(scaleway_obj_to_dict(apikey))
+    return formatted_apikeys
+
+
+@timeit
+def load_apikeys(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Scaleway ApiKeys into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        ScalewayApiKeySchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=org_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(ScalewayApiKeySchema(), common_job_parameters).run(
+        neo4j_session
+    )