cartography 0.106.0rc2__py3-none-any.whl → 0.107.0__py3-none-any.whl

cartography/intel/aws/inspector.py

@@ -3,14 +3,17 @@ from typing import Any
 from typing import Dict
 from typing import Iterator
 from typing import List
+from typing import Set
 from typing import Tuple
 
 import boto3
 import neo4j
 
 from cartography.client.core.tx import load
+from cartography.client.core.tx import load_matchlinks
 from cartography.graph.job import GraphJob
 from cartography.models.aws.inspector.findings import AWSInspectorFindingSchema
+from cartography.models.aws.inspector.findings import InspectorFindingToPackageMatchLink
 from cartography.models.aws.inspector.packages import AWSInspectorPackageSchema
 from cartography.util import aws_handle_regions
 from cartography.util import aws_paginate
@@ -107,9 +110,10 @@ def get_inspector_findings(
 
 def transform_inspector_findings(
     results: List[Dict[str, Any]],
-) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
+) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], List[Dict[str, str]]]:
     findings_list: List[Dict] = []
-    packages: Dict[str, Any] = {}
+    packages_set: Set[frozenset] = set()
+    finding_to_package_map: List[Dict[str, str]] = []
 
     for f in results:
         finding: Dict = {}
@@ -163,55 +167,45 @@ def transform_inspector_findings(
                 "vendorUpdatedAt",
             )
 
-            new_packages = _process_packages(
-                f["packageVulnerabilityDetails"],
-                f["awsAccountId"],
-                f["findingArn"],
-            )
-            finding["vulnerablepackageids"] = list(new_packages.keys())
-            packages = {**packages, **new_packages}
-
+            packages = transform_inspector_packages(f["packageVulnerabilityDetails"])
+            finding["vulnerablepackageids"] = list(packages.keys())
+            for package_id, package in packages.items():
+                finding_to_package_map.append(
+                    {
+                        "findingarn": finding["id"],
+                        "packageid": package_id,
+                        "remediation": package.get("remediation"),
+                        "fixedInVersion": package.get("fixedInVersion"),
+                        "filePath": package.get("filePath"),
+                        "sourceLayerHash": package.get("sourceLayerHash"),
+                        "sourceLambdaLayerArn": package.get("sourceLambdaLayerArn"),
+                    }
+                )
+                packages_set.add(frozenset(package.items()))
         findings_list.append(finding)
-    packages_list = transform_inspector_packages(packages)
-    return findings_list, packages_list
-
+    packages_list = [dict(p) for p in packages_set]
+    return findings_list, packages_list, finding_to_package_map
 
-def transform_inspector_packages(packages: Dict[str, Any]) -> List[Dict[str, Any]]:
-    packages_list: List[Dict] = []
-    for package_id in packages.keys():
-        packages_list.append(packages[package_id])
 
-    return packages_list
-
-
-def _process_packages(
+def transform_inspector_packages(
     package_details: Dict[str, Any],
-    aws_account_id: str,
-    finding_arn: str,
 ) -> Dict[str, Any]:
     packages: Dict[str, Any] = {}
     for package in package_details["vulnerablePackages"]:
-        new_package = {}
-        new_package["id"] = (
-            f"{package.get('name', '')}|"
-            f"{package.get('arch', '')}|"
-            f"{package.get('version', '')}|"
-            f"{package.get('release', '')}|"
-            f"{package.get('epoch', '')}"
-        )
-        new_package["name"] = package.get("name")
-        new_package["arch"] = package.get("arch")
-        new_package["version"] = package.get("version")
-        new_package["release"] = package.get("release")
-        new_package["epoch"] = package.get("epoch")
-        new_package["manager"] = package.get("packageManager")
-        new_package["filepath"] = package.get("filePath")
-        new_package["fixedinversion"] = package.get("fixedInVersion")
-        new_package["sourcelayerhash"] = package.get("sourceLayerHash")
-        new_package["awsaccount"] = aws_account_id
-        new_package["findingarn"] = finding_arn
-
-        packages[new_package["id"]] = new_package
+        # Following RPM package naming convention for consistency
+        name = package["name"]  # Mandatory field
+        epoch = str(package.get("epoch", ""))
+        if epoch:
+            epoch = f"{epoch}:"
+        version = package["version"]  # Mandatory field
+        release = package.get("release", "")
+        if release:
+            release = f"-{release}"
+        arch = package.get("arch", "")
+        if arch:
+            arch = f".{arch}"
+        id = f"{name}|{epoch}{version}{release}{arch}"
+        packages[id] = {**package, "id": id}
 
     return packages
 
@@ -244,7 +238,6 @@ def load_inspector_findings(
 def load_inspector_packages(
     neo4j_session: neo4j.Session,
     packages: List[Dict[str, Any]],
-    region: str,
     aws_update_tag: int,
     current_aws_account_id: str,
 ) -> None:
@@ -252,12 +245,28 @@ def load_inspector_packages(
         neo4j_session,
         AWSInspectorPackageSchema(),
         packages,
-        Region=region,
         AWS_ID=current_aws_account_id,
         lastupdated=aws_update_tag,
     )
 
 
+@timeit
+def load_inspector_finding_to_package_match_links(
+    neo4j_session: neo4j.Session,
+    finding_to_package_map: List[Dict[str, str]],
+    aws_update_tag: int,
+    current_aws_account_id: str,
+) -> None:
+    load_matchlinks(
+        neo4j_session,
+        InspectorFindingToPackageMatchLink(),
+        finding_to_package_map,
+        lastupdated=aws_update_tag,
+        _sub_resource_label="AWSAccount",
+        _sub_resource_id=current_aws_account_id,
+    )
+
+
 @timeit
 def cleanup(
     neo4j_session: neo4j.Session,
@@ -270,6 +279,14 @@ def cleanup(
     GraphJob.from_node_schema(AWSInspectorPackageSchema(), common_job_parameters).run(
         neo4j_session,
     )
+    GraphJob.from_matchlink(
+        InspectorFindingToPackageMatchLink(),
+        "AWSAccount",
+        common_job_parameters["ACCOUNT_ID"],
+        common_job_parameters["UPDATE_TAG"],
+    ).run(
+        neo4j_session,
+    )
 
 
 def _sync_findings_for_account(
@@ -288,7 +305,9 @@ def _sync_findings_for_account(
         logger.info(f"No findings to sync for account {account_id} in region {region}")
         return
     for f_batch in findings:
-        finding_data, package_data = transform_inspector_findings(f_batch)
+        finding_data, package_data, finding_to_package_map = (
+            transform_inspector_findings(f_batch)
+        )
         logger.info(f"Loading {len(finding_data)} findings from account {account_id}")
         load_inspector_findings(
             neo4j_session,
@@ -301,7 +320,15 @@ def _sync_findings_for_account(
         load_inspector_packages(
             neo4j_session,
             package_data,
-            region,
+            update_tag,
+            current_aws_account_id,
+        )
+        logger.info(
+            f"Loading {len(finding_to_package_map)} finding to package relationships"
+        )
+        load_inspector_finding_to_package_match_links(
+            neo4j_session,
+            finding_to_package_map,
             update_tag,
             current_aws_account_id,
         )
@@ -337,5 +364,7 @@ def sync(
                 update_tag,
                 current_aws_account_id,
             )
+        common_job_parameters["ACCOUNT_ID"] = current_aws_account_id
+        common_job_parameters["UPDATE_TAG"] = update_tag
 
     cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/resources.py

@@ -6,7 +6,9 @@ from cartography.intel.aws.ec2.route_tables import sync_route_tables
 from . import acm
 from . import apigateway
 from . import cloudtrail
+from . import cloudtrail_management_events
 from . import cloudwatch
+from . import codebuild
 from . import config
 from . import dynamodb
 from . import ecr
@@ -106,6 +108,8 @@ RESOURCE_FUNCTIONS: Dict[str, Callable[..., None]] = {
     "config": config.sync,
     "identitycenter": identitycenter.sync_identity_center_instances,
     "cloudtrail": cloudtrail.sync,
+    "cloudtrail_management_events": cloudtrail_management_events.sync,
     "cloudwatch": cloudwatch.sync,
     "efs": efs.sync,
+    "codebuild": codebuild.sync,
 }

cartography/intel/aws/sns.py

@@ -1,4 +1,5 @@
 import logging
+from typing import Any
 from typing import Dict
 from typing import List
 from typing import Optional
@@ -9,6 +10,7 @@ import neo4j
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
 from cartography.models.aws.sns.topic import SNSTopicSchema
+from cartography.models.aws.sns.topic_subscription import SNSTopicSubscriptionSchema
 from cartography.stats import get_stats_client
 from cartography.util import aws_handle_regions
 from cartography.util import merge_module_sync_metadata
@@ -108,6 +110,48 @@ def load_sns_topics(
     )
 
 
+@timeit
+@aws_handle_regions
+def get_subscriptions(
+    boto3_session: boto3.session.Session, region: str
+) -> List[Dict[str, Any]]:
+    """
+    Get all SNS Topics Subscriptions for a region.
+    """
+    client = boto3_session.client("sns", region_name=region)
+    paginator = client.get_paginator("list_subscriptions")
+    subscriptions = []
+    for page in paginator.paginate():
+        subscriptions.extend(page.get("Subscriptions", []))
+
+    return subscriptions
+
+
+@timeit
+def load_sns_topic_subscription(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    aws_account_id: str,
+    update_tag: int,
+) -> None:
+    """
+    Load SNS Topic Subscription information into the graph
+    """
+    logger.info(
+        f"Loading {len(data)} SNS topic subscription for region {region} into graph."
+    )
+
+    load(
+        neo4j_session,
+        SNSTopicSubscriptionSchema(),
+        data,
+        lastupdated=update_tag,
+        Region=region,
+        AWS_ID=aws_account_id,
+    )
+
+
 @timeit
 def cleanup_sns(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
     """
@@ -117,6 +161,11 @@ def cleanup_sns(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> No
     cleanup_job = GraphJob.from_node_schema(SNSTopicSchema(), common_job_parameters)
     cleanup_job.run(neo4j_session)
 
+    cleanup_job = GraphJob.from_node_schema(
+        SNSTopicSubscriptionSchema(), common_job_parameters
+    )
+    cleanup_job.run(neo4j_session)
+
 
 @timeit
 def sync(
@@ -128,7 +177,7 @@ def sync(
     common_job_parameters: Dict,
 ) -> None:
     """
-    Sync SNS Topics for all regions
+    Sync SNS Topics and Subscriptions for all regions
     """
     for region in regions:
         logger.info(
@@ -153,9 +202,20 @@ def sync(
             update_tag,
         )
 
+        # Get and load subscriptions
+        subscriptions = get_subscriptions(boto3_session, region)
+
+        load_sns_topic_subscription(
+            neo4j_session,
+            subscriptions,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    # Cleanup and metadata update (outside region loop)
     cleanup_sns(neo4j_session, common_job_parameters)
 
-    # Record that we've synced this module
     merge_module_sync_metadata(
         neo4j_session,
         group_type="AWSAccount",

cartography/intel/entra/users.py

@@ -15,6 +15,51 @@ from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
+# NOTE:
+# Microsoft Graph imposes limits on the length of the $select clause as well as
+# the number of properties that can be selected in a single request.  In
+# practice we have seen 400 Bad Request responses that bubble up as
+# `Microsoft.SharePoint.Client.InvalidClientQueryException` once that limit is
+# breached (Graph internally rewrites the next-link using a SharePoint style
+# `id in (…)` filter which is then rejected).
+#
+# To avoid tripping this bug we only request a *core* subset of user attributes
+# that are most commonly used in downstream analysis.  The transform() function
+# tolerates missing attributes (the generated MS Graph SDK simply returns
+# `None` for properties that are not present in the payload), so fetching fewer
+# fields is safe – we merely get more `null` values in the graph.
+#
+# If you need additional attributes in the future, append them here but keep the
+# total character count of the comma-separated list comfortably below 500 and
+# stay within the official v1.0 contract (beta-only fields cause similar
+# failures). 20–25 fields is a good rule-of-thumb.
+#
+# References:
+#   • https://learn.microsoft.com/graph/query-parameters#select-parameter
+#   • https://learn.microsoft.com/graph/api/user-list?view=graph-rest-1.0
+#
+USER_SELECT_FIELDS = [
+    "id",
+    "userPrincipalName",
+    "displayName",
+    "givenName",
+    "surname",
+    "mail",
+    "mobilePhone",
+    "businessPhones",
+    "jobTitle",
+    "department",
+    "officeLocation",
+    "city",
+    "country",
+    "companyName",
+    "preferredLanguage",
+    "employeeId",
+    "employeeType",
+    "accountEnabled",
+    "ageGroup",
+]
+
 
 @timeit
 async def get_tenant(client: GraphServiceClient) -> Organization:
@@ -27,14 +72,20 @@ async def get_tenant(client: GraphServiceClient) -> Organization:
 
 @timeit
 async def get_users(client: GraphServiceClient) -> list[User]:
+    """Fetch all users with their manager reference in as few requests as possible.
+
+    We leverage `$expand=manager($select=id)` so the manager's *id* is hydrated
+    alongside every user record.  This avoids making a second round-trip per
+    user – vastly reducing latency and eliminating the noisy 404s that occur
+    when a user has no manager assigned.
     """
-    Get all users from Microsoft Graph API with pagination support
-    """
+
     all_users: list[User] = []
     request_configuration = client.users.UsersRequestBuilderGetRequestConfiguration(
         query_parameters=client.users.UsersRequestBuilderGetQueryParameters(
-            # Request more items per page to reduce number of API calls
             top=999,
+            select=USER_SELECT_FIELDS,
+            expand=["manager($select=id)"],
         ),
     )
 
@@ -43,18 +94,32 @@ async def get_users(client: GraphServiceClient) -> list[User]:
         all_users.extend(page.value)
         if not page.odata_next_link:
             break
-        page = await client.users.with_url(page.odata_next_link).get()
+
+        try:
+            page = await client.users.with_url(page.odata_next_link).get()
+        except Exception as e:
+            logger.error(
+                "Failed to fetch next page of Entra ID users – stopping pagination early: %s",
+                e,
+            )
+            break
 
     return all_users
 
 
 @timeit
+# The manager reference is now embedded in the user objects courtesy of the
+# `$expand` we added above, so we no longer need a separate `manager_map`.
 def transform_users(users: list[User]) -> list[dict[str, Any]]:
-    """
-    Transform the API response into the format expected by our schema
-    """
+    """Convert MS Graph SDK `User` models into dicts matching our schema."""
+
     result: list[dict[str, Any]] = []
     for user in users:
+        manager_id: str | None = None
+        if getattr(user, "manager", None) is not None:
+            # The SDK materialises `manager` as a DirectoryObject (or subclass)
+            manager_id = getattr(user.manager, "id", None)
+
         transformed_user = {
             "id": user.id,
             "user_principal_name": user.user_principal_name,
@@ -62,47 +127,24 @@ def transform_users(users: list[User]) -> list[dict[str, Any]]:
             "given_name": user.given_name,
             "surname": user.surname,
             "mail": user.mail,
-            "other_mails": user.other_mails,
-            "preferred_language": user.preferred_language,
-            "preferred_name": user.preferred_name,
-            "state": user.state,
-            "usage_location": user.usage_location,
-            "user_type": user.user_type,
-            "show_in_address_list": user.show_in_address_list,
-            "sign_in_sessions_valid_from_date_time": user.sign_in_sessions_valid_from_date_time,
-            "security_identifier": user.on_premises_security_identifier,
-            "account_enabled": user.account_enabled,
-            "age_group": user.age_group,
+            "mobile_phone": user.mobile_phone,
             "business_phones": user.business_phones,
+            "job_title": user.job_title,
+            "department": user.department,
+            "office_location": user.office_location,
             "city": user.city,
-            "company_name": user.company_name,
-            "consent_provided_for_minor": user.consent_provided_for_minor,
+            "state": user.state,
             "country": user.country,
-            "created_date_time": user.created_date_time,
-            "creation_type": user.creation_type,
-            "deleted_date_time": user.deleted_date_time,
-            "department": user.department,
+            "company_name": user.company_name,
+            "preferred_language": user.preferred_language,
             "employee_id": user.employee_id,
             "employee_type": user.employee_type,
-            "external_user_state": user.external_user_state,
-            "external_user_state_change_date_time": user.external_user_state_change_date_time,
-            "hire_date": user.hire_date,
-            "is_management_restricted": user.is_management_restricted,
-            "is_resource_account": user.is_resource_account,
-            "job_title": user.job_title,
-            "last_password_change_date_time": user.last_password_change_date_time,
-            "mail_nickname": user.mail_nickname,
-            "office_location": user.office_location,
-            "on_premises_distinguished_name": user.on_premises_distinguished_name,
-            "on_premises_domain_name": user.on_premises_domain_name,
-            "on_premises_immutable_id": user.on_premises_immutable_id,
-            "on_premises_last_sync_date_time": user.on_premises_last_sync_date_time,
-            "on_premises_sam_account_name": user.on_premises_sam_account_name,
-            "on_premises_security_identifier": user.on_premises_security_identifier,
-            "on_premises_sync_enabled": user.on_premises_sync_enabled,
-            "on_premises_user_principal_name": user.on_premises_user_principal_name,
+            "account_enabled": user.account_enabled,
+            "age_group": user.age_group,
+            "manager_id": manager_id,
         }
         result.append(transformed_user)
+
     return result
 
 
@@ -198,7 +240,7 @@ async def sync_entra_users(
         credential, scopes=["https://graph.microsoft.com/.default"]
     )
 
-    # Get tenant information
+    # Fetch tenant and users (with manager reference already populated by `$expand`)
     tenant = await get_tenant(client)
     users = await get_users(client)
 

cartography/intel/scaleway/__init__.py

@@ -0,0 +1,127 @@
+import logging
+
+import neo4j
+import scaleway
+
+import cartography.intel.scaleway.iam.apikeys
+import cartography.intel.scaleway.iam.applications
+import cartography.intel.scaleway.iam.groups
+import cartography.intel.scaleway.iam.users
+import cartography.intel.scaleway.instances.flexibleips
+import cartography.intel.scaleway.instances.instances
+import cartography.intel.scaleway.projects
+import cartography.intel.scaleway.storage.snapshots
+import cartography.intel.scaleway.storage.volumes
+from cartography.config import Config
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def start_scaleway_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of Scaleway data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+
+    if (
+        not config.scaleway_access_key
+        or not config.scaleway_secret_key
+        or not config.scaleway_org
+    ):
+        logger.info(
+            "Tailscale import is not configured - skipping this module. "
+            "See docs to configure.",
+        )
+        return
+
+    # Create client
+    client = scaleway.Client(
+        access_key=config.scaleway_access_key,
+        secret_key=config.scaleway_secret_key,
+    )
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+        "ORG_ID": config.scaleway_org,
+    }
+
+    # Organization level
+    projects = cartography.intel.scaleway.projects.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        update_tag=config.update_tag,
+    )
+    projects_id = [project["id"] for project in projects]
+    cartography.intel.scaleway.iam.users.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        update_tag=config.update_tag,
+    )
+    cartography.intel.scaleway.iam.applications.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        update_tag=config.update_tag,
+    )
+    cartography.intel.scaleway.iam.groups.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        update_tag=config.update_tag,
+    )
+    cartography.intel.scaleway.iam.apikeys.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        update_tag=config.update_tag,
+    )
+
+    # Storage
+    cartography.intel.scaleway.storage.volumes.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        projects_id=projects_id,
+        update_tag=config.update_tag,
+    )
+    cartography.intel.scaleway.storage.snapshots.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        projects_id=projects_id,
+        update_tag=config.update_tag,
+    )
+
+    # Instances
+    # DISABLED due to https://github.com/scaleway/scaleway-sdk-python/issues/1040
+    """
+    cartography.intel.scaleway.instances.flexibleips.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        projects_id=projects_id,
+        update_tag=config.update_tag,
+    )
+    """
+    cartography.intel.scaleway.instances.instances.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        projects_id=projects_id,
+        update_tag=config.update_tag,
+    )