cartography 0.108.0rc1__py3-none-any.whl → 0.109.0__py3-none-any.whl

cartography/intel/aws/secretsmanager.py

@@ -7,6 +7,7 @@ import neo4j
 
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
+from cartography.models.aws.secretsmanager.secret import SecretsManagerSecretSchema
 from cartography.models.aws.secretsmanager.secret_version import (
     SecretsManagerSecretVersionSchema,
 )
@@ -14,7 +15,6 @@ from cartography.stats import get_stats_client
 from cartography.util import aws_handle_regions
 from cartography.util import dict_date_to_epoch
 from cartography.util import merge_module_sync_metadata
-from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -32,6 +32,37 @@ def get_secret_list(boto3_session: boto3.session.Session, region: str) -> List[D
     return secrets
 
 
+def transform_secrets(
+    secrets: List[Dict],
+) -> List[Dict]:
+    """
+    Transform AWS Secrets Manager Secrets to match the data model.
+    """
+    transformed_data = []
+    for secret in secrets:
+        # Start with a copy of the original secret data
+        transformed = dict(secret)
+
+        # Convert date fields to epoch timestamps
+        transformed["CreatedDate"] = dict_date_to_epoch(secret, "CreatedDate")
+        transformed["LastRotatedDate"] = dict_date_to_epoch(secret, "LastRotatedDate")
+        transformed["LastChangedDate"] = dict_date_to_epoch(secret, "LastChangedDate")
+        transformed["LastAccessedDate"] = dict_date_to_epoch(secret, "LastAccessedDate")
+        transformed["DeletedDate"] = dict_date_to_epoch(secret, "DeletedDate")
+
+        # Flatten nested RotationRules.AutomaticallyAfterDays property
+        if "RotationRules" in secret and secret["RotationRules"]:
+            rotation_rules = secret["RotationRules"]
+            if "AutomaticallyAfterDays" in rotation_rules:
+                transformed["RotationRulesAutomaticallyAfterDays"] = rotation_rules[
+                    "AutomaticallyAfterDays"
+                ]
+
+        transformed_data.append(transformed)
+
+    return transformed_data
+
+
 @timeit
 def load_secrets(
     neo4j_session: neo4j.Session,
@@ -40,48 +71,33 @@ def load_secrets(
     current_aws_account_id: str,
     aws_update_tag: int,
 ) -> None:
-    ingest_secrets = """
-    UNWIND $Secrets as secret
-        MERGE (s:SecretsManagerSecret{id: secret.ARN})
-        ON CREATE SET s.firstseen = timestamp()
-        SET s.name = secret.Name, s.arn = secret.ARN, s.description = secret.Description,
-            s.kms_key_id = secret.KmsKeyId, s.rotation_enabled = secret.RotationEnabled,
-            s.rotation_lambda_arn = secret.RotationLambdaARN,
-            s.rotation_rules_automatically_after_days = secret.RotationRules.AutomaticallyAfterDays,
-            s.last_rotated_date = secret.LastRotatedDate, s.last_changed_date = secret.LastChangedDate,
-            s.last_accessed_date = secret.LastAccessedDate, s.deleted_date = secret.DeletedDate,
-            s.owning_service = secret.OwningService, s.created_date = secret.CreatedDate,
-            s.primary_region = secret.PrimaryRegion, s.region = $Region,
-            s.lastupdated = $aws_update_tag
-        WITH s
-        MATCH (owner:AWSAccount{id: $AWS_ACCOUNT_ID})
-        MERGE (owner)-[r:RESOURCE]->(s)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $aws_update_tag
-    """
-    for secret in data:
-        secret["LastRotatedDate"] = dict_date_to_epoch(secret, "LastRotatedDate")
-        secret["LastChangedDate"] = dict_date_to_epoch(secret, "LastChangedDate")
-        secret["LastAccessedDate"] = dict_date_to_epoch(secret, "LastAccessedDate")
-        secret["DeletedDate"] = dict_date_to_epoch(secret, "DeletedDate")
-        secret["CreatedDate"] = dict_date_to_epoch(secret, "CreatedDate")
-
-    neo4j_session.run(
-        ingest_secrets,
-        Secrets=data,
+    """
+    Load transformed secrets into Neo4j using the data model.
+    Expects data to already be transformed by transform_secrets().
+    """
+    logger.info(f"Loading {len(data)} Secrets for region {region} into graph.")
+
+    # Load using the schema-based approach
+    load(
+        neo4j_session,
+        SecretsManagerSecretSchema(),
+        data,
+        lastupdated=aws_update_tag,
         Region=region,
-        AWS_ACCOUNT_ID=current_aws_account_id,
-        aws_update_tag=aws_update_tag,
+        AWS_ID=current_aws_account_id,
     )
 
 
 @timeit
 def cleanup_secrets(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
-    run_cleanup_job(
-        "aws_import_secrets_cleanup.json",
-        neo4j_session,
-        common_job_parameters,
+    """
+    Run Secrets cleanup job using the data model.
+    """
+    logger.debug("Running Secrets cleanup job.")
+    cleanup_job = GraphJob.from_node_schema(
+        SecretsManagerSecretSchema(), common_job_parameters
     )
+    cleanup_job.run(neo4j_session)
 
 
 @timeit
@@ -121,8 +137,6 @@ def get_secret_versions(
 
 def transform_secret_versions(
     versions: List[Dict],
-    region: str,
-    aws_account_id: str,
 ) -> List[Dict]:
     """
     Transform AWS Secrets Manager Secret Versions to match the data model.
@@ -203,7 +217,15 @@ def sync(
         )
         secrets = get_secret_list(boto3_session, region)
 
-        load_secrets(neo4j_session, secrets, region, current_aws_account_id, update_tag)
+        transformed_secrets = transform_secrets(secrets)
+
+        load_secrets(
+            neo4j_session,
+            transformed_secrets,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
 
         all_versions = []
         for secret in secrets:
@@ -216,11 +238,7 @@ def sync(
             )
             all_versions.extend(versions)
 
-        transformed_data = transform_secret_versions(
-            all_versions,
-            region,
-            current_aws_account_id,
-        )
+        transformed_data = transform_secret_versions(all_versions)
 
         load_secret_versions(
             neo4j_session,

cartography/intel/entra/groups.py

@@ -59,10 +59,29 @@ async def get_group_members(
     return user_ids, group_ids
 
 
+@timeit
+async def get_group_owners(client: GraphServiceClient, group_id: str) -> list[str]:
+    """Get owner user IDs for a given group."""
+    owner_ids: list[str] = []
+    request_builder = client.groups.by_group_id(group_id).owners
+    page = await request_builder.get()
+    while page:
+        if page.value:
+            for obj in page.value:
+                odata_type = getattr(obj, "odata_type", "")
+                if odata_type == "#microsoft.graph.user":
+                    owner_ids.append(obj.id)
+        if not page.odata_next_link:
+            break
+        page = await request_builder.with_url(page.odata_next_link).get()
+    return owner_ids
+
+
 def transform_groups(
     groups: list[Group],
     user_member_map: dict[str, list[str]],
     group_member_map: dict[str, list[str]],
+    group_owner_map: dict[str, list[str]],
 ) -> list[dict[str, Any]]:
     """Transform API responses into dictionaries for ingestion."""
     result: list[dict[str, Any]] = []
@@ -82,6 +101,7 @@ def transform_groups(
             "deleted_date_time": g.deleted_date_time,
             "member_ids": user_member_map.get(g.id, []),
             "member_group_ids": group_member_map.get(g.id, []),
+            "owner_ids": group_owner_map.get(g.id, []),
         }
         result.append(transformed)
     return result
@@ -134,6 +154,12 @@ async def sync_entra_groups(
 
     user_member_map: dict[str, list[str]] = {}
     group_member_map: dict[str, list[str]] = {}
+    group_owner_map: dict[str, list[str]] = {}
+
+    for group in groups:
+        owners = await get_group_owners(client, group.id)
+        group_owner_map[group.id] = owners
+
     for group in groups:
         try:
             users, subgroups = await get_group_members(client, group.id)
@@ -144,7 +170,9 @@ async def sync_entra_groups(
             user_member_map[group.id] = []
             group_member_map[group.id] = []
 
-    transformed_groups = transform_groups(groups, user_member_map, group_member_map)
+    transformed_groups = transform_groups(
+        groups, user_member_map, group_member_map, group_owner_map
+    )
 
     load_tenant(neo4j_session, {"id": tenant_id}, update_tag)
     load_groups(neo4j_session, transformed_groups, update_tag, tenant_id)

cartography/intel/gcp/__init__.py

@@ -391,6 +391,7 @@ def _sync_multiple_projects(
     # Compute data sync
     for project in projects:
         project_id = project["projectId"]
+        common_job_parameters["PROJECT_ID"] = project_id
         logger.info("Syncing GCP project %s for Compute.", project_id)
         _sync_single_project_compute(
             neo4j_session,
@@ -399,10 +400,12 @@ def _sync_multiple_projects(
             gcp_update_tag,
             common_job_parameters,
         )
+        del common_job_parameters["PROJECT_ID"]
 
     # Storage data sync
     for project in projects:
         project_id = project["projectId"]
+        common_job_parameters["PROJECT_ID"] = project_id
         logger.info("Syncing GCP project %s for Storage", project_id)
         _sync_single_project_storage(
             neo4j_session,
@@ -411,10 +414,12 @@ def _sync_multiple_projects(
             gcp_update_tag,
             common_job_parameters,
         )
+        del common_job_parameters["PROJECT_ID"]
 
     # GKE data sync
     for project in projects:
         project_id = project["projectId"]
+        common_job_parameters["PROJECT_ID"] = project_id
         logger.info("Syncing GCP project %s for GKE", project_id)
         _sync_single_project_gke(
             neo4j_session,
@@ -423,10 +428,12 @@ def _sync_multiple_projects(
             gcp_update_tag,
             common_job_parameters,
         )
+        del common_job_parameters["PROJECT_ID"]
 
     # DNS data sync
     for project in projects:
         project_id = project["projectId"]
+        common_job_parameters["PROJECT_ID"] = project_id
         logger.info("Syncing GCP project %s for DNS", project_id)
         _sync_single_project_dns(
             neo4j_session,
@@ -435,14 +442,17 @@ def _sync_multiple_projects(
             gcp_update_tag,
             common_job_parameters,
         )
+        del common_job_parameters["PROJECT_ID"]
 
     # IAM data sync
     for project in projects:
         project_id = project["projectId"]
+        common_job_parameters["PROJECT_ID"] = project_id
         logger.info("Syncing GCP project %s for IAM", project_id)
         _sync_single_project_iam(
             neo4j_session, resources, project_id, gcp_update_tag, common_job_parameters
         )
+        del common_job_parameters["PROJECT_ID"]
 
 
 @timeit

cartography/intel/gcp/compute.py

@@ -14,6 +14,9 @@ import neo4j
 from googleapiclient.discovery import HttpError
 from googleapiclient.discovery import Resource
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.gcp.compute.vpc import GCPVpcSchema
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
@@ -600,48 +603,17 @@ def load_gcp_instances(
 @timeit
 def load_gcp_vpcs(
     neo4j_session: neo4j.Session,
-    vpcs: List[Dict],
+    vpcs: list[dict[str, Any]],
     gcp_update_tag: int,
+    project_id: str,
 ) -> None:
-    """
-    Ingest VPCs to Neo4j
-    :param neo4j_session: The Neo4j session object
-    :param vpcs: List of VPCs to ingest
-    :param gcp_update_tag: The timestamp value to set our new Neo4j nodes with
-    :return: Nothing
-    """
-    query = """
-    MERGE(p:GCPProject{id:$ProjectId})
-    ON CREATE SET p.firstseen = timestamp()
-    SET p.lastupdated = $gcp_update_tag
-
-    MERGE(vpc:GCPVpc{id:$PartialUri})
-    ON CREATE SET vpc.firstseen = timestamp(),
-    vpc.partial_uri = $PartialUri
-    SET vpc.self_link = $SelfLink,
-    vpc.name = $VpcName,
-    vpc.project_id = $ProjectId,
-    vpc.auto_create_subnetworks = $AutoCreateSubnetworks,
-    vpc.routing_config_routing_mode = $RoutingMode,
-    vpc.description = $Description,
-    vpc.lastupdated = $gcp_update_tag
-
-    MERGE (p)-[r:RESOURCE]->(vpc)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $gcp_update_tag
-    """
-    for vpc in vpcs:
-        neo4j_session.run(
-            query,
-            ProjectId=vpc["project_id"],
-            PartialUri=vpc["partial_uri"],
-            SelfLink=vpc["self_link"],
-            VpcName=vpc["name"],
-            AutoCreateSubnetworks=vpc["auto_create_subnetworks"],
-            RoutingMode=vpc["routing_config_routing_mode"],
-            Description=vpc["description"],
-            gcp_update_tag=gcp_update_tag,
-        )
+    load(
+        neo4j_session,
+        GCPVpcSchema(),
+        vpcs,
+        PROJECT_ID=project_id,
+        LASTUPDATED=gcp_update_tag,
+    )
 
 
 @timeit
@@ -1159,6 +1131,12 @@ def cleanup_gcp_vpcs(neo4j_session: neo4j.Session, common_job_parameters: Dict)
     :param common_job_parameters: dict of other job parameters to pass to Neo4j
     :return: Nothing
     """
+    GraphJob.from_node_schema(
+        GCPVpcSchema(),
+        common_job_parameters,
+    ).run(neo4j_session)
+
+    # TODO: remove this once we refactor GCP instances and add the instance to vpc rel as an object
     run_cleanup_job(
         "gcp_compute_vpc_cleanup.json",
         neo4j_session,
@@ -1267,8 +1245,7 @@ def sync_gcp_vpcs(
     """
     vpc_res = get_gcp_vpcs(project_id, compute)
     vpcs = transform_gcp_vpcs(vpc_res)
-    load_gcp_vpcs(neo4j_session, vpcs, gcp_update_tag)
-    # TODO scope the cleanup to the current project - https://github.com/cartography-cncf/cartography/issues/381
+    load_gcp_vpcs(neo4j_session, vpcs, gcp_update_tag, project_id)
     cleanup_gcp_vpcs(neo4j_session, common_job_parameters)
 
 

cartography/intel/trivy/__init__.py

@@ -1,3 +1,4 @@
+import json
 import logging
 from typing import Any
 
@@ -8,7 +9,9 @@ from cartography.client.aws import list_accounts
 from cartography.client.aws.ecr import get_ecr_images
 from cartography.config import Config
 from cartography.intel.trivy.scanner import cleanup
+from cartography.intel.trivy.scanner import get_json_files_in_dir
 from cartography.intel.trivy.scanner import get_json_files_in_s3
+from cartography.intel.trivy.scanner import sync_single_image_from_file
 from cartography.intel.trivy.scanner import sync_single_image_from_s3
 from cartography.stats import get_stats_client
 from cartography.util import timeit
@@ -39,13 +42,13 @@ def get_scan_targets(
 
 
 def _get_intersection(
-    images_in_graph: set[str], json_files: set[str], trivy_s3_prefix: str
+    image_uris: set[str], json_files: set[str], trivy_s3_prefix: str
 ) -> list[tuple[str, str]]:
     """
     Get the intersection of ECR images in the graph and S3 scan results.
 
     Args:
-        images_in_graph: Set of ECR images in the graph
+        image_uris: Set of ECR images in the graph
         json_files: Set of S3 object keys for JSON files
         trivy_s3_prefix: S3 prefix path containing scan results
 
@@ -60,7 +63,7 @@ def _get_intersection(
         # Remove the prefix and the .json suffix
         image_uri = s3_object_key[prefix_len:-5]
 
-        if image_uri in images_in_graph:
+        if image_uri in image_uris:
             intersection.append((image_uri, s3_object_key))
 
     return intersection
@@ -90,12 +93,12 @@ def sync_trivy_aws_ecr_from_s3(
         f"Using Trivy scan results from s3://{trivy_s3_bucket}/{trivy_s3_prefix}"
     )
 
-    images_in_graph: set[str] = get_scan_targets(neo4j_session)
+    image_uris: set[str] = get_scan_targets(neo4j_session)
     json_files: set[str] = get_json_files_in_s3(
         trivy_s3_bucket, trivy_s3_prefix, boto3_session
     )
     intersection: list[tuple[str, str]] = _get_intersection(
-        images_in_graph, json_files, trivy_s3_prefix
+        image_uris, json_files, trivy_s3_prefix
     )
 
     if len(intersection) == 0:
@@ -124,21 +127,79 @@ def sync_trivy_aws_ecr_from_s3(
     cleanup(neo4j_session, common_job_parameters)
 
 
+@timeit
+def sync_trivy_aws_ecr_from_dir(
+    neo4j_session: Session,
+    results_dir: str,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    """Sync Trivy scan results from local files for AWS ECR images."""
+    logger.info(f"Using Trivy scan results from {results_dir}")
+
+    image_uris: set[str] = get_scan_targets(neo4j_session)
+    json_files: set[str] = get_json_files_in_dir(results_dir)
+
+    if not json_files:
+        logger.error(
+            f"Trivy sync was configured, but no json files were found in {results_dir}."
+        )
+        raise ValueError("No Trivy json results found on disk")
+
+    logger.info(f"Processing {len(json_files)} local Trivy result files")
+
+    for file_path in json_files:
+        # First, check if the image exists in the graph before syncing
+        try:
+            # Peek at the artifact name without processing the file
+            with open(file_path, encoding="utf-8") as f:
+                trivy_data = json.load(f)
+                artifact_name = trivy_data.get("ArtifactName")
+
+            if artifact_name and artifact_name not in image_uris:
+                logger.debug(
+                    f"Skipping results for {artifact_name} since the image is not present in the graph"
+                )
+                continue
+
+        except (json.JSONDecodeError, KeyError) as e:
+            logger.error(f"Failed to read artifact name from {file_path}: {e}")
+            continue
+
+        # Now sync the file since we know the image exists in the graph
+        sync_single_image_from_file(
+            neo4j_session,
+            file_path,
+            update_tag,
+        )
+
+    cleanup(neo4j_session, common_job_parameters)
+
+
 @timeit
 def start_trivy_ingestion(neo4j_session: Session, config: Config) -> None:
-    """
-    Start Trivy scan ingestion from S3.
+    """Start Trivy scan ingestion from S3 or local files.
 
     Args:
         neo4j_session: Neo4j session for database operations
-        config: Configuration object containing S3 settings
+        config: Configuration object containing S3 or directory paths
     """
-    # Check if S3 configuration is provided
-    if not config.trivy_s3_bucket:
-        logger.info("Trivy S3 configuration not provided. Skipping Trivy ingestion.")
+    if not config.trivy_s3_bucket and not config.trivy_results_dir:
+        logger.info("Trivy configuration not provided. Skipping Trivy ingestion.")
+        return
+
+    if config.trivy_results_dir:
+        common_job_parameters = {
+            "UPDATE_TAG": config.update_tag,
+        }
+        sync_trivy_aws_ecr_from_dir(
+            neo4j_session,
+            config.trivy_results_dir,
+            config.update_tag,
+            common_job_parameters,
+        )
         return
 
-    # Default to empty string if s3 prefix is not provided
     if config.trivy_s3_prefix is None:
         config.trivy_s3_prefix = ""
 
@@ -146,7 +207,6 @@ def start_trivy_ingestion(neo4j_session: Session, config: Config) -> None:
         "UPDATE_TAG": config.update_tag,
     }
 
-    # Get ECR images to scan
     boto3_session = boto3.Session()
 
     sync_trivy_aws_ecr_from_s3(