cartography 0.109.0rc1__py3-none-any.whl → 0.110.0rc1__py3-none-any.whl

cartography/intel/entra/__init__.py

@@ -1,13 +1,14 @@
 import asyncio
+import datetime
 import logging
+from traceback import TracebackException
+from typing import Awaitable
+from typing import Callable
 
 import neo4j
 
 from cartography.config import Config
-from cartography.intel.entra.applications import sync_entra_applications
-from cartography.intel.entra.groups import sync_entra_groups
-from cartography.intel.entra.ou import sync_entra_ous
-from cartography.intel.entra.users import sync_entra_users
+from cartography.intel.entra.resources import RESOURCE_FUNCTIONS
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -39,45 +40,46 @@ def start_entra_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
     }
 
     async def main() -> None:
-        # Run user sync
-        await sync_entra_users(
-            neo4j_session,
-            config.entra_tenant_id,
-            config.entra_client_id,
-            config.entra_client_secret,
-            config.update_tag,
-            common_job_parameters,
-        )
+        failed_stages = []
+        exception_tracebacks = []
 
-        # Run group sync
-        await sync_entra_groups(
-            neo4j_session,
-            config.entra_tenant_id,
-            config.entra_client_id,
-            config.entra_client_secret,
-            config.update_tag,
-            common_job_parameters,
-        )
+        async def run_stage(name: str, func: Callable[..., Awaitable[None]]) -> None:
+            try:
+                await func(
+                    neo4j_session,
+                    config.entra_tenant_id,
+                    config.entra_client_id,
+                    config.entra_client_secret,
+                    config.update_tag,
+                    common_job_parameters,
+                )
+            except Exception as e:
+                if config.entra_best_effort_mode:
+                    timestamp = datetime.datetime.now()
+                    failed_stages.append(name)
+                    exception_traceback = TracebackException.from_exception(e)
+                    traceback_string = "".join(exception_traceback.format())
+                    exception_tracebacks.append(
+                        f"{timestamp} - Exception for stage {name}\n{traceback_string}"
+                    )
+                    logger.warning(
+                        f"Caught exception syncing {name}. entra-best-effort-mode is on so we are continuing "
+                        "on to the next Entra sync. All exceptions will be aggregated and re-logged at the end of the sync.",
+                        exc_info=True,
+                    )
+                else:
+                    logger.error("Error during Entra sync", exc_info=True)
+                    raise
 
-        # Run OU sync
-        await sync_entra_ous(
-            neo4j_session,
-            config.entra_tenant_id,
-            config.entra_client_id,
-            config.entra_client_secret,
-            config.update_tag,
-            common_job_parameters,
-        )
+        for name, func in RESOURCE_FUNCTIONS:
+            await run_stage(name, func)
 
-        # Run application sync
-        await sync_entra_applications(
-            neo4j_session,
-            config.entra_tenant_id,
-            config.entra_client_id,
-            config.entra_client_secret,
-            config.update_tag,
-            common_job_parameters,
-        )
+        if failed_stages:
+            logger.error(
+                f"Entra sync failed for the following stages: {', '.join(failed_stages)}. "
+                "See the logs for more details.",
+            )
+            raise Exception("\n".join(exception_tracebacks))
 
-    # Execute both syncs in sequence
+    # Execute all syncs in sequence
     asyncio.run(main())

cartography/intel/entra/applications.py

@@ -172,12 +172,11 @@ async def get_app_role_assignments(
             )
             continue
         except Exception as e:
-            # Only catch truly unexpected errors - these should be rare
             logger.error(
                 f"Unexpected error when fetching app role assignments for application {app.app_id} ({app.display_name}): {e}",
                 exc_info=True,
             )
-            continue
+            raise
 
     logger.info(f"Retrieved {len(assignments)} app role assignments total")
     return assignments

cartography/intel/entra/ou.py

@@ -43,7 +43,7 @@ async def get_entra_ous(client: GraphServiceClient) -> list[AdministrativeUnit]:
                 current_request = None
         except Exception as e:
             logger.error(f"Failed to retrieve administrative units: {str(e)}")
-            current_request = None
+            raise
 
     return all_units
 

cartography/intel/entra/resources.py

@@ -0,0 +1,20 @@
+from cartography.intel.entra.applications import sync_entra_applications
+from cartography.intel.entra.groups import sync_entra_groups
+from cartography.intel.entra.ou import sync_entra_ous
+from cartography.intel.entra.users import sync_entra_users
+
+# This is a list so that we sync these resources in order.
+# Data shape: [("resource_name", sync_function), ...]
+# Each sync function will be called with the following arguments:
+#   - neo4j_session
+#   - config.entra_tenant_id
+#   - config.entra_client_id
+#   - config.entra_client_secret
+#   - config.update_tag
+#   - common_job_parameters
+RESOURCE_FUNCTIONS = [
+    ("users", sync_entra_users),
+    ("groups", sync_entra_groups),
+    ("ous", sync_entra_ous),
+    ("applications", sync_entra_applications),
+]

cartography/intel/trivy/__init__.py

@@ -1,3 +1,4 @@
+import json
 import logging
 from typing import Any
 
@@ -8,7 +9,9 @@ from cartography.client.aws import list_accounts
 from cartography.client.aws.ecr import get_ecr_images
 from cartography.config import Config
 from cartography.intel.trivy.scanner import cleanup
+from cartography.intel.trivy.scanner import get_json_files_in_dir
 from cartography.intel.trivy.scanner import get_json_files_in_s3
+from cartography.intel.trivy.scanner import sync_single_image_from_file
 from cartography.intel.trivy.scanner import sync_single_image_from_s3
 from cartography.stats import get_stats_client
 from cartography.util import timeit
@@ -39,13 +42,13 @@ def get_scan_targets(
 
 
 def _get_intersection(
-    images_in_graph: set[str], json_files: set[str], trivy_s3_prefix: str
+    image_uris: set[str], json_files: set[str], trivy_s3_prefix: str
 ) -> list[tuple[str, str]]:
     """
     Get the intersection of ECR images in the graph and S3 scan results.
 
     Args:
-        images_in_graph: Set of ECR images in the graph
+        image_uris: Set of ECR images in the graph
         json_files: Set of S3 object keys for JSON files
         trivy_s3_prefix: S3 prefix path containing scan results
 
@@ -60,7 +63,7 @@ def _get_intersection(
         # Remove the prefix and the .json suffix
         image_uri = s3_object_key[prefix_len:-5]
 
-        if image_uri in images_in_graph:
+        if image_uri in image_uris:
             intersection.append((image_uri, s3_object_key))
 
     return intersection
@@ -90,12 +93,12 @@ def sync_trivy_aws_ecr_from_s3(
         f"Using Trivy scan results from s3://{trivy_s3_bucket}/{trivy_s3_prefix}"
     )
 
-    images_in_graph: set[str] = get_scan_targets(neo4j_session)
+    image_uris: set[str] = get_scan_targets(neo4j_session)
     json_files: set[str] = get_json_files_in_s3(
         trivy_s3_bucket, trivy_s3_prefix, boto3_session
     )
     intersection: list[tuple[str, str]] = _get_intersection(
-        images_in_graph, json_files, trivy_s3_prefix
+        image_uris, json_files, trivy_s3_prefix
     )
 
     if len(intersection) == 0:
@@ -124,21 +127,79 @@ def sync_trivy_aws_ecr_from_s3(
     cleanup(neo4j_session, common_job_parameters)
 
 
+@timeit
+def sync_trivy_aws_ecr_from_dir(
+    neo4j_session: Session,
+    results_dir: str,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    """Sync Trivy scan results from local files for AWS ECR images."""
+    logger.info(f"Using Trivy scan results from {results_dir}")
+
+    image_uris: set[str] = get_scan_targets(neo4j_session)
+    json_files: set[str] = get_json_files_in_dir(results_dir)
+
+    if not json_files:
+        logger.error(
+            f"Trivy sync was configured, but no json files were found in {results_dir}."
+        )
+        raise ValueError("No Trivy json results found on disk")
+
+    logger.info(f"Processing {len(json_files)} local Trivy result files")
+
+    for file_path in json_files:
+        # First, check if the image exists in the graph before syncing
+        try:
+            # Peek at the artifact name without processing the file
+            with open(file_path, encoding="utf-8") as f:
+                trivy_data = json.load(f)
+                artifact_name = trivy_data.get("ArtifactName")
+
+            if artifact_name and artifact_name not in image_uris:
+                logger.debug(
+                    f"Skipping results for {artifact_name} since the image is not present in the graph"
+                )
+                continue
+
+        except (json.JSONDecodeError, KeyError) as e:
+            logger.error(f"Failed to read artifact name from {file_path}: {e}")
+            continue
+
+        # Now sync the file since we know the image exists in the graph
+        sync_single_image_from_file(
+            neo4j_session,
+            file_path,
+            update_tag,
+        )
+
+    cleanup(neo4j_session, common_job_parameters)
+
+
 @timeit
 def start_trivy_ingestion(neo4j_session: Session, config: Config) -> None:
-    """
-    Start Trivy scan ingestion from S3.
+    """Start Trivy scan ingestion from S3 or local files.
 
     Args:
         neo4j_session: Neo4j session for database operations
-        config: Configuration object containing S3 settings
+        config: Configuration object containing S3 or directory paths
     """
-    # Check if S3 configuration is provided
-    if not config.trivy_s3_bucket:
-        logger.info("Trivy S3 configuration not provided. Skipping Trivy ingestion.")
+    if not config.trivy_s3_bucket and not config.trivy_results_dir:
+        logger.info("Trivy configuration not provided. Skipping Trivy ingestion.")
+        return
+
+    if config.trivy_results_dir:
+        common_job_parameters = {
+            "UPDATE_TAG": config.update_tag,
+        }
+        sync_trivy_aws_ecr_from_dir(
+            neo4j_session,
+            config.trivy_results_dir,
+            config.update_tag,
+            common_job_parameters,
+        )
         return
 
-    # Default to empty string if s3 prefix is not provided
     if config.trivy_s3_prefix is None:
         config.trivy_s3_prefix = ""
 
@@ -146,7 +207,6 @@ def start_trivy_ingestion(neo4j_session: Session, config: Config) -> None:
         "UPDATE_TAG": config.update_tag,
     }
 
-    # Get ECR images to scan
     boto3_session = boto3.Session()
 
     sync_trivy_aws_ecr_from_s3(

cartography/intel/trivy/scanner.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import os
 from typing import Any
 
 import boto3
@@ -127,6 +128,90 @@ def transform_scan_results(
     return findings_list, packages_list, fixes_list
 
 
+def _parse_trivy_data(
+    trivy_data: dict, source: str
+) -> tuple[str | None, list[dict], str]:
+    """
+    Parse Trivy scan data and extract common fields.
+
+    Args:
+        trivy_data: Raw JSON Trivy data
+        source: Source identifier for error messages (file path or S3 URI)
+
+    Returns:
+        Tuple of (artifact_name, results, image_digest)
+    """
+    # Extract artifact name if present (only for file-based)
+    artifact_name = trivy_data.get("ArtifactName")
+
+    if "Results" not in trivy_data:
+        logger.error(
+            f"Scan data did not contain a `Results` key for {source}. This indicates a malformed scan result."
+        )
+        raise ValueError(f"Missing 'Results' key in scan data for {source}")
+
+    results = trivy_data["Results"]
+    if not results:
+        stat_handler.incr("image_scan_no_results_count")
+        logger.info(f"No vulnerabilities found for {source}")
+
+    if "Metadata" not in trivy_data or not trivy_data["Metadata"]:
+        raise ValueError(f"Missing 'Metadata' in scan data for {source}")
+
+    repo_digests = trivy_data["Metadata"].get("RepoDigests", [])
+    if not repo_digests:
+        raise ValueError(f"Missing 'RepoDigests' in scan metadata for {source}")
+
+    repo_digest = repo_digests[0]
+    if "@" not in repo_digest:
+        raise ValueError(f"Invalid repo digest format in {source}: {repo_digest}")
+
+    image_digest = repo_digest.split("@")[1]
+    if not image_digest:
+        raise ValueError(f"Empty image digest for {source}")
+
+    return artifact_name, results, image_digest
+
+
+@timeit
+def sync_single_image(
+    neo4j_session: Session,
+    trivy_data: dict,
+    source: str,
+    update_tag: int,
+) -> None:
+    """
+    Sync a single image's Trivy scan results to Neo4j.
+
+    Args:
+        neo4j_session: Neo4j session for database operations
+        trivy_data: Raw Trivy JSON data
+        source: Source identifier for logging (file path or image URI)
+        update_tag: Update tag for tracking
+    """
+    try:
+        _, results, image_digest = _parse_trivy_data(trivy_data, source)
+
+        # Transform all data in one pass
+        findings_list, packages_list, fixes_list = transform_scan_results(
+            results,
+            image_digest,
+        )
+
+        num_findings = len(findings_list)
+        stat_handler.incr("image_scan_cve_count", num_findings)
+
+        # Load the transformed data
+        load_scan_vulns(neo4j_session, findings_list, update_tag=update_tag)
+        load_scan_packages(neo4j_session, packages_list, update_tag=update_tag)
+        load_scan_fixes(neo4j_session, fixes_list, update_tag=update_tag)
+        stat_handler.incr("images_processed_count")
+
+    except Exception as e:
+        logger.error(f"Failed to process scan results for {source}: {e}")
+        raise
+
+
 @timeit
 def get_json_files_in_s3(
     s3_bucket: str, s3_prefix: str, boto3_session: boto3.Session
@@ -177,6 +262,18 @@ def get_json_files_in_s3(
     return results
 
 
+@timeit
+def get_json_files_in_dir(results_dir: str) -> set[str]:
+    """Return set of JSON file paths under a directory."""
+    results = set()
+    for root, _dirs, files in os.walk(results_dir):
+        for filename in files:
+            if filename.endswith(".json"):
+                results.add(os.path.join(root, filename))
+    logger.info(f"Found {len(results)} json files in {results_dir}")
+    return results
+
+
 @timeit
 def cleanup(neo4j_session: Session, common_job_parameters: dict[str, Any]) -> None:
     """
@@ -245,58 +342,6 @@ def load_scan_fixes(
     )
 
 
-@timeit
-def read_scan_results_from_s3(
-    boto3_session: boto3.Session,
-    s3_bucket: str,
-    s3_object_key: str,
-    image_uri: str,
-) -> tuple[list[dict], str | None]:
-    """
-    Read and parse Trivy scan results from S3.
-
-    Args:
-        boto3_session: boto3 session for S3 operations
-        s3_bucket: S3 bucket containing scan results
-        s3_object_key: S3 object key for the scan results
-        image_uri: ECR image URI (for logging purposes)
-
-    Returns:
-        Tuple of (list of scan result dictionaries from the "Results" key, image digest)
-    """
-    s3_client = boto3_session.client("s3")
-
-    # Read JSON scan results from S3
-    logger.debug(f"Reading scan results from S3: s3://{s3_bucket}/{s3_object_key}")
-    response = s3_client.get_object(Bucket=s3_bucket, Key=s3_object_key)
-    scan_data_json = response["Body"].read().decode("utf-8")
-
-    # Parse JSON data
-    trivy_data = json.loads(scan_data_json)
-
-    # Extract results using the same logic as binary scanning
-    if "Results" in trivy_data and trivy_data["Results"]:
-        results = trivy_data["Results"]
-    else:
-        stat_handler.incr("image_scan_no_results_count")
-        logger.warning(
-            f"S3 scan data did not contain a `Results` key for URI = {image_uri}; continuing."
-        )
-        results = []
-
-    image_digest = None
-    if "Metadata" in trivy_data and trivy_data["Metadata"]:
-        repo_digests = trivy_data["Metadata"].get("RepoDigests", [])
-        if repo_digests:
-            # Sample input: 000000000000.dkr.ecr.us-east-1.amazonaws.com/test-repository@sha256:88016
-            # Sample output: sha256:88016
-            repo_digest = repo_digests[0]
-            if "@" in repo_digest:
-                image_digest = repo_digest.split("@")[1]
-
-    return results, image_digest
-
-
 @timeit
 def sync_single_image_from_s3(
     neo4j_session: Session,
@@ -317,47 +362,25 @@ def sync_single_image_from_s3(
         s3_object_key: S3 object key for this image's scan results
         boto3_session: boto3 session for S3 operations
     """
-    try:
-        # Read and parse scan results from S3
-        results, image_digest = read_scan_results_from_s3(
-            boto3_session,
-            s3_bucket,
-            s3_object_key,
-            image_uri,
-        )
-        if not image_digest:
-            logger.warning(f"No image digest found for {image_uri}; skipping over.")
-            return
+    s3_client = boto3_session.client("s3")
 
-        # Transform all data in one pass using existing function
-        findings_list, packages_list, fixes_list = transform_scan_results(
-            results,
-            image_digest,
-        )
+    logger.debug(f"Reading scan results from S3: s3://{s3_bucket}/{s3_object_key}")
+    response = s3_client.get_object(Bucket=s3_bucket, Key=s3_object_key)
+    scan_data_json = response["Body"].read().decode("utf-8")
 
-        num_findings = len(findings_list)
-        stat_handler.incr("image_scan_cve_count", num_findings)
+    trivy_data = json.loads(scan_data_json)
+    sync_single_image(neo4j_session, trivy_data, image_uri, update_tag)
 
-        # Load the transformed data using existing functions
-        load_scan_vulns(
-            neo4j_session,
-            findings_list,
-            update_tag=update_tag,
-        )
-        load_scan_packages(
-            neo4j_session,
-            packages_list,
-            update_tag=update_tag,
-        )
-        load_scan_fixes(
-            neo4j_session,
-            fixes_list,
-            update_tag=update_tag,
-        )
-        stat_handler.incr("images_processed_count")
 
-    except Exception as e:
-        logger.error(
-            f"Failed to process S3 scan results for {image_uri} from {s3_object_key}: {e}"
-        )
-        raise
+@timeit
+def sync_single_image_from_file(
+    neo4j_session: Session,
+    file_path: str,
+    update_tag: int,
+) -> None:
+    """Read a Trivy JSON file from disk and sync to Neo4j."""
+    logger.debug(f"Reading scan results from file: {file_path}")
+    with open(file_path, encoding="utf-8") as f:
+        trivy_data = json.load(f)
+
+    sync_single_image(neo4j_session, trivy_data, file_path, update_tag)

cartography/models/aws/eventbridge/rule.py

@@ -0,0 +1,77 @@
+from dataclasses import dataclass
+
+from cartography.models.core.common import PropertyRef
+from cartography.models.core.nodes import CartographyNodeProperties
+from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.relationships import CartographyRelProperties
+from cartography.models.core.relationships import CartographyRelSchema
+from cartography.models.core.relationships import LinkDirection
+from cartography.models.core.relationships import make_target_node_matcher
+from cartography.models.core.relationships import OtherRelationships
+from cartography.models.core.relationships import TargetNodeMatcher
+
+
+@dataclass(frozen=True)
+class EventBridgeRuleNodeProperties(CartographyNodeProperties):
+    id: PropertyRef = PropertyRef("Arn")
+    arn: PropertyRef = PropertyRef("Arn", extra_index=True)
+    name: PropertyRef = PropertyRef("Name")
+    region: PropertyRef = PropertyRef("Region", set_in_kwargs=True)
+    event_pattern: PropertyRef = PropertyRef("EventPattern")
+    state: PropertyRef = PropertyRef("State")
+    description: PropertyRef = PropertyRef("Description")
+    schedule_expression: PropertyRef = PropertyRef("ScheduleExpression")
+    role_arn: PropertyRef = PropertyRef("RoleArn")
+    managed_by: PropertyRef = PropertyRef("ManagedBy")
+    event_bus_name: PropertyRef = PropertyRef("EventBusName")
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class EventBridgeRuleToAwsAccountRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class EventBridgeRuleToAWSAccountRel(CartographyRelSchema):
+    target_node_label: str = "AWSAccount"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {"id": PropertyRef("AWS_ID", set_in_kwargs=True)},
+    )
+    direction: LinkDirection = LinkDirection.INWARD
+    rel_label: str = "RESOURCE"
+    properties: EventBridgeRuleToAwsAccountRelProperties = (
+        EventBridgeRuleToAwsAccountRelProperties()
+    )
+
+
+@dataclass(frozen=True)
+class EventBridgeRuleToAWSRoleRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class EventBridgeRuleToAWSRoleRel(CartographyRelSchema):
+    target_node_label: str = "AWSRole"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {"arn": PropertyRef("RoleArn")},
+    )
+    direction: LinkDirection = LinkDirection.OUTWARD
+    rel_label: str = "ASSOCIATED_WITH"
+    properties: EventBridgeRuleToAWSRoleRelProperties = (
+        EventBridgeRuleToAWSRoleRelProperties()
+    )
+
+
+@dataclass(frozen=True)
+class EventBridgeRuleSchema(CartographyNodeSchema):
+    label: str = "EventBridgeRule"
+    properties: EventBridgeRuleNodeProperties = EventBridgeRuleNodeProperties()
+    sub_resource_relationship: EventBridgeRuleToAWSAccountRel = (
+        EventBridgeRuleToAWSAccountRel()
+    )
+    other_relationships: OtherRelationships = OtherRelationships(
+        [
+            EventBridgeRuleToAWSRoleRel(),
+        ]
+    )

cartography/models/aws/glue/connection.py

@@ -0,0 +1,51 @@
+from dataclasses import dataclass
+
+from cartography.models.core.common import PropertyRef
+from cartography.models.core.nodes import CartographyNodeProperties
+from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.relationships import CartographyRelProperties
+from cartography.models.core.relationships import CartographyRelSchema
+from cartography.models.core.relationships import LinkDirection
+from cartography.models.core.relationships import make_target_node_matcher
+from cartography.models.core.relationships import TargetNodeMatcher
+
+
+@dataclass(frozen=True)
+class GlueConnectionNodeProperties(CartographyNodeProperties):
+    id: PropertyRef = PropertyRef("Name")
+    arn: PropertyRef = PropertyRef("Name", extra_index=True)
+    region: PropertyRef = PropertyRef("Region", set_in_kwargs=True)
+    description: PropertyRef = PropertyRef("Description")
+    connection_type: PropertyRef = PropertyRef("ConnectionType")
+    status: PropertyRef = PropertyRef("Status")
+    status_reason: PropertyRef = PropertyRef("StatusReason")
+    authentication_type: PropertyRef = PropertyRef("AuthenticationType")
+    secret_arn: PropertyRef = PropertyRef("SecretArn")
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class GlueConnectionToAwsAccountRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class GlueConnectionToAWSAccountRel(CartographyRelSchema):
+    target_node_label: str = "AWSAccount"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {"id": PropertyRef("AWS_ID", set_in_kwargs=True)},
+    )
+    direction: LinkDirection = LinkDirection.INWARD
+    rel_label: str = "RESOURCE"
+    properties: GlueConnectionToAwsAccountRelProperties = (
+        GlueConnectionToAwsAccountRelProperties()
+    )
+
+
+@dataclass(frozen=True)
+class GlueConnectionSchema(CartographyNodeSchema):
+    label: str = "GlueConnection"
+    properties: GlueConnectionNodeProperties = GlueConnectionNodeProperties()
+    sub_resource_relationship: GlueConnectionToAWSAccountRel = (
+        GlueConnectionToAWSAccountRel()
+    )