cartography 0.117.0__tar.gz → 0.118.0__tar.gz

{cartography-0.117.0 → cartography-0.118.0}/.github/workflows/ossf-scorecard.yml

@@ -57,6 +57,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@64d10c13136e1c5bce3e5fbde8d4906eeaafc885 # v3.29.5
+        uses: github/codeql-action/upload-sarif@f443b600d91635bebf5b0d9ebc620189c0d6fba5 # v3.29.5
         with:
           sarif_file: results.sarif

{cartography-0.117.0 → cartography-0.118.0}/.github/workflows/publish-to-ghcr-and-pypi.yml

@@ -19,7 +19,7 @@ jobs:
         with:
           python-version: "3.10"
       - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
+        uses: astral-sh/setup-uv@3259c6206f993105e3a61b142c2d97bf4b9ef83d # v7.1.0
         with:
           enable-cache: true
           cache-dependency-glob: "uv.lock"

{cartography-0.117.0 → cartography-0.118.0}/.github/workflows/sphinx.yml

@@ -31,7 +31,7 @@ jobs:
         with:
           python-version: "3.10"
       - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
+        uses: astral-sh/setup-uv@3259c6206f993105e3a61b142c2d97bf4b9ef83d # v7.1.0
         with:
           enable-cache: true
           cache-dependency-glob: "uv.lock"

{cartography-0.117.0 → cartography-0.118.0}/.github/workflows/test_suite.yml

@@ -40,7 +40,7 @@ jobs:
         with:
           python-version: "3.10"
       - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
+        uses: astral-sh/setup-uv@3259c6206f993105e3a61b142c2d97bf4b9ef83d # v7.1.0
         with:
           enable-cache: true
           cache-dependency-glob: "uv.lock"
@@ -72,7 +72,7 @@ jobs:
         with:
           python-version: "3.10"
       - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
+        uses: astral-sh/setup-uv@3259c6206f993105e3a61b142c2d97bf4b9ef83d # v7.1.0
         with:
           enable-cache: true
           cache-dependency-glob: "uv.lock"

{cartography-0.117.0 → cartography-0.118.0}/Dockerfile

@@ -1,5 +1,5 @@
 # Base image
-FROM python:3.10-slim@sha256:77f72bc159950ab208b8bbb9a7b22868984e004ce62966d1ca9268c346c73c64 AS base
+FROM python:3.10.19-slim@sha256:b1e484d92b32414c67cca8c7e2105e2e31d197855546b6dd71c337889aacdf23 AS base
 # Default to ''. Overridden with a specific version specifier e.g. '==0.98.0' by build args or from GitHub actions.
 ARG VERSION_SPECIFIER
 # the UID and GID to run cartography as

{cartography-0.117.0 → cartography-0.118.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cartography
-Version: 0.117.0
+Version: 0.118.0
 Summary: Explore assets and their relationships across your technical infrastructure.
 Maintainer: Cartography Contributors
 License-Expression: Apache-2.0
@@ -59,7 +59,7 @@ Requires-Dist: python-dateutil
 Requires-Dist: xmltodict
 Requires-Dist: duo-client
 Requires-Dist: cloudflare<5.0.0,>=4.1.0
-Requires-Dist: scaleway>=2.9.0
+Requires-Dist: scaleway>=2.10.0
 Requires-Dist: google-cloud-resource-manager>=1.14.2
 Requires-Dist: types-aiobotocore-ecr
 Requires-Dist: typer>=0.9.0
@@ -89,7 +89,7 @@ You can learn more about the story behind Cartography in our [presentation at BS
 
 ## Supported platforms
 - [Airbyte](https://cartography-cncf.github.io/cartography/modules/airbyte/index.html) - Organization, Workspace, User, Source, Destination, Connection, Tag, Stream
-- [Amazon Web Services](https://cartography-cncf.github.io/cartography/modules/aws/index.html) - ACM, API Gateway, CloudWatch, CodeBuild, Config, Cognito, EC2, ECS, ECR (including image layers), EFS, Elasticsearch, Elastic Kubernetes Service (EKS), DynamoDB, Glue,  GuardDuty, IAM, Inspector, KMS, Lambda, RDS, Redshift, Route53, S3, Secrets Manager(Secret Versions), Security Hub, SNS, SQS, SSM, STS, Tags
+- [Amazon Web Services](https://cartography-cncf.github.io/cartography/modules/aws/index.html) - ACM, API Gateway, CloudWatch, CodeBuild, Config, Cognito, EC2, ECS, ECR (including multi-arch images, image layers, and attestations), EFS, Elasticsearch, Elastic Kubernetes Service (EKS), DynamoDB, Glue,  GuardDuty, IAM, Inspector, KMS, Lambda, RDS, Redshift, Route53, S3, Secrets Manager(Secret Versions), Security Hub, SNS, SQS, SSM, STS, Tags
 - [Anthropic](https://cartography-cncf.github.io/cartography/modules/anthropic/index.html) - Organization, ApiKey, User, Workspace
 - [BigFix](https://cartography-cncf.github.io/cartography/modules/bigfix/index.html) - Computers
 - [Cloudflare](https://cartography-cncf.github.io/cartography/modules/cloudflare/index.html) - Account, Role, Member, Zone, DNSRecord

{cartography-0.117.0 → cartography-0.118.0}/README.md

@@ -22,7 +22,7 @@ You can learn more about the story behind Cartography in our [presentation at BS
 
 ## Supported platforms
 - [Airbyte](https://cartography-cncf.github.io/cartography/modules/airbyte/index.html) - Organization, Workspace, User, Source, Destination, Connection, Tag, Stream
-- [Amazon Web Services](https://cartography-cncf.github.io/cartography/modules/aws/index.html) - ACM, API Gateway, CloudWatch, CodeBuild, Config, Cognito, EC2, ECS, ECR (including image layers), EFS, Elasticsearch, Elastic Kubernetes Service (EKS), DynamoDB, Glue,  GuardDuty, IAM, Inspector, KMS, Lambda, RDS, Redshift, Route53, S3, Secrets Manager(Secret Versions), Security Hub, SNS, SQS, SSM, STS, Tags
+- [Amazon Web Services](https://cartography-cncf.github.io/cartography/modules/aws/index.html) - ACM, API Gateway, CloudWatch, CodeBuild, Config, Cognito, EC2, ECS, ECR (including multi-arch images, image layers, and attestations), EFS, Elasticsearch, Elastic Kubernetes Service (EKS), DynamoDB, Glue,  GuardDuty, IAM, Inspector, KMS, Lambda, RDS, Redshift, Route53, S3, Secrets Manager(Secret Versions), Security Hub, SNS, SQS, SSM, STS, Tags
 - [Anthropic](https://cartography-cncf.github.io/cartography/modules/anthropic/index.html) - Organization, ApiKey, User, Workspace
 - [BigFix](https://cartography-cncf.github.io/cartography/modules/bigfix/index.html) - Computers
 - [Cloudflare](https://cartography-cncf.github.io/cartography/modules/cloudflare/index.html) - Account, Role, Member, Zone, DNSRecord

{cartography-0.117.0 → cartography-0.118.0}/cartography/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.117.0'
-__version_tuple__ = version_tuple = (0, 117, 0)
+__version__ = version = '0.118.0'
+__version_tuple__ = version_tuple = (0, 118, 0)
 
-__commit_id__ = commit_id = 'gffeda28bf'
+__commit_id__ = commit_id = 'g3e9cd1a2d'

{cartography-0.117.0 → cartography-0.118.0}/cartography/cli.py

@@ -279,6 +279,17 @@ class CLI:
                 "Example: 'HIGH' will sync only HIGH and CRITICAL findings, filtering out LOW and MEDIUM severity findings."
             ),
         )
+        parser.add_argument(
+            "--experimental-aws-inspector-batch",
+            type=int,
+            default=1000,
+            help=(
+                "EXPERIMENTAL: This feature is experimental and may be removed in the future. "
+                "Batch size for AWS Inspector findings sync. Controls how many findings are fetched, processed and cleaned up at a time. "
+                "Default is 1000. Increase this value if you have a large number of findings and want to reduce API calls, "
+                "or decrease it if you're experiencing memory issues."
+            ),
+        )
         parser.add_argument(
             "--analysis-job-directory",
             type=str,

{cartography-0.117.0 → cartography-0.118.0}/cartography/config.py

@@ -58,6 +58,9 @@ class Config:
     :type aws_guardduty_severity_threshold: str
     :param aws_guardduty_severity_threshold: GuardDuty severity threshold filter. Only findings at or above this
         severity level will be synced. Valid values: LOW, MEDIUM, HIGH, CRITICAL. Optional.
+    :type experimental_aws_inspector_batch: int
+    :param experimental_aws_inspector_batch: EXPERIMENTAL: Batch size for AWS Inspector findings sync. Controls how
+        many findings are fetched, processed and cleaned up at a time. Default is 1000. Optional.
     :type analysis_job_directory: str
     :param analysis_job_directory: Path to a directory tree containing analysis jobs to run. Optional.
     :type oci_sync_all_profiles: bool
@@ -195,6 +198,7 @@ class Config:
         aws_regions=None,
         aws_best_effort_mode=False,
         aws_cloudtrail_management_events_lookback_hours=None,
+        experimental_aws_inspector_batch=1000,
         azure_sync_all_subscriptions=False,
         azure_sp_auth=None,
         azure_tenant_id=None,
@@ -287,6 +291,7 @@ class Config:
         self.aws_cloudtrail_management_events_lookback_hours = (
             aws_cloudtrail_management_events_lookback_hours
         )
+        self.experimental_aws_inspector_batch = experimental_aws_inspector_batch
         self.azure_sync_all_subscriptions = azure_sync_all_subscriptions
         self.azure_sp_auth = azure_sp_auth
         self.azure_tenant_id = azure_tenant_id

{cartography-0.117.0 → cartography-0.118.0}/cartography/graph/job.py

@@ -139,11 +139,13 @@ class GraphJob:
         cls,
         node_schema: CartographyNodeSchema,
         parameters: Dict[str, Any],
+        iterationsize: int = 100,
     ) -> "GraphJob":
         """
         Create a cleanup job from a CartographyNodeSchema object.
         For a given node, the fields used in the node_schema.sub_resource_relationship.target_node_node_matcher.keys()
         must be provided as keys and values in the params dict.
+        :param iterationsize: The number of items to process in each iteration. Defaults to 100.
         """
         queries: List[str] = build_cleanup_queries(node_schema)
 
@@ -165,7 +167,7 @@ class GraphJob:
                 query,
                 parameters=parameters,
                 iterative=True,
-                iterationsize=100,
+                iterationsize=iterationsize,
                 parent_job_name=node_schema.label,
                 parent_job_sequence_num=idx,
             )
@@ -185,6 +187,7 @@ class GraphJob:
         sub_resource_label: str,
         sub_resource_id: str,
         update_tag: int,
+        iterationsize: int = 100,
     ) -> "GraphJob":
         """
         Create a cleanup job from a CartographyRelSchema object (specifically, a MatchLink).
@@ -194,6 +197,7 @@ class GraphJob:
         - For a given rel_schema, the fields used in the rel_schema.properties._sub_resource_label.name and
         rel_schema.properties._sub_resource_id.name must be provided as keys and values in the params dict.
         - The rel_schema must have a source_node_matcher and target_node_matcher.
+        :param iterationsize: The number of items to process in each iteration. Defaults to 100.
         """
         cleanup_link_query = build_cleanup_query_for_matchlink(rel_schema)
         logger.debug(f"Cleanup query: {cleanup_link_query}")
@@ -208,7 +212,7 @@ class GraphJob:
             cleanup_link_query,
             parameters=parameters,
             iterative=True,
-            iterationsize=100,
+            iterationsize=iterationsize,
             parent_job_name=rel_schema.rel_label,
         )
 

{cartography-0.117.0 → cartography-0.118.0}/cartography/graph/statement.py

@@ -52,6 +52,10 @@ class GraphStatement:
         self.parameters = parameters or {}
         self.iterative = iterative
         self.iterationsize = iterationsize
+        if iterationsize < 0:
+            raise ValueError(
+                f"iterationsize must be a positive integer, got {iterationsize}",
+            )
         self.parameters["LIMIT_SIZE"] = self.iterationsize
 
         self.parent_job_name = parent_job_name if parent_job_name else None

{cartography-0.117.0 → cartography-0.118.0}/cartography/intel/aws/__init__.py

@@ -312,6 +312,7 @@ def start_aws_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
         "permission_relationships_file": config.permission_relationships_file,
         "aws_guardduty_severity_threshold": config.aws_guardduty_severity_threshold,
         "aws_cloudtrail_management_events_lookback_hours": config.aws_cloudtrail_management_events_lookback_hours,
+        "experimental_aws_inspector_batch": config.experimental_aws_inspector_batch,
     }
     try:
         boto3_session = boto3.Session()

{cartography-0.117.0 → cartography-0.118.0}/cartography/intel/aws/apigateway.py

@@ -178,11 +178,24 @@ def get_rest_api_resources_methods_integrations(
                 method["apiId"] = api["id"]
                 method["httpMethod"] = http_method
                 methods.append(method)
-                integration = client.get_integration(
-                    restApiId=api["id"],
-                    resourceId=resource_id,
-                    httpMethod=http_method,
-                )
+                try:
+                    integration = client.get_integration(
+                        restApiId=api["id"],
+                        resourceId=resource_id,
+                        httpMethod=http_method,
+                    )
+                except ClientError as e:
+                    error_code = e.response.get("Error", {}).get("Code")
+                    if error_code == "NotFoundException":
+                        logger.warning(
+                            "No integration found for API %s resource %s method %s: %s",
+                            api["id"],
+                            resource_id,
+                            http_method,
+                            e,
+                        )
+                        continue
+                    raise
                 integration["resourceId"] = resource_id
                 integration["apiId"] = api["id"]
                 integration["integrationHttpMethod"] = integration.get("httpMethod")

{cartography-0.117.0 → cartography-0.118.0}/cartography/intel/aws/ec2/elastic_ip_addresses.py

@@ -6,6 +6,7 @@ import boto3
 import neo4j
 from botocore.exceptions import ClientError
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
@@ -83,7 +84,8 @@ def load_elastic_ip_addresses(
         SET r.lastupdated = $update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_addresses,
         elastic_ip_addresses=elastic_ip_addresses,
         Region=region,

{cartography-0.117.0 → cartography-0.118.0}/cartography/intel/aws/ec2/internet_gateways.py

@@ -5,6 +5,7 @@ from typing import List
 import boto3
 import neo4j
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
@@ -63,13 +64,14 @@ def load_internet_gateways(
         SET r.lastupdated = $aws_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         query,
         internet_gateways=internet_gateways,
         region=region,
         aws_account_id=current_aws_account_id,
         aws_update_tag=update_tag,
-    ).consume()
+    )
 
 
 @timeit

{cartography-0.117.0 → cartography-0.118.0}/cartography/intel/aws/ec2/load_balancer_v2s.py

@@ -6,6 +6,7 @@ import boto3
 import botocore
 import neo4j
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
@@ -104,7 +105,8 @@ def load_load_balancer_v2s(
             logger.warning("Skipping load balancer entry with missing DNSName: %r", lb)
             continue
 
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             ingest_load_balancer_v2,
             ID=load_balancer_id,
             CREATED_TIME=str(lb["CreatedTime"]),
@@ -138,7 +140,8 @@ def load_load_balancer_v2s(
             SET r.lastupdated = $update_tag
             """
             for group in lb["SecurityGroups"]:
-                neo4j_session.run(
+                run_write_query(
+                    neo4j_session,
                     ingest_load_balancer_v2_security_group,
                     ID=load_balancer_id,
                     GROUP_ID=str(group),
@@ -182,7 +185,8 @@ def load_load_balancer_v2_subnets(
     SET r.lastupdated = $update_tag
     """
     for az in az_data:
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             ingest_load_balancer_subnet,
             ID=load_balancer_id,
             SubnetId=az["SubnetId"],
@@ -219,7 +223,8 @@ def load_load_balancer_v2_target_groups(
             continue
 
         for instance in target_group["Targets"]:
-            neo4j_session.run(
+            run_write_query(
+                neo4j_session,
                 ingest_instances,
                 ID=load_balancer_id,
                 INSTANCE_ID=instance,
@@ -253,7 +258,8 @@ def load_load_balancer_v2_listeners(
         ON CREATE SET r.firstseen = timestamp()
         SET r.lastupdated = $update_tag
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_listener,
         LoadBalancerId=load_balancer_id,
         Listeners=listener_data,

{cartography-0.117.0 → cartography-0.118.0}/cartography/intel/aws/ec2/network_interfaces.py

@@ -98,6 +98,10 @@ def transform_network_interface_data(
                 "SourceDestCheck": network_interface["SourceDestCheck"],
                 "Status": network_interface["Status"],
                 "SubnetId": network_interface["SubnetId"],
+                "AttachTime": network_interface.get("Attachment", {}).get("AttachTime"),
+                "DeviceIndex": network_interface.get("Attachment", {}).get(
+                    "DeviceIndex"
+                ),
                 "ElbV1Id": elb_v1_id,
                 "ElbV2Id": elb_v2_id,
             },

{cartography-0.117.0 → cartography-0.118.0}/cartography/intel/aws/ec2/reserved_instances.py

@@ -6,6 +6,7 @@ import boto3
 import neo4j
 from botocore.exceptions import ClientError
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
@@ -64,7 +65,8 @@ def load_reserved_instances(
         r_instance["Start"] = str(r_instance["Start"])
         r_instance["End"] = str(r_instance["End"])
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_reserved_instances,
         reserved_instances_list=data,
         AWS_ACCOUNT_ID=current_aws_account_id,

{cartography-0.117.0 → cartography-0.118.0}/cartography/intel/aws/ec2/tgw.py

@@ -6,6 +6,7 @@ import boto3
 import botocore.exceptions
 import neo4j
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
@@ -120,7 +121,8 @@ def load_transit_gateways(
     for tgw in data:
         tgw_id = tgw["TransitGatewayId"]
 
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             ingest_transit_gateway,
             TgwId=tgw_id,
             ARN=tgw["TransitGatewayArn"],
@@ -161,7 +163,8 @@ def _attach_shared_transit_gateway(
     """
 
     if tgw["OwnerId"] != current_aws_account_id:
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             attach_tgw,
             ARN=tgw["TransitGatewayArn"],
             TransitGatewayId=tgw["TransitGatewayId"],
@@ -202,7 +205,8 @@ def load_tgw_attachments(
     for tgwa in data:
         tgwa_id = tgwa["TransitGatewayAttachmentId"]
 
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             ingest_transit_gateway,
             TgwAttachmentId=tgwa_id,
             TransitGatewayId=tgwa["TransitGatewayId"],
@@ -261,7 +265,8 @@ def _attach_tgw_vpc_attachment_to_vpc_subnets(
     SET p.lastupdated = $update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         attach_vpc_tgw_attachment_to_vpc,
         VpcId=tgw_vpc_attachment["VpcId"],
         TgwAttachmentId=tgw_vpc_attachment["TransitGatewayAttachmentId"],
@@ -269,7 +274,8 @@ def _attach_tgw_vpc_attachment_to_vpc_subnets(
     )
 
     for subnet_id in tgw_vpc_attachment["SubnetIds"]:
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             attach_vpc_tgw_attachment_to_subnet,
             SubnetId=subnet_id,
             TgwAttachmentId=tgw_vpc_attachment["TransitGatewayAttachmentId"],

{cartography-0.117.0 → cartography-0.118.0}/cartography/intel/aws/ec2/volumes.py

@@ -70,7 +70,7 @@ def transform_volumes(
 
         for attachment in active_attachments:
             vol_with_attachment = raw_vol.copy()
-            vol_with_attachment["InstanceId"] = attachment["InstanceId"]
+            vol_with_attachment["InstanceId"] = attachment.get("InstanceId")
             result.append(vol_with_attachment)
 
     return result