cartography 0.108.0rc1__py3-none-any.whl → 0.109.0__py3-none-any.whl

cartography/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.108.0rc1'
-__version_tuple__ = version_tuple = (0, 108, 0, 'rc1')
+__version__ = version = '0.109.0'
+__version_tuple__ = version_tuple = (0, 109, 0)

cartography/cli.py

@@ -700,6 +700,15 @@ class CLI:
                 "Required if you are using the Trivy module. Ignored otherwise."
             ),
         )
+        parser.add_argument(
+            "--trivy-results-dir",
+            type=str,
+            default=None,
+            help=(
+                "Path to a directory containing Trivy JSON results on disk. "
+                "Required if you are using the Trivy module with local results."
+            ),
+        )
         parser.add_argument(
             "--scaleway-org",
             type=str,
@@ -1089,6 +1098,9 @@ class CLI:
         if config.trivy_s3_prefix:
             logger.debug(f"Trivy S3 prefix: {config.trivy_s3_prefix}")
 
+        if config.trivy_results_dir:
+            logger.debug(f"Trivy results dir: {config.trivy_results_dir}")
+
         # Scaleway config
         if config.scaleway_secret_key_env_var:
             logger.debug(
@@ -1118,6 +1130,8 @@ class CLI:
             config.sentinelone_api_token = os.environ.get(
                 config.sentinelone_api_token_env_var
             )
+        else:
+            config.sentinelone_api_token = None
 
         # Run cartography
         try:

cartography/config.py

@@ -152,6 +152,8 @@ class Config:
     :param trivy_s3_bucket: The S3 bucket name containing Trivy scan results. Optional.
     :type trivy_s3_prefix: str
     :param trivy_s3_prefix: The S3 prefix path containing Trivy scan results. Optional.
+    :type trivy_results_dir: str
+    :param trivy_results_dir: Local directory containing Trivy scan results. Optional.
     :type scaleway_access_key: str
     :param scaleway_access_key: Scaleway access key. Optional.
     :type scaleway_secret_key: str
@@ -243,6 +245,7 @@ class Config:
         airbyte_api_url=None,
         trivy_s3_bucket=None,
         trivy_s3_prefix=None,
+        trivy_results_dir=None,
         scaleway_access_key=None,
         scaleway_secret_key=None,
         scaleway_org=None,
@@ -327,6 +330,7 @@ class Config:
         self.airbyte_api_url = airbyte_api_url
         self.trivy_s3_bucket = trivy_s3_bucket
         self.trivy_s3_prefix = trivy_s3_prefix
+        self.trivy_results_dir = trivy_results_dir
         self.scaleway_access_key = scaleway_access_key
         self.scaleway_secret_key = scaleway_secret_key
         self.scaleway_org = scaleway_org

cartography/data/indexes.cypher

@@ -29,14 +29,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:AWSIpv4CidrBlock) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSIpv4CidrBlock) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSIpv6CidrBlock) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSIpv6CidrBlock) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambda) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambda) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaEventSourceMapping) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaEventSourceMapping) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaFunctionAlias) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaFunctionAlias) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaLayer) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaLayer) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSPeeringConnection) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSPeeringConnection) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSPolicy) ON (n.id);
@@ -158,13 +150,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:IpRange) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:IpRange) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:JamfComputerGroup) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:JamfComputerGroup) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:KMSKey) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:KMSKey) ON (n.arn);
-CREATE INDEX IF NOT EXISTS FOR (n:KMSKey) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:KMSAlias) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:KMSAlias) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:KMSGrant) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:KMSGrant) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:LaunchConfiguration) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:LaunchConfiguration) ON (n.name);
 CREATE INDEX IF NOT EXISTS FOR (n:LaunchConfiguration) ON (n.lastupdated);
@@ -259,8 +244,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:S3Bucket) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:S3Bucket) ON (n.name);
 CREATE INDEX IF NOT EXISTS FOR (n:S3Bucket) ON (n.arn);
 CREATE INDEX IF NOT EXISTS FOR (n:S3Bucket) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:SecretsManagerSecret) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:SecretsManagerSecret) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:SecurityHub) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:SecurityHub) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:SpotlightVulnerability) ON (n.id);

cartography/data/jobs/cleanup/gcp_compute_vpc_cleanup.json

@@ -1,17 +1,5 @@
 {
   "statements": [
-    {
-      "query": "MATCH (n:GCPVpc) WHERE n.lastupdated <> $UPDATE_TAG WITH n LIMIT $LIMIT_SIZE DETACH DELETE (n)",
-      "iterative": true,
-      "iterationsize": 100,
-      "__comment__": "Delete GCP VPCs that no longer exist and detach them from all previously connected nodes."
-    },
-    {
-      "query": "MATCH (:GCPVpc)<-[r:RESOURCE]-(:GCPProject) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
-      "iterative": true,
-      "iterationsize": 100,
-      "__comment__": "Remove GCP VPC-to-Project relationships that are out of date."
-    },
     {
       "query": "MATCH (:GCPInstance)-[r:MEMBER_OF_GCP_VPC]->(:GCPVpc) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
       "iterative": true,

cartography/intel/aws/cloudtrail.py

@@ -26,13 +26,25 @@ def get_cloudtrail_trails(
     )
 
     trails = client.describe_trails()["trailList"]
-
-    # CloudTrail multi-region trails are shown in list_trails,
-    # but the get_trail call only works in the home region
     trails_filtered = [trail for trail in trails if trail.get("HomeRegion") == region]
+
     return trails_filtered
 
 
+def transform_cloudtrail_trails(
+    trails: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    """
+    Transform CloudTrail trail data for ingestion
+    """
+    for trail in trails:
+        arn = trail.get("CloudWatchLogsLogGroupArn")
+        if arn:
+            trail["CloudWatchLogsLogGroupArn"] = arn.split(":*")[0]
+
+    return trails
+
+
 @timeit
 def load_cloudtrail_trails(
     neo4j_session: neo4j.Session,
@@ -79,7 +91,8 @@ def sync(
         logger.info(
             f"Syncing CloudTrail for region '{region}' in account '{current_aws_account_id}'.",
         )
-        trails = get_cloudtrail_trails(boto3_session, region)
+        trails_filtered = get_cloudtrail_trails(boto3_session, region)
+        trails = transform_cloudtrail_trails(trails_filtered, region)
 
         load_cloudtrail_trails(
             neo4j_session,