cartography 0.107.0rc2__py3-none-any.whl → 0.108.0rc1__py3-none-any.whl

cartography/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.107.0rc2'
-__version_tuple__ = version_tuple = (0, 107, 0, 'rc2')
+__version__ = version = '0.108.0rc1'
+__version_tuple__ = version_tuple = (0, 108, 0, 'rc1')

cartography/cli.py

@@ -264,6 +264,16 @@ class CLI:
                 " If not specified, cartography by default will run all AWS sync modules available."
             ),
         )
+        parser.add_argument(
+            "--aws-guardduty-severity-threshold",
+            type=str,
+            default=None,
+            help=(
+                "GuardDuty severity threshold filter. Only findings at or above this severity level will be synced. "
+                "Valid values: LOW, MEDIUM, HIGH, CRITICAL. If not specified, all findings (except archived) will be synced. "
+                "Example: 'HIGH' will sync only HIGH and CRITICAL findings, filtering out LOW and MEDIUM severity findings."
+            ),
+        )
         parser.add_argument(
             "--analysis-job-directory",
             type=str,

cartography/config.py

@@ -53,6 +53,9 @@ class Config:
     :param entra_client_secret: Client Secret for connecting in a Service Principal Authentication approach. Optional.
     :type aws_requested_syncs: str
     :param aws_requested_syncs: Comma-separated list of AWS resources to sync. Optional.
+    :type aws_guardduty_severity_threshold: str
+    :param aws_guardduty_severity_threshold: GuardDuty severity threshold filter. Only findings at or above this
+        severity level will be synced. Valid values: LOW, MEDIUM, HIGH, CRITICAL. Optional.
     :type analysis_job_directory: str
     :param analysis_job_directory: Path to a directory tree containing analysis jobs to run. Optional.
     :type oci_sync_all_profiles: bool
@@ -185,6 +188,7 @@ class Config:
         entra_client_id=None,
         entra_client_secret=None,
         aws_requested_syncs=None,
+        aws_guardduty_severity_threshold=None,
         analysis_job_directory=None,
         oci_sync_all_profiles=None,
         okta_org_id=None,
@@ -268,6 +272,7 @@ class Config:
         self.entra_client_id = entra_client_id
         self.entra_client_secret = entra_client_secret
         self.aws_requested_syncs = aws_requested_syncs
+        self.aws_guardduty_severity_threshold = aws_guardduty_severity_threshold
         self.analysis_job_directory = analysis_job_directory
         self.oci_sync_all_profiles = oci_sync_all_profiles
         self.okta_org_id = okta_org_id

cartography/data/indexes.cypher

@@ -81,8 +81,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:DODroplet) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:DODroplet) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:DOProject) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:DOProject) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:EBSSnapshot) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:EBSSnapshot) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:EC2KeyPair) ON (n.keyfingerprint);
 CREATE INDEX IF NOT EXISTS FOR (n:EC2ReservedInstance) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:EC2ReservedInstance) ON (n.lastupdated);
@@ -156,14 +154,8 @@ CREATE INDEX IF NOT EXISTS FOR (n:GSuiteUser) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:Ip) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:Ip) ON (n.ip);
 CREATE INDEX IF NOT EXISTS FOR (n:Ip) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:IpPermissionInbound) ON (n.ruleid);
-CREATE INDEX IF NOT EXISTS FOR (n:IpPermissionInbound) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:IpPermissionsEgress) ON (n.ruleid);
-CREATE INDEX IF NOT EXISTS FOR (n:IpPermissionsEgress) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:IpRange) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:IpRange) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:IpRule) ON (n.ruleid);
-CREATE INDEX IF NOT EXISTS FOR (n:IpRule) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:JamfComputerGroup) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:JamfComputerGroup) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:KMSKey) ON (n.id);

cartography/data/jobs/cleanup/github_repos_cleanup.json

@@ -19,6 +19,7 @@
     "iterative": true,
     "iterationsize": 100
   },
+
   {
     "query": "MATCH (:GitHubBranch)-[r:BRANCH]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
     "iterative": true,
@@ -39,6 +40,7 @@
     "iterative": true,
     "iterationsize": 100
   },
+
   {
     "query": "MATCH (:GitHubUser)-[r:OUTSIDE_COLLAB_ADMIN]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
     "iterative": true,

cartography/intel/aws/__init__.py

@@ -310,6 +310,7 @@ def start_aws_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
     common_job_parameters = {
         "UPDATE_TAG": config.update_tag,
         "permission_relationships_file": config.permission_relationships_file,
+        "aws_guardduty_severity_threshold": config.aws_guardduty_severity_threshold,
         "aws_cloudtrail_management_events_lookback_hours": config.aws_cloudtrail_management_events_lookback_hours,
     }
     try:

cartography/intel/aws/cloudwatch.py

@@ -9,6 +9,9 @@ import neo4j
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
 from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.cloudwatch.log_metric_filter import (
+    CloudWatchLogMetricFilterSchema,
+)
 from cartography.models.aws.cloudwatch.loggroup import CloudWatchLogGroupSchema
 from cartography.util import aws_handle_regions
 from cartography.util import timeit
@@ -31,6 +34,47 @@ def get_cloudwatch_log_groups(
     return logGroups
 
 
+@timeit
+@aws_handle_regions
+def get_cloudwatch_log_metric_filters(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    logs_client = boto3_session.client(
+        "logs", region_name=region, config=get_botocore_config()
+    )
+    paginator = logs_client.get_paginator("describe_metric_filters")
+    metric_filters = []
+
+    for page in paginator.paginate():
+        metric_filters.extend(page.get("metricFilters", []))
+
+    return metric_filters
+
+
+def transform_metric_filters(
+    metric_filters: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    """
+    Transform CloudWatch log metric filter data for ingestion into Neo4j.
+    Ensures that the 'id' field is a unique combination of logGroupName and filterName.
+    """
+    transformed_filters = []
+    for filter in metric_filters:
+        transformed_filter = {
+            "id": f"{filter['logGroupName']}:{filter['filterName']}",
+            "arn": f"{filter['logGroupName']}:{filter['filterName']}",
+            "filterName": filter["filterName"],
+            "filterPattern": filter.get("filterPattern"),
+            "logGroupName": filter["logGroupName"],
+            "metricName": filter["metricTransformations"][0]["metricName"],
+            "metricNamespace": filter["metricTransformations"][0]["metricNamespace"],
+            "metricValue": filter["metricTransformations"][0]["metricValue"],
+            "Region": region,
+        }
+        transformed_filters.append(transformed_filter)
+    return transformed_filters
+
+
 @timeit
 def load_cloudwatch_log_groups(
     neo4j_session: neo4j.Session,
@@ -52,6 +96,27 @@ def load_cloudwatch_log_groups(
     )
 
 
+@timeit
+def load_cloudwatch_log_metric_filters(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading CloudWatch {len(data)} log metric filters for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        CloudWatchLogMetricFilterSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
 @timeit
 def cleanup(
     neo4j_session: neo4j.Session,
@@ -62,6 +127,9 @@ def cleanup(
         CloudWatchLogGroupSchema(), common_job_parameters
     )
     cleanup_job.run(neo4j_session)
+    GraphJob.from_node_schema(
+        CloudWatchLogMetricFilterSchema(), common_job_parameters
+    ).run(neo4j_session)
 
 
 @timeit
@@ -90,4 +158,13 @@ def sync(
             update_tag,
         )
 
+        metric_filters = get_cloudwatch_log_metric_filters(boto3_session, region)
+        transformed_filters = transform_metric_filters(metric_filters, region)
+        load_cloudwatch_log_metric_filters(
+            neo4j_session,
+            transformed_filters,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
     cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/ec2/security_groups.py

@@ -1,17 +1,22 @@
 import logging
-from string import Template
+from collections import namedtuple
+from typing import Any
 from typing import Dict
 from typing import List
 
 import boto3
 import neo4j
 
+from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
+from cartography.models.aws.ec2.security_group_rules import IpPermissionInboundSchema
+from cartography.models.aws.ec2.security_group_rules import IpRangeSchema
+from cartography.models.aws.ec2.security_group_rules import IpRuleSchema
+from cartography.models.aws.ec2.security_groups import EC2SecurityGroupSchema
 from cartography.models.aws.ec2.securitygroup_instance import (
     EC2SecurityGroupInstanceSchema,
 )
 from cartography.util import aws_handle_regions
-from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 from .util import get_botocore_config
@@ -37,138 +42,146 @@ def get_ec2_security_group_data(
     return security_groups
 
 
+Ec2SecurityGroupData = namedtuple(
+    "Ec2SecurityGroupData",
+    ["groups", "inbound_rules", "egress_rules", "ranges"],
+)
+
+
+def transform_ec2_security_group_data(
+    data: List[Dict[str, Any]],
+) -> Ec2SecurityGroupData:
+    groups: List[Dict[str, Any]] = []
+    inbound_rules: List[Dict[str, Any]] = []
+    egress_rules: List[Dict[str, Any]] = []
+    ranges: List[Dict[str, Any]] = []
+
+    for group in data:
+        group_record = {
+            "GroupId": group["GroupId"],
+            "GroupName": group.get("GroupName"),
+            "Description": group.get("Description"),
+            "VpcId": group.get("VpcId"),
+        }
+        # Collect referenced security groups for relationship loading
+        source_group_ids: set[str] = set()
+
+        for rule_type, target in (
+            ("IpPermissions", inbound_rules),
+            ("IpPermissionsEgress", egress_rules),
+        ):
+            for rule in group.get(rule_type, []):
+                protocol = rule.get("IpProtocol", "all")
+                from_port = rule.get("FromPort")
+                to_port = rule.get("ToPort")
+                rule_id = (
+                    f"{group['GroupId']}/{rule_type}/{from_port}{to_port}{protocol}"
+                )
+                target.append(
+                    {
+                        "RuleId": rule_id,
+                        "GroupId": group["GroupId"],
+                        "Protocol": protocol,
+                        "FromPort": from_port,
+                        "ToPort": to_port,
+                    },
+                )
+                for ip_range in rule.get("IpRanges", []):
+                    ranges.append({"RangeId": ip_range["CidrIp"], "RuleId": rule_id})
+                for pair in rule.get("UserIdGroupPairs", []):
+                    sg_id = pair.get("GroupId")
+                    if sg_id:
+                        source_group_ids.add(sg_id)
+
+        group_record["SOURCE_GROUP_IDS"] = list(source_group_ids)
+        groups.append(group_record)
+
+    return Ec2SecurityGroupData(
+        groups=groups,
+        inbound_rules=inbound_rules,
+        egress_rules=egress_rules,
+        ranges=ranges,
+    )
+
+
 @timeit
-def load_ec2_security_group_rule(
+def load_ip_rules(
     neo4j_session: neo4j.Session,
-    group: Dict,
-    rule_type: str,
+    data: List[Dict[str, Any]],
+    inbound: bool,
+    region: str,
+    aws_account_id: str,
     update_tag: int,
 ) -> None:
-    INGEST_RULE_TEMPLATE = Template(
-        """
-    MERGE (rule:$rule_label{ruleid: $RuleId})
-    ON CREATE SET rule :IpRule, rule.firstseen = timestamp(), rule.fromport = $FromPort, rule.toport = $ToPort,
-    rule.protocol = $Protocol
-    SET rule.lastupdated = $update_tag
-    WITH rule
-    MATCH (group:EC2SecurityGroup{groupid: $GroupId})
-    MERGE (group)<-[r:MEMBER_OF_EC2_SECURITY_GROUP]-(rule)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $update_tag;
-    """,
+    schema = IpPermissionInboundSchema() if inbound else IpRuleSchema()
+    load(
+        neo4j_session,
+        schema,
+        data,
+        Region=region,
+        AWS_ID=aws_account_id,
+        lastupdated=update_tag,
     )
 
-    ingest_rule_group_pair = """
-    MERGE (group:EC2SecurityGroup{id: $GroupId})
-    ON CREATE SET group.firstseen = timestamp(), group.groupid = $GroupId
-    SET group.lastupdated = $update_tag
-    WITH group
-    MATCH (inbound:IpRule{ruleid: $RuleId})
-    MERGE (inbound)-[r:MEMBER_OF_EC2_SECURITY_GROUP]->(group)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $update_tag
-    """
-
-    ingest_range = """
-    MERGE (range:IpRange{id: $RangeId})
-    ON CREATE SET range.firstseen = timestamp(), range.range = $RangeId
-    SET range.lastupdated = $update_tag
-    WITH range
-    MATCH (rule:IpRule{ruleid: $RuleId})
-    MERGE (rule)<-[r:MEMBER_OF_IP_RULE]-(range)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $update_tag
-    """
-
-    group_id = group["GroupId"]
-    rule_type_map = {
-        "IpPermissions": "IpPermissionInbound",
-        "IpPermissionsEgress": "IpPermissionEgress",
-    }
-
-    if group.get(rule_type):
-        for rule in group[rule_type]:
-            protocol = rule.get("IpProtocol", "all")
-            from_port = rule.get("FromPort")
-            to_port = rule.get("ToPort")
-
-            ruleid = f"{group_id}/{rule_type}/{from_port}{to_port}{protocol}"
-            # NOTE Cypher query syntax is incompatible with Python string formatting, so we have to do this awkward
-            # NOTE manual formatting instead.
-            neo4j_session.run(
-                INGEST_RULE_TEMPLATE.safe_substitute(
-                    rule_label=rule_type_map[rule_type],
-                ),
-                RuleId=ruleid,
-                FromPort=from_port,
-                ToPort=to_port,
-                Protocol=protocol,
-                GroupId=group_id,
-                update_tag=update_tag,
-            )
-
-            neo4j_session.run(
-                ingest_rule_group_pair,
-                GroupId=group_id,
-                RuleId=ruleid,
-                update_tag=update_tag,
-            )
-
-            for ip_range in rule["IpRanges"]:
-                range_id = ip_range["CidrIp"]
-                neo4j_session.run(
-                    ingest_range,
-                    RangeId=range_id,
-                    RuleId=ruleid,
-                    update_tag=update_tag,
-                )
+
+@timeit
+def load_ip_ranges(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    aws_account_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        IpRangeSchema(),
+        data,
+        Region=region,
+        AWS_ID=aws_account_id,
+        lastupdated=update_tag,
+    )
 
 
 @timeit
 def load_ec2_security_groupinfo(
     neo4j_session: neo4j.Session,
-    data: List[Dict],
+    data: Ec2SecurityGroupData,
     region: str,
     current_aws_account_id: str,
     update_tag: int,
 ) -> None:
-    ingest_security_group = """
-    MERGE (group:EC2SecurityGroup{id: $GroupId})
-    ON CREATE SET group.firstseen = timestamp(), group.groupid = $GroupId
-    SET group.name = $GroupName, group.description = $Description, group.region = $Region,
-    group.lastupdated = $update_tag
-    WITH group
-    MATCH (aa:AWSAccount{id: $AWS_ACCOUNT_ID})
-    MERGE (aa)-[r:RESOURCE]->(group)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $update_tag
-    WITH group
-    MATCH (vpc:AWSVpc{id: $VpcId})
-    MERGE (vpc)-[rg:MEMBER_OF_EC2_SECURITY_GROUP]->(group)
-    ON CREATE SET rg.firstseen = timestamp()
-    """
-
-    for group in data:
-        group_id = group["GroupId"]
-
-        neo4j_session.run(
-            ingest_security_group,
-            GroupId=group_id,
-            GroupName=group.get("GroupName"),
-            Description=group.get("Description"),
-            VpcId=group.get("VpcId", None),
-            Region=region,
-            AWS_ACCOUNT_ID=current_aws_account_id,
-            update_tag=update_tag,
-        )
+    load(
+        neo4j_session,
+        EC2SecurityGroupSchema(),
+        data.groups,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+        lastupdated=update_tag,
+    )
 
-        load_ec2_security_group_rule(neo4j_session, group, "IpPermissions", update_tag)
-        load_ec2_security_group_rule(
-            neo4j_session,
-            group,
-            "IpPermissionsEgress",
-            update_tag,
-        )
+    load_ip_rules(
+        neo4j_session,
+        data.inbound_rules,
+        inbound=True,
+        region=region,
+        aws_account_id=current_aws_account_id,
+        update_tag=update_tag,
+    )
+    load_ip_rules(
+        neo4j_session,
+        data.egress_rules,
+        inbound=False,
+        region=region,
+        aws_account_id=current_aws_account_id,
+        update_tag=update_tag,
+    )
+    load_ip_ranges(
+        neo4j_session,
+        data.ranges,
+        region,
+        current_aws_account_id,
+        update_tag,
+    )
 
 
 @timeit
@@ -176,11 +189,15 @@ def cleanup_ec2_security_groupinfo(
     neo4j_session: neo4j.Session,
     common_job_parameters: Dict,
 ) -> None:
-    run_cleanup_job(
-        "aws_import_ec2_security_groupinfo_cleanup.json",
-        neo4j_session,
+    GraphJob.from_node_schema(
+        EC2SecurityGroupSchema(),
         common_job_parameters,
+    ).run(neo4j_session)
+    GraphJob.from_node_schema(IpPermissionInboundSchema(), common_job_parameters).run(
+        neo4j_session,
     )
+    GraphJob.from_node_schema(IpRuleSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(IpRangeSchema(), common_job_parameters).run(neo4j_session)
     GraphJob.from_node_schema(
         EC2SecurityGroupInstanceSchema(),
         common_job_parameters,
@@ -203,9 +220,10 @@ def sync_ec2_security_groupinfo(
             current_aws_account_id,
         )
         data = get_ec2_security_group_data(boto3_session, region)
+        transformed = transform_ec2_security_group_data(data)
         load_ec2_security_groupinfo(
             neo4j_session,
-            data,
+            transformed,
             region,
             current_aws_account_id,
             update_tag,