cartography 0.95.0__tar.gz → 0.96.0rc1__tar.gz

{cartography-0.95.0 → cartography-0.96.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cartography
-Version: 0.95.0
+Version: 0.96.0rc1
 Summary: Explore assets and their relationships across your technical infrastructure.
 Home-page: https://www.github.com/cartography-cncf/cartography
 Maintainer: Cartography Contributors

{cartography-0.95.0 → cartography-0.96.0rc1}/README.md

@@ -80,11 +80,10 @@ Directly querying Neo4j is already very useful as a sort of "swiss army knife" f
 
 ## Community
 
-- Join us on `#cartography` on the [Lyft OSS Slack](https://join.slack.com/t/lyftoss/shared_invite/enQtOTYzODg5OTQwNDE2LTFiYjgwZWM3NTNhMTFkZjc4Y2IxOTI4NTdiNTdhNjQ4M2Q5NTIzMjVjOWI4NmVlNjRiZmU2YzA5NTc3MmFjYTQ).
+- Hang out with us on Slack: Join the CNCF Slack workspace [here](https://communityinviter.com/apps/cloud-native/cncf), and then join the `#cartography` channel.
 - Talk to us and see what we're working on at our [monthly community meeting](https://calendar.google.com/calendar/embed?src=lyft.com_p10o6ceuiieq9sqcn1ef61v1io%40group.calendar.google.com&ctz=America%2FLos_Angeles).
   - Meeting minutes are [here](https://docs.google.com/document/d/1VyRKmB0dpX185I15BmNJZpfAJ_Ooobwz0U1WIhjDxvw).
   - Recorded videos are posted [here](https://www.youtube.com/playlist?list=PLMga2YJvAGzidUWJB_fnG7EHI4wsDDsE1).
-- Our current project roadmap is [here](https://github.com/orgs/lyft/projects/26/views/1).
 
 ## License
 

{cartography-0.95.0 → cartography-0.96.0rc1}/cartography/graph/querybuilder.py

@@ -118,6 +118,7 @@ def _build_where_clause_for_rel_match(node_var: str, matcher: TargetNodeMatcher)
     """
     match = Template("$node_var.$key = $prop_ref")
     case_insensitive_match = Template("toLower($node_var.$key) = toLower($prop_ref)")
+    fuzzy_and_ignorecase_match = Template("toLower($node_var.$key) CONTAINS toLower($prop_ref)")
 
     matcher_asdict = asdict(matcher)
 
@@ -125,7 +126,10 @@ def _build_where_clause_for_rel_match(node_var: str, matcher: TargetNodeMatcher)
     for key, prop_ref in matcher_asdict.items():
         if prop_ref.ignore_case:
             prop_line = case_insensitive_match.safe_substitute(node_var=node_var, key=key, prop_ref=prop_ref)
+        elif prop_ref.fuzzy_and_ignore_case:
+            prop_line = fuzzy_and_ignorecase_match.safe_substitute(node_var=node_var, key=key, prop_ref=prop_ref)
         else:
+            # Exact match (default; most efficient)
             prop_line = match.safe_substitute(node_var=node_var, key=key, prop_ref=prop_ref)
         result.append(prop_line)
     return ' AND\n'.join(result)

cartography-0.96.0rc1/cartography/intel/aws/ec2/network_acls.py

@@ -0,0 +1,208 @@
+import logging
+from collections import namedtuple
+from typing import Any
+
+import boto3
+import neo4j
+
+from .util import get_botocore_config
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.ec2.network_acl_rules import EC2NetworkAclEgressRuleSchema
+from cartography.models.aws.ec2.network_acl_rules import EC2NetworkAclInboundRuleSchema
+from cartography.models.aws.ec2.network_acls import EC2NetworkAclSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+Ec2AclObjects = namedtuple(
+    "Ec2AclObjects", [
+        'network_acls',
+        'inbound_rules',
+        'outbound_rules',
+    ],
+)
+
+
+@timeit
+@aws_handle_regions
+def get_network_acl_data(boto3_session: boto3.session.Session, region: str) -> list[dict[str, Any]]:
+    client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
+    paginator = client.get_paginator('describe_network_acls')
+    acls = []
+    for page in paginator.paginate():
+        acls.extend(page['NetworkAcls'])
+    return acls
+
+
+def transform_network_acl_data(
+        data_list: list[dict[str, Any]],
+        region: str,
+        current_aws_account_id: str,
+) -> Ec2AclObjects:
+    network_acls = []
+    inbound_rules = []
+    outbound_rules = []
+
+    for network_acl in data_list:
+        network_acl_id = network_acl['NetworkAclId']
+        base_network_acl = {
+            'Id': network_acl_id,
+            'Arn': f'arn:aws:ec2:{region}:{current_aws_account_id}:network-acl/{network_acl_id}',
+            'IsDefault': network_acl['IsDefault'],
+            'VpcId': network_acl['VpcId'],
+            'OwnerId': network_acl['OwnerId'],
+        }
+        if network_acl.get('Associations') and network_acl['Associations']:
+            # Include subnet associations in the data object if they exist
+            for association in network_acl['Associations']:
+                base_network_acl['NetworkAclAssociationId'] = association['NetworkAclAssociationId']
+                base_network_acl['SubnetId'] = association['SubnetId']
+                network_acls.append(base_network_acl)
+        else:
+            # Otherwise if there's no associations then don't include that in the data object
+            network_acls.append(base_network_acl)
+
+        if network_acl.get("Entries"):
+            for rule in network_acl["Entries"]:
+                direction = 'egress' if rule['Egress'] else 'inbound'
+                transformed_rule = {
+                    'Id': f"{network_acl['NetworkAclId']}/{direction}/{rule['RuleNumber']}",
+                    'CidrBlock': rule['CidrBlock'],
+                    'Egress': rule['Egress'],
+                    'Protocol': rule['Protocol'],
+                    'RuleAction': rule['RuleAction'],
+                    'RuleNumber': rule['RuleNumber'],
+                    # Add pointer back to the nacl to create an edge
+                    'NetworkAclId': network_acl_id,
+                    'FromPort': rule.get('PortRange', {}).get('FromPort'),
+                    'ToPort': rule.get('PortRange', {}).get('ToPort'),
+                }
+                if transformed_rule['Egress']:
+                    outbound_rules.append(transformed_rule)
+                else:
+                    inbound_rules.append(transformed_rule)
+    return Ec2AclObjects(
+        network_acls=network_acls,
+        inbound_rules=inbound_rules,
+        outbound_rules=outbound_rules,
+    )
+
+
+@timeit
+def load_all_nacl_data(
+        neo4j_session: neo4j.Session,
+        ec2_acl_objects: Ec2AclObjects,
+        region: str,
+        aws_account_id: str,
+        update_tag: int,
+) -> None:
+    load_network_acls(
+        neo4j_session,
+        ec2_acl_objects.network_acls,
+        region,
+        aws_account_id,
+        update_tag,
+    )
+    load_network_acl_inbound_rules(
+        neo4j_session,
+        ec2_acl_objects.inbound_rules,
+        region,
+        aws_account_id,
+        update_tag,
+    )
+    load_network_acl_egress_rules(
+        neo4j_session,
+        ec2_acl_objects.outbound_rules,
+        region,
+        aws_account_id,
+        update_tag,
+    )
+
+
+@timeit
+def load_network_acls(
+        neo4j_session: neo4j.Session,
+        data: list[dict[str, Any]],
+        region: str,
+        aws_account_id: str,
+        update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(data)} network acls in {region}.")
+    load(
+        neo4j_session,
+        EC2NetworkAclSchema(),
+        data,
+        Region=region,
+        AWS_ID=aws_account_id,
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+def load_network_acl_inbound_rules(
+        neo4j_session: neo4j.Session,
+        data: list[dict[str, Any]],
+        region: str,
+        aws_account_id: str,
+        update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(data)} network acl inbound rules in {region}.")
+    load(
+        neo4j_session,
+        EC2NetworkAclInboundRuleSchema(),
+        data,
+        Region=region,
+        AWS_ID=aws_account_id,
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+def load_network_acl_egress_rules(
+        neo4j_session: neo4j.Session,
+        data: list[dict[str, Any]],
+        region: str,
+        aws_account_id: str,
+        update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(data)} network acl egress rules in {region}.")
+    load(
+        neo4j_session,
+        EC2NetworkAclEgressRuleSchema(),
+        data,
+        Region=region,
+        AWS_ID=aws_account_id,
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+def cleanup_network_acls(neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]) -> None:
+    GraphJob.from_node_schema(EC2NetworkAclSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(EC2NetworkAclInboundRuleSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(EC2NetworkAclEgressRuleSchema(), common_job_parameters).run(neo4j_session)
+
+
+@timeit
+def sync_network_acls(
+        neo4j_session: neo4j.Session,
+        boto3_session: boto3.session.Session,
+        regions: list[str],
+        current_aws_account_id: str,
+        update_tag: int,
+        common_job_parameters: dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(f"Syncing EC2 network ACLs for region '{region}' in account '{current_aws_account_id}'.")
+        data = get_network_acl_data(boto3_session, region)
+        ec2_acl_data = transform_network_acl_data(data, region, current_aws_account_id)
+        load_all_nacl_data(
+            neo4j_session,
+            ec2_acl_data,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+        cleanup_network_acls(neo4j_session, common_job_parameters)

{cartography-0.95.0 → cartography-0.96.0rc1}/cartography/intel/aws/resources.py

@@ -32,6 +32,7 @@ from .ec2.key_pairs import sync_ec2_key_pairs
 from .ec2.launch_templates import sync_ec2_launch_templates
 from .ec2.load_balancer_v2s import sync_load_balancer_v2s
 from .ec2.load_balancers import sync_load_balancers
+from .ec2.network_acls import sync_network_acls
 from .ec2.network_interfaces import sync_network_interfaces
 from .ec2.reserved_instances import sync_ec2_reserved_instances
 from .ec2.security_groups import sync_ec2_security_groupinfo
@@ -55,6 +56,7 @@ RESOURCE_FUNCTIONS: Dict = {
     'ec2:keypair': sync_ec2_key_pairs,
     'ec2:load_balancer': sync_load_balancers,
     'ec2:load_balancer_v2': sync_load_balancer_v2s,
+    'ec2:network_acls': sync_network_acls,
     'ec2:network_interface': sync_network_interfaces,
     'ec2:security_group': sync_ec2_security_groupinfo,
     'ec2:subnet': sync_subnets,

cartography-0.96.0rc1/cartography/models/aws/ec2/network_acl_rules.py

@@ -0,0 +1,97 @@
+from dataclasses import dataclass
+
+from cartography.models.core.common import PropertyRef
+from cartography.models.core.nodes import CartographyNodeProperties
+from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.nodes import ExtraNodeLabels
+from cartography.models.core.relationships import CartographyRelProperties
+from cartography.models.core.relationships import CartographyRelSchema
+from cartography.models.core.relationships import LinkDirection
+from cartography.models.core.relationships import make_target_node_matcher
+from cartography.models.core.relationships import OtherRelationships
+from cartography.models.core.relationships import TargetNodeMatcher
+
+
+@dataclass(frozen=True)
+class EC2NetworkAclRuleNodeProperties(CartographyNodeProperties):
+    id: PropertyRef = PropertyRef('Id')
+    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+    network_acl_id: PropertyRef = PropertyRef('NetworkAclId')
+    protocol: PropertyRef = PropertyRef('Protocol')
+    fromport: PropertyRef = PropertyRef('FromPort')
+    toport: PropertyRef = PropertyRef('ToPort')
+    cidrblock: PropertyRef = PropertyRef('CidrBlock')
+    egress: PropertyRef = PropertyRef('Egress')
+    rulenumber: PropertyRef = PropertyRef('RuleNumber')
+    ruleaction: PropertyRef = PropertyRef('RuleAction')
+    region: PropertyRef = PropertyRef('Region', set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class EC2NetworkAclRuleAclRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class EC2NetworkAclRuleToAcl(CartographyRelSchema):
+    target_node_label: str = 'EC2NetworkAcl'
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {'network_acl_id': PropertyRef('NetworkAclId')},
+    )
+    direction: LinkDirection = LinkDirection.OUTWARD
+    rel_label: str = "MEMBER_OF_NACL"
+    properties: EC2NetworkAclRuleAclRelProperties = EC2NetworkAclRuleAclRelProperties()
+
+
+@dataclass(frozen=True)
+class EC2NetworkAclRuleToAwsAccountRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class EC2NetworkAclRuleToAWSAccount(CartographyRelSchema):
+    target_node_label: str = 'AWSAccount'
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {'id': PropertyRef('AWS_ID', set_in_kwargs=True)},
+    )
+    direction: LinkDirection = LinkDirection.INWARD
+    rel_label: str = "RESOURCE"
+    properties: EC2NetworkAclRuleToAwsAccountRelProperties = EC2NetworkAclRuleToAwsAccountRelProperties()
+
+
+@dataclass(frozen=True)
+class EC2NetworkAclInboundRuleSchema(CartographyNodeSchema):
+    """
+    Network interface as known by describe-network-interfaces.
+    """
+    label: str = 'EC2NetworkAclRule'
+    extra_node_labels: ExtraNodeLabels = ExtraNodeLabels(
+        ['IpPermissionInbound'],
+    )
+    properties: EC2NetworkAclRuleNodeProperties = EC2NetworkAclRuleNodeProperties()
+    sub_resource_relationship: EC2NetworkAclRuleToAWSAccount = EC2NetworkAclRuleToAWSAccount()
+    other_relationships: OtherRelationships = OtherRelationships(
+        [
+            EC2NetworkAclRuleToAcl(),
+        ],
+    )
+
+
+@dataclass(frozen=True)
+class EC2NetworkAclEgressRuleSchema(CartographyNodeSchema):
+    """
+    Network interface as known by describe-network-interfaces.
+    """
+    label: str = 'EC2NetworkAclRule'
+    extra_node_labels: ExtraNodeLabels = ExtraNodeLabels(
+        [
+            'IpPermissionEgress',
+        ],
+    )
+    properties: EC2NetworkAclRuleNodeProperties = EC2NetworkAclRuleNodeProperties()
+    sub_resource_relationship: EC2NetworkAclRuleToAWSAccount = EC2NetworkAclRuleToAWSAccount()
+    other_relationships: OtherRelationships = OtherRelationships(
+        [
+            EC2NetworkAclRuleToAcl(),
+        ],
+    )

cartography-0.96.0rc1/cartography/models/aws/ec2/network_acls.py

@@ -0,0 +1,86 @@
+from dataclasses import dataclass
+
+from cartography.models.core.common import PropertyRef
+from cartography.models.core.nodes import CartographyNodeProperties
+from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.relationships import CartographyRelProperties
+from cartography.models.core.relationships import CartographyRelSchema
+from cartography.models.core.relationships import LinkDirection
+from cartography.models.core.relationships import make_target_node_matcher
+from cartography.models.core.relationships import OtherRelationships
+from cartography.models.core.relationships import TargetNodeMatcher
+
+
+@dataclass(frozen=True)
+class EC2NetworkAclNodeProperties(CartographyNodeProperties):
+    id: PropertyRef = PropertyRef('Arn')
+    arn: PropertyRef = PropertyRef('Arn')
+    network_acl_id: PropertyRef = PropertyRef('Id')
+    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+    is_default: PropertyRef = PropertyRef('IsDefault')
+    region: PropertyRef = PropertyRef('Region', set_in_kwargs=True)
+    vpc_id: PropertyRef = PropertyRef('VpcId')
+
+
+@dataclass(frozen=True)
+class EC2NetworkAclToVpcRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class EC2NetworkAclToVpc(CartographyRelSchema):
+    target_node_label: str = 'AWSVpc'
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {'vpcid': PropertyRef('VpcId')},
+    )
+    direction: LinkDirection = LinkDirection.OUTWARD
+    rel_label: str = "MEMBER_OF_AWS_VPC"
+    properties: EC2NetworkAclToVpcRelProperties = EC2NetworkAclToVpcRelProperties()
+
+
+@dataclass(frozen=True)
+class EC2NetworkAclToSubnetRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class EC2NetworkAclToSubnet(CartographyRelSchema):
+    target_node_label: str = 'EC2Subnet'
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {'subnetid': PropertyRef('SubnetId')},
+    )
+    direction: LinkDirection = LinkDirection.OUTWARD
+    rel_label: str = "PART_OF_SUBNET"
+    properties: EC2NetworkAclToSubnetRelProperties = EC2NetworkAclToSubnetRelProperties()
+
+
+@dataclass(frozen=True)
+class EC2NetworkAclToAwsAccountRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class EC2NetworkAclToAWSAccount(CartographyRelSchema):
+    target_node_label: str = 'AWSAccount'
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {'id': PropertyRef('AWS_ID', set_in_kwargs=True)},
+    )
+    direction: LinkDirection = LinkDirection.INWARD
+    rel_label: str = "RESOURCE"
+    properties: EC2NetworkAclToAwsAccountRelProperties = EC2NetworkAclToAwsAccountRelProperties()
+
+
+@dataclass(frozen=True)
+class EC2NetworkAclSchema(CartographyNodeSchema):
+    """
+    Network interface as known by describe-network-interfaces.
+    """
+    label: str = 'EC2NetworkAcl'
+    properties: EC2NetworkAclNodeProperties = EC2NetworkAclNodeProperties()
+    sub_resource_relationship: EC2NetworkAclToAWSAccount = EC2NetworkAclToAWSAccount()
+    other_relationships: OtherRelationships = OtherRelationships(
+        [
+            EC2NetworkAclToVpc(),
+            EC2NetworkAclToSubnet(),
+        ],
+    )

{cartography-0.95.0 → cartography-0.96.0rc1}/cartography/models/core/common.py

@@ -8,7 +8,14 @@ class PropertyRef:
     (PropertyRef.set_in_kwargs=True).
     """
 
-    def __init__(self, name: str, set_in_kwargs=False, extra_index=False, ignore_case=False):
+    def __init__(
+        self,
+        name: str,
+        set_in_kwargs=False,
+        extra_index=False,
+        ignore_case=False,
+        fuzzy_and_ignore_case=False,
+    ):
         """
         :param name: The name of the property
         :param set_in_kwargs: Optional. If True, the property is not defined on the data dict, and we expect to find the
@@ -33,11 +40,21 @@ class PropertyRef:
             cartography catalog of GitHubUser nodes. Therefore, you would need `ignore_case=True` in the PropertyRef
             that points to the GitHubUser node's name field, otherwise if one of your employees' GitHub usernames
             contains capital letters, you would not be able to map them properly to a GitHubUser node in your graph.
+        :param fuzzy_and_ignore_case: If True, performs a fuzzy + case-insensitive match when comparing the value of
+        this property using the `CONTAINS` operator.
+        query. Defaults to False. This only has effect as part of a TargetNodeMatcher and is not supported for the
+        sub resource relationship.
         """
         self.name = name
         self.set_in_kwargs = set_in_kwargs
         self.extra_index = extra_index
         self.ignore_case = ignore_case
+        self.fuzzy_and_ignore_case = fuzzy_and_ignore_case
+        if self.fuzzy_and_ignore_case and self.ignore_case:
+            raise ValueError(
+                f'Error setting PropertyRef "{self.name}": ignore_case cannot be used together with'
+                'fuzzy_and_ignore_case. Pick one or the other.',
+            )
 
     def _parameterize_name(self) -> str:
         return f"${self.name}"

{cartography-0.95.0 → cartography-0.96.0rc1}/cartography.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cartography
-Version: 0.95.0
+Version: 0.96.0rc1
 Summary: Explore assets and their relationships across your technical infrastructure.
 Home-page: https://www.github.com/cartography-cncf/cartography
 Maintainer: Cartography Contributors

{cartography-0.95.0 → cartography-0.96.0rc1}/cartography.egg-info/SOURCES.txt

@@ -187,6 +187,7 @@ cartography/intel/aws/ec2/key_pairs.py
 cartography/intel/aws/ec2/launch_templates.py
 cartography/intel/aws/ec2/load_balancer_v2s.py
 cartography/intel/aws/ec2/load_balancers.py
+cartography/intel/aws/ec2/network_acls.py
 cartography/intel/aws/ec2/network_interfaces.py
 cartography/intel/aws/ec2/reserved_instances.py
 cartography/intel/aws/ec2/security_groups.py
@@ -298,6 +299,8 @@ cartography/models/aws/ec2/keypairs.py
 cartography/models/aws/ec2/launch_template_versions.py
 cartography/models/aws/ec2/launch_templates.py
 cartography/models/aws/ec2/loadbalancerv2.py
+cartography/models/aws/ec2/network_acl_rules.py
+cartography/models/aws/ec2/network_acls.py
 cartography/models/aws/ec2/networkinterface_instance.py
 cartography/models/aws/ec2/networkinterfaces.py
 cartography/models/aws/ec2/privateip_networkinterface.py

{cartography-0.95.0 → cartography-0.96.0rc1}/setup.py

@@ -1,7 +1,7 @@
 from setuptools import find_packages
 from setuptools import setup
 
-__version__ = '0.95.0'
+__version__ = '0.96.0rc1'
 
 
 setup(