cartography 0.73.1__tar.gz → 0.75.0__tar.gz

{cartography-0.73.1/cartography.egg-info → cartography-0.75.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cartography
-Version: 0.73.1
+Version: 0.75.0
 Summary: Explore assets and their relationships across your technical infrastructure.
 Home-page: https://www.github.com/lyft/cartography
 Maintainer: Lyft

{cartography-0.73.1 → cartography-0.75.0}/cartography/client/core/tx.py

@@ -7,6 +7,9 @@ from typing import Union
 
 import neo4j
 
+from cartography.graph.querybuilder import build_create_index_queries
+from cartography.graph.querybuilder import build_ingestion_query
+from cartography.models.core.nodes import CartographyNodeSchema
 from cartography.util import batch
 
 
@@ -210,3 +213,42 @@ def load_graph_data(
             DictList=data_batch,
             **kwargs,
         )
+
+
+def ensure_indexes(neo4j_session: neo4j.Session, node_schema: CartographyNodeSchema) -> None:
+    """
+    Creates indexes if they don't exist for the given CartographyNodeSchema object, as well as for all of the
+    relationships defined on its `other_relationships` and `sub_resource_relationship` fields. This operation is
+    idempotent.
+
+    This ensures that every time we need to MATCH on a node to draw a relationship to it, the field used for the MATCH
+    will be indexed, making the operation fast.
+    :param neo4j_session: The neo4j session
+    :param node_schema: The node_schema object to create indexes for.
+    """
+    queries = build_create_index_queries(node_schema)
+
+    for query in queries:
+        if not query.startswith('CREATE INDEX IF NOT EXISTS'):
+            raise ValueError('Query provided to `ensure_indexes()` does not start with "CREATE INDEX IF NOT EXISTS".')
+        neo4j_session.run(query)
+
+
+def load(
+        neo4j_session: neo4j.Session,
+        node_schema: CartographyNodeSchema,
+        dict_list: List[Dict[str, Any]],
+        **kwargs,
+) -> None:
+    """
+    Main entrypoint for intel modules to write data to the graph. Ensures that indexes exist for the datatypes loaded
+    to the graph and then performs the load operation.
+    :param neo4j_session: The Neo4j session
+    :param node_schema: The CartographyNodeSchema object to create indexes for and generate a query.
+    :param dict_list: The data to load to the graph represented as a list of dicts.
+    :param kwargs: Allows additional keyword args to be supplied to the Neo4j query.
+    :return: None
+    """
+    ensure_indexes(neo4j_session, node_schema)
+    ingestion_query = build_ingestion_query(node_schema)
+    load_graph_data(neo4j_session, ingestion_query, dict_list, **kwargs)

{cartography-0.73.1 → cartography-0.75.0}/cartography/data/indexes.cypher

@@ -140,6 +140,7 @@ CREATE INDEX IF NOT EXISTS FOR (n:ECSContainerInstance) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:ECSService) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:ECSService) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:ECSTaskDefinition) ON (n.id);
+CREATE INDEX IF NOT EXISTS FOR (n:ECSTaskDefinition) ON (n.arn);
 CREATE INDEX IF NOT EXISTS FOR (n:ECSTaskDefinition) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:ECSTask) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:ECSTask) ON (n.lastupdated);
@@ -158,9 +159,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:ELBListener) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:ELBListener) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:ELBV2Listener) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:ELBV2Listener) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:EMRCluster) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:EMRCluster) ON (n.arn);
-CREATE INDEX IF NOT EXISTS FOR (n:EMRCluster) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:Endpoint) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:Endpoint) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:ESDomain) ON (n.arn);

{cartography-0.73.1 → cartography-0.75.0}/cartography/data/jobs/analysis/aws_ec2_asset_exposure.json

@@ -1,12 +1,12 @@
 {
   "statements": [
   {
-    "query": "MATCH (n) where EXISTS(n.exposed_internet) AND labels(n) IN ['AutoScalingGroup', 'EC2Instance', 'LoadBalancer', 'LoadBalancerV2'] WITH n LIMIT $LIMIT_SIZE REMOVE n.exposed_internet, n.exposed_internet_type return COUNT(*) as TotalCompleted",
+    "query": "MATCH (n) where n.exposed_internet IS NOT NULL AND labels(n) IN ['AutoScalingGroup', 'EC2Instance', 'LoadBalancer', 'LoadBalancerV2'] WITH n LIMIT $LIMIT_SIZE REMOVE n.exposed_internet, n.exposed_internet_type return COUNT(*) as TotalCompleted",
     "iterative": true,
     "iterationsize": 1000
   },
   {
-    "query": "MATCH (:IpRange{id: '0.0.0.0/0'})-[:MEMBER_OF_IP_RULE]->(:IpPermissionInbound)-[:MEMBER_OF_EC2_SECURITY_GROUP]->(group:EC2SecurityGroup)<-[:MEMBER_OF_EC2_SECURITY_GROUP|NETWORK_INTERFACE*..2]-(instance:EC2Instance)\nWITH instance\nWHERE (EXISTS(instance.publicipaddress)) AND (NOT EXISTS(instance.exposed_internet_type)) OR (NOT 'direct' IN instance.exposed_internet_type)\nSET instance.exposed_internet = true, instance.exposed_internet_type = coalesce(instance.exposed_internet_type , []) + 'direct';",
+    "query": "MATCH (:IpRange{id: '0.0.0.0/0'})-[:MEMBER_OF_IP_RULE]->(:IpPermissionInbound)-[:MEMBER_OF_EC2_SECURITY_GROUP]->(group:EC2SecurityGroup)<-[:MEMBER_OF_EC2_SECURITY_GROUP|NETWORK_INTERFACE*..2]-(instance:EC2Instance)\nWITH instance\nWHERE (instance.publicipaddress IS NOT NULL) AND (instance.exposed_internet_type IS NULL) OR (NOT 'direct' IN instance.exposed_internet_type)\nSET instance.exposed_internet = true, instance.exposed_internet_type = coalesce(instance.exposed_internet_type , []) + 'direct';",
     "iterative": false
   },
   {
@@ -18,15 +18,15 @@
     "iterative": false
   },
   {
-    "query": "MATCH (elb:LoadBalancer{exposed_internet: true})-[:EXPOSE]->(e:EC2Instance)\nWITH e\nWHERE (NOT EXISTS(e.exposed_internet_type)) OR (NOT 'elb' IN e.exposed_internet_type)\nSET e.exposed_internet = true, e.exposed_internet_type = coalesce(e.exposed_internet_type, []) + 'elb'",
+    "query": "MATCH (elb:LoadBalancer{exposed_internet: true})-[:EXPOSE]->(e:EC2Instance)\nWITH e\nWHERE (e.exposed_internet_type IS NULL) OR (NOT 'elb' IN e.exposed_internet_type)\nSET e.exposed_internet = true, e.exposed_internet_type = coalesce(e.exposed_internet_type, []) + 'elb'",
     "iterative": false
   },
   {
-    "query": "MATCH (elbv2:LoadBalancerV2{exposed_internet: true})-[:EXPOSE]->(e:EC2Instance)\nWITH e\nWHERE (NOT EXISTS(e.exposed_internet_type)) OR (NOT 'elbv2' IN e.exposed_internet_type)\nSET e.exposed_internet = true, e.exposed_internet_type = coalesce(e.exposed_internet_type, []) + 'elbv2'",
+    "query": "MATCH (elbv2:LoadBalancerV2{exposed_internet: true})-[:EXPOSE]->(e:EC2Instance)\nWITH e\nWHERE (e.exposed_internet_type IS NULL) OR (NOT 'elbv2' IN e.exposed_internet_type)\nSET e.exposed_internet = true, e.exposed_internet_type = coalesce(e.exposed_internet_type, []) + 'elbv2'",
     "iterative": false
   },
   {
-    "query": "MATCH (instance:EC2Instance{exposed_internet: true})-[:MEMBER_AUTO_SCALE_GROUP]->(asg:AutoScalingGroup)\nWITH distinct instance.exposed_internet_type as types, asg\nUNWIND types as type\nWITH type, asg\nWHERE NOT EXISTS(asg.exposed_internet_type) OR (NOT type IN asg.exposed_internet_type)\nSET asg.exposed_internet = true, asg.exposed_internet_type = coalesce(asg.exposed_internet_type, []) + type;",
+    "query": "MATCH (instance:EC2Instance{exposed_internet: true})-[:MEMBER_AUTO_SCALE_GROUP]->(asg:AutoScalingGroup)\nWITH distinct instance.exposed_internet_type as types, asg\nUNWIND types as type\nWITH type, asg\nWHERE asg.exposed_internet_type IS NULL OR (NOT type IN asg.exposed_internet_type)\nSET asg.exposed_internet = true, asg.exposed_internet_type = coalesce(asg.exposed_internet_type, []) + type;",
     "iterative": false
   }
 ],

{cartography-0.73.1 → cartography-0.75.0}/cartography/data/jobs/analysis/aws_ec2_keypair_analysis.json

@@ -3,12 +3,12 @@
     "statements": [
         {
             "__comment__": "Delete the attribute user_uploaded",
-            "query": "MATCH (k:EC2KeyPair) WHERE EXISTS (k.user_uploaded) REMOVE k.user_uploaded return COUNT(*) as TotalCompleted",
+            "query": "MATCH (k:EC2KeyPair) WHERE k.user_uploaded IS NOT NULL REMOVE k.user_uploaded return COUNT(*) as TotalCompleted",
             "iterative": false
         },
         {
             "__comment__": "Delete the attribute duplicate_keyfingerprint",
-            "query": "MATCH (k:EC2KeyPair) WHERE EXISTS (k.duplicate_keyfingerprint) REMOVE k.duplicate_keyfingerprint return COUNT(*) as TotalCompleted",
+            "query": "MATCH (k:EC2KeyPair) WHERE k.duplicate_keyfingerprint IS NOT NULL REMOVE k.duplicate_keyfingerprint return COUNT(*) as TotalCompleted",
             "iterative": false
         },
         {

{cartography-0.73.1 → cartography-0.75.0}/cartography/data/jobs/analysis/aws_eks_asset_exposure.json

@@ -2,7 +2,7 @@
   "statements": [
     {
       "__comment": "This is a clean-up statement to remove custom attributes",
-      "query": "MATCH (cluster:EKSCluster) WHERE EXISTS(cluster.exposed_internet) REMOVE cluster.exposed_internet return COUNT(*) as TotalCompleted",
+      "query": "MATCH (cluster:EKSCluster) WHERE cluster.exposed_internet IS NOT NULL REMOVE cluster.exposed_internet return COUNT(*) as TotalCompleted",
       "iterative": false
     },
     {

{cartography-0.73.1 → cartography-0.75.0}/cartography/data/jobs/analysis/aws_foreign_accounts.json

@@ -2,12 +2,12 @@
   "statements": [
     {
       "__comment": "This analyze AWS accounts we created and tag the ones that are foreign. Foreign accounts are ones that were not in the sync scope",
-      "query": "MATCH (foreign:AWSAccount) where NOT EXISTS(foreign.inscope) SET foreign.foreign = true",
+      "query": "MATCH (foreign:AWSAccount) where foreign.inscope IS NULL SET foreign.foreign = true",
       "iterative": false
     },
     {
       "__comment": "Remove accounts that were set with foreign and inscope. This can happen as we finish the list of sync accounts through assume role mapping and vpc peering",
-      "query": "MATCH (a:AWSAccount) where EXISTS(a.inscope) AND EXISTS(a.foreign) REMOVE a.foreign",
+      "query": "MATCH (a:AWSAccount) where a.inscope IS NOT NULL AND a.foreign IS NOT NULL REMOVE a.foreign",
       "iterative": false
     }
   ],

{cartography-0.73.1 → cartography-0.75.0}/cartography/data/jobs/analysis/gcp_compute_asset_inet_exposure.json

@@ -1,7 +1,7 @@
 {
   "statements": [
   {
-    "query": "MATCH (n) where EXISTS(n.exposed_internet) AND labels(n) IN ['GCPInstance'] WITH n LIMIT $LIMIT_SIZE REMOVE n.exposed_internet, n.exposed_internet_type return COUNT(*) as TotalCompleted",
+    "query": "MATCH (n) where n.exposed_internet IS NOT NULL AND labels(n) IN ['GCPInstance'] WITH n LIMIT $LIMIT_SIZE REMOVE n.exposed_internet, n.exposed_internet_type return COUNT(*) as TotalCompleted",
     "iterative": true,
     "iterationsize": 1000,
     "__comment__": "Delete exposed_internet off nodes so we can start fresh"
@@ -22,17 +22,17 @@
     "__comment__": "Delete stale firewall ingress relationships"
   },
   {
-    "query": "MATCH (ac:GCPNicAccessConfig)<-[:RESOURCE]-(:GCPNetworkInterface)<-[:NETWORK_INTERFACE]-(n:GCPInstance)<-[:FIREWALL_INGRESS]-(firewall_a:GCPFirewall)<-[:ALLOWED_BY]-(allow_rule:GCPIpRule{protocol:'tcp'})<-[:MEMBER_OF_IP_RULE]-(:IpRange{id:\"0.0.0.0/0\"})\nOPTIONAL MATCH (n)<-[:FIREWALL_INGRESS]-(firewall_b:GCPFirewall)<-[:DENIED_BY]-(deny_rule:GCPIpRule{protocol:'tcp'})\nWHERE exists(ac.public_ip) and (\n\tdeny_rule is NULL\n\tOR firewall_b.priority > firewall_a.priority\n\tOR NOT allow_rule.fromport IN RANGE(deny_rule.fromport, deny_rule.toport)\n\tOR NOT allow_rule.toport IN RANGE(deny_rule.fromport, deny_rule.toport)\n)\nSET n.exposed_internet = True, n.exposed_internet_type='direct'\nRETURN count(*) as TotalCompleted",
+    "query": "MATCH (ac:GCPNicAccessConfig)<-[:RESOURCE]-(:GCPNetworkInterface)<-[:NETWORK_INTERFACE]-(n:GCPInstance)<-[:FIREWALL_INGRESS]-(firewall_a:GCPFirewall)<-[:ALLOWED_BY]-(allow_rule:GCPIpRule{protocol:'tcp'})<-[:MEMBER_OF_IP_RULE]-(:IpRange{id:\"0.0.0.0/0\"})\nOPTIONAL MATCH (n)<-[:FIREWALL_INGRESS]-(firewall_b:GCPFirewall)<-[:DENIED_BY]-(deny_rule:GCPIpRule{protocol:'tcp'})\nWHERE ac.public_ip IS NOT NULL and (\n\tdeny_rule is NULL\n\tOR firewall_b.priority > firewall_a.priority\n\tOR NOT allow_rule.fromport IN RANGE(deny_rule.fromport, deny_rule.toport)\n\tOR NOT allow_rule.toport IN RANGE(deny_rule.fromport, deny_rule.toport)\n)\nSET n.exposed_internet = True, n.exposed_internet_type='direct'\nRETURN count(*) as TotalCompleted",
     "iterative": false,
     "__comment__": "Mark a GCP instance with exposed_internet = True and exposed_internet_type = 'direct' if its attached firewalls and TCP rules expose it to the internet."
   },
   {
-    "query": "MATCH (ac:GCPNicAccessConfig)<-[:RESOURCE]-(:GCPNetworkInterface)<-[:NETWORK_INTERFACE]-(n:GCPInstance)<-[:FIREWALL_INGRESS]-(firewall_a:GCPFirewall)<-[:ALLOWED_BY]-(allow_rule:GCPIpRule{protocol:'udp'})<-[:MEMBER_OF_IP_RULE]-(:IpRange{id:\"0.0.0.0/0\"})\nOPTIONAL MATCH (n)<-[:FIREWALL_INGRESS]-(firewall_b:GCPFirewall)<-[:DENIED_BY]-(deny_rule:GCPIpRule{protocol:'udp'})\nWHERE exists(ac.public_ip) and (\n\tdeny_rule is NULL\n\tOR firewall_b.priority > firewall_a.priority\n\tOR NOT allow_rule.fromport IN RANGE(deny_rule.fromport, deny_rule.toport)\n\tOR NOT allow_rule.toport IN RANGE(deny_rule.fromport, deny_rule.toport)\n)\nSET n.exposed_internet = True, n.exposed_internet_type='direct'\nRETURN count(*) as TotalCompleted",
+    "query": "MATCH (ac:GCPNicAccessConfig)<-[:RESOURCE]-(:GCPNetworkInterface)<-[:NETWORK_INTERFACE]-(n:GCPInstance)<-[:FIREWALL_INGRESS]-(firewall_a:GCPFirewall)<-[:ALLOWED_BY]-(allow_rule:GCPIpRule{protocol:'udp'})<-[:MEMBER_OF_IP_RULE]-(:IpRange{id:\"0.0.0.0/0\"})\nOPTIONAL MATCH (n)<-[:FIREWALL_INGRESS]-(firewall_b:GCPFirewall)<-[:DENIED_BY]-(deny_rule:GCPIpRule{protocol:'udp'})\nWHERE ac.public_ip IS NOT NULL and (\n\tdeny_rule is NULL\n\tOR firewall_b.priority > firewall_a.priority\n\tOR NOT allow_rule.fromport IN RANGE(deny_rule.fromport, deny_rule.toport)\n\tOR NOT allow_rule.toport IN RANGE(deny_rule.fromport, deny_rule.toport)\n)\nSET n.exposed_internet = True, n.exposed_internet_type='direct'\nRETURN count(*) as TotalCompleted",
     "iterative": false,
     "__comment__": "Mark a GCP instance with exposed_internet = True and exposed_internet_type = 'direct' if its attached firewalls and UDP rules expose it to the internet."
   },
   {
-    "query": "MATCH (ac:GCPNicAccessConfig)<-[:RESOURCE]-(:GCPNetworkInterface)<-[:NETWORK_INTERFACE]-(n:GCPInstance)<-[:FIREWALL_INGRESS]-(firewall_a:GCPFirewall)<-[:ALLOWED_BY]-(allow_rule:GCPIpRule{protocol:'all'})<-[:MEMBER_OF_IP_RULE]-(:IpRange{id:\"0.0.0.0/0\"})\nOPTIONAL MATCH (n)<-[:FIREWALL_INGRESS]-(firewall_b:GCPFirewall)<-[:DENIED_BY]-(deny_rule:GCPIpRule{protocol:'all'})\nWHERE exists(ac.public_ip) and exists(allow_rule.fromport) and exists(allow_rule.toport) and (\n\tdeny_rule is NULL\n\tOR firewall_b.priority > firewall_a.priority\n\tOR NOT allow_rule.fromport IN RANGE(deny_rule.fromport, deny_rule.toport)\n\tOR NOT allow_rule.toport IN RANGE(deny_rule.fromport, deny_rule.toport)\n)\nSET n.exposed_internet = True, n.exposed_internet_type='direct'\nRETURN count(*) as TotalCompleted",
+    "query": "MATCH (ac:GCPNicAccessConfig)<-[:RESOURCE]-(:GCPNetworkInterface)<-[:NETWORK_INTERFACE]-(n:GCPInstance)<-[:FIREWALL_INGRESS]-(firewall_a:GCPFirewall)<-[:ALLOWED_BY]-(allow_rule:GCPIpRule{protocol:'all'})<-[:MEMBER_OF_IP_RULE]-(:IpRange{id:\"0.0.0.0/0\"})\nOPTIONAL MATCH (n)<-[:FIREWALL_INGRESS]-(firewall_b:GCPFirewall)<-[:DENIED_BY]-(deny_rule:GCPIpRule{protocol:'all'})\nWHERE ac.public_ip IS NOT NULL and allow_rule.fromport IS NOT NULL and allow_rule.toport IS NOT NULL and (\n\tdeny_rule is NULL\n\tOR firewall_b.priority > firewall_a.priority\n\tOR NOT allow_rule.fromport IN RANGE(deny_rule.fromport, deny_rule.toport)\n\tOR NOT allow_rule.toport IN RANGE(deny_rule.fromport, deny_rule.toport)\n)\nSET n.exposed_internet = True, n.exposed_internet_type='direct'\nRETURN count(*) as TotalCompleted",
     "iterative": false,
     "__comment__": "Mark a GCP instance with exposed_internet = True and exposed_internet_type = 'direct' if its attached firewalls and ALL rules expose it to the internet."
   }

{cartography-0.73.1 → cartography-0.75.0}/cartography/data/jobs/analysis/gcp_gke_asset_exposure.json

@@ -2,7 +2,7 @@
   "statements": [
     {
       "__comment": "This is a clean-up statement to remove custom attributes",
-      "query": "MATCH (cluster:GKECluster) WHERE EXISTS(cluster.exposed_internet) REMOVE cluster.exposed_internet return COUNT(*) as TotalCompleted",
+      "query": "MATCH (cluster:GKECluster) WHERE cluster.exposed_internet IS NOT NULL REMOVE cluster.exposed_internet return COUNT(*) as TotalCompleted",
       "iterative": false
     },
     {

cartography-0.75.0/cartography/data/jobs/analysis/gcp_gke_basic_auth.json

@@ -0,0 +1,15 @@
+{
+  "statements": [
+    {
+      "__comment": "This is a clean-up statement to remove custom attributes",
+      "query": "MATCH (cluster:GKECluster) WHERE cluster.basic_auth IS NOT NULL REMOVE cluster.basic_auth return COUNT(*) as TotalCompleted",
+      "iterative": false
+    },
+    {
+      "__comment": "This sets the basic_auth attribute",
+      "query": "MATCH (cluster:GKECluster) WHERE (cluster.masterauth_username IS NOT NULL AND NOT cluster.masterauth_username = '') AND (cluster.masterauth_password IS NOT NULL AND NOT cluster.masterauth.password = '') SET cluster.basic_auth = true",
+      "iterative": false
+    }
+  ],
+  "name": "GCP GKE basic authentication exposure"
+}

{cartography-0.73.1 → cartography-0.75.0}/cartography/data/jobs/cleanup/aws_apigateway_details.json

@@ -1,7 +1,7 @@
 {
   "statements": [
     {
-      "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(s:RestAPI) WHERE EXISTS(s.anonymous_access)\n WITH s LIMIT $LIMIT_SIZE\nREMOVE s.anonymous_access, s.anonymous_actions",
+      "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(s:RestAPI) WHERE s.anonymous_access IS NOT NULL\n WITH s LIMIT $LIMIT_SIZE\nREMOVE s.anonymous_access, s.anonymous_actions",
       "iterative": true,
       "iterationsize": 100
     }

{cartography-0.73.1 → cartography-0.75.0}/cartography/data/jobs/cleanup/aws_kms_details.json

@@ -1,7 +1,7 @@
 {
   "statements": [
     {
-      "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(s:KMSKey) WHERE EXISTS(s.anonymous_access)\n WITH s LIMIT $LIMIT_SIZE\nREMOVE s.anonymous_access, s.anonymous_actions",
+      "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(s:KMSKey) WHERE s.anonymous_access IS NOT NULL\n WITH s LIMIT $LIMIT_SIZE\nREMOVE s.anonymous_access, s.anonymous_actions",
       "iterative": true,
       "iterationsize": 100
     }

{cartography-0.73.1 → cartography-0.75.0}/cartography/data/jobs/cleanup/aws_s3_details.json

@@ -1,7 +1,7 @@
 {
   "statements": [
     {
-      "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(s:S3Bucket) WHERE EXISTS(s.anonymous_access)\n WITH s LIMIT $LIMIT_SIZE\nREMOVE s.anonymous_access, s.anonymous_actions",
+      "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(s:S3Bucket) WHERE s.anonymous_access IS NOT NULL\n WITH s LIMIT $LIMIT_SIZE\nREMOVE s.anonymous_access, s.anonymous_actions",
       "iterative": true,
       "iterationsize": 100
     }

{cartography-0.73.1 → cartography-0.75.0}/cartography/graph/cleanupbuilder.py

@@ -1,11 +1,8 @@
 from dataclasses import asdict
 from string import Template
 from typing import List
-from typing import Optional
-from typing import Set
 
 from cartography.graph.querybuilder import _build_match_clause
-from cartography.graph.querybuilder import filter_selected_relationships
 from cartography.graph.querybuilder import rel_present_on_node_schema
 from cartography.models.core.nodes import CartographyNodeSchema
 from cartography.models.core.relationships import CartographyRelSchema
@@ -13,49 +10,32 @@ from cartography.models.core.relationships import LinkDirection
 from cartography.models.core.relationships import TargetNodeMatcher
 
 
-def build_cleanup_queries(
-        node_schema: CartographyNodeSchema,
-        selected_rels: Optional[Set[CartographyRelSchema]] = None,
-) -> List[str]:
+def build_cleanup_queries(node_schema: CartographyNodeSchema) -> List[str]:
     """
     Generates queries to clean up stale nodes and relationships from the given CartographyNodeSchema.
-    :param node_schema: The given CartographyNodeSchema to generate cleanup queries for.
-    :param selected_rels: Optional. If specified, only generate cleanup queries where the `node_schema` is bound to this
-    given set of selected relationships. Raises an exception if any of the rels in `selected_rels` aren't actually
-    defined on the `node_schema`.
-    If `selected_rels` is not specified (default), we generate cleanup queries against all relationships defined on the
-    `node_schema`.
-    :return: A list of Neo4j queries to clean up nodes and relationships. Order matters: we always clean up the sub
-    resource relationship last because we only clean up stale nodes and rels that are associated with a given sub
-    resource, so if we delete the sub resource first then we will not be able to reach the stale nodes and rels, thus
-    leaving orphaned objects behind.
-    Note also that we return the empty list if the node_schema has no relationships. Doing cleanups of nodes without
-    relationships can be resource expensive for a large graph, and you might risk deleting unintended objects. Please
-    write a manual cleanup job if you wish to do this.
+    Note that auto-cleanups for a node with no relationships is not currently supported.
+    Algorithm:
+    1. First delete all stale nodes attached to the node_schema's sub resource
+    2. Delete all stale node to sub resource relationships
+        - We don't expect this to be very common (never for AWS resources, at least), but in case it is possible for an
+          asset to change sub resources, we want to handle it properly.
+    3. For all relationships defined on the node schema, delete all stale ones.
+    :param node_schema: The given CartographyNodeSchema
+    :return: A list of Neo4j queries to clean up nodes and relationships.
     """
-    other_rels = node_schema.other_relationships
-    sub_resource_rel = node_schema.sub_resource_relationship
-
-    if selected_rels:
-        # Ensure that the selected rels actually exist on the node_schema
-        sub_resource_rel, other_rels = filter_selected_relationships(node_schema, selected_rels)
-
-    if not sub_resource_rel:
+    if not node_schema.sub_resource_relationship:
         raise ValueError(
             "Auto-creating a cleanup job for a node_schema without a sub resource relationship is not supported. "
-            f'Please check the class definition of "{node_schema.__class__.__name__}". If the optional `selected_rels` '
-            'param was specified to build_cleanup_queries(), then ensure that the sub resource relationship is '
-            'present.',
+            f'Please check the class definition of "{node_schema.__class__.__name__}".',
         )
 
-    result = []
-    if other_rels:
-        for rel in other_rels.rels:
-            result.extend(_build_cleanup_node_and_rel_queries(node_schema, rel))
+    result = _build_cleanup_node_and_rel_queries(node_schema, node_schema.sub_resource_relationship)
+    if node_schema.other_relationships:
+        for rel in node_schema.other_relationships.rels:
+            # [0] is the delete node query, [1] is the delete relationship query. We only want the latter.
+            _, rel_query = _build_cleanup_node_and_rel_queries(node_schema, rel)
+            result.append(rel_query)
 
-    # Make sure that the sub resource cleanup job is last in the list; order matters.
-    result.extend(_build_cleanup_node_and_rel_queries(node_schema, sub_resource_rel))
-    # Note that auto-cleanups for a node with no relationships does not happen at all - we don't support it.
     return result
 
 

{cartography-0.73.1 → cartography-0.75.0}/cartography/graph/job.py

@@ -16,7 +16,6 @@ from cartography.graph.cleanupbuilder import build_cleanup_queries
 from cartography.graph.statement import get_job_shortname
 from cartography.graph.statement import GraphStatement
 from cartography.models.core.nodes import CartographyNodeSchema
-from cartography.models.core.relationships import CartographyRelSchema
 
 logger = logging.getLogger(__name__)
 
@@ -129,14 +128,13 @@ class GraphJob:
             cls,
             node_schema: CartographyNodeSchema,
             parameters: Dict[str, Any],
-            selected_rels: Optional[Set[CartographyRelSchema]] = None,
     ) -> 'GraphJob':
         """
         Create a cleanup job from a CartographyNodeSchema object.
         For a given node, the fields used in the node_schema.sub_resource_relationship.target_node_node_matcher.keys()
         must be provided as keys and values in the params dict.
         """
-        queries: List[str] = build_cleanup_queries(node_schema, selected_rels)
+        queries: List[str] = build_cleanup_queries(node_schema)
 
         expected_param_keys: Set[str] = get_parameters(queries)
         actual_param_keys: Set[str] = set(parameters.keys())

{cartography-0.73.1 → cartography-0.75.0}/cartography/graph/querybuilder.py

@@ -2,6 +2,7 @@ import logging
 from dataclasses import asdict
 from string import Template
 from typing import Dict
+from typing import List
 from typing import Optional
 from typing import Set
 from typing import Tuple
@@ -108,6 +109,28 @@ def _build_match_clause(matcher: TargetNodeMatcher) -> str:
     return ', '.join(match.safe_substitute(Key=key, PropRef=prop_ref) for key, prop_ref in matcher_asdict.items())
 
 
+def _build_where_clause_for_rel_match(node_var: str, matcher: TargetNodeMatcher) -> str:
+    """
+    Same as _build_match_clause, but puts the matching logic in a WHERE clause.
+    This is intended specifically to use for joining with relationships where we need a case-insensitive match.
+    :param matcher: A TargetNodeMatcher object
+    :return: a Neo4j where clause
+    """
+    match = Template("$node_var.$key = $prop_ref")
+    case_insensitive_match = Template("toLower($node_var.$key) = toLower($prop_ref)")
+
+    matcher_asdict = asdict(matcher)
+
+    result = []
+    for key, prop_ref in matcher_asdict.items():
+        if prop_ref.ignore_case:
+            prop_line = case_insensitive_match.safe_substitute(node_var=node_var, key=key, prop_ref=prop_ref)
+        else:
+            prop_line = match.safe_substitute(node_var=node_var, key=key, prop_ref=prop_ref)
+        result.append(prop_line)
+    return ' AND\n'.join(result)
+
+
 def _asdict_with_validate_relprops(link: CartographyRelSchema) -> Dict[str, PropertyRef]:
     """
     Give a helpful error message when forgetting to put `()` when instantiating a CartographyRelSchema, as this
@@ -145,6 +168,7 @@ def _build_attach_sub_resource_statement(sub_resource_link: Optional[Cartography
 
     sub_resource_attach_template = Template(
         """
+        WITH i, item
         OPTIONAL MATCH (j:$SubResourceLabel{$MatchClause})
         WITH i, item, j WHERE j IS NOT NULL
         $RelMergeClause
@@ -191,7 +215,9 @@ def _build_attach_additional_links_statement(
     additional_links_template = Template(
         """
         WITH i, item
-        OPTIONAL MATCH ($node_var:$AddlLabel{$MatchClause})
+        OPTIONAL MATCH ($node_var:$AddlLabel)
+        WHERE
+            $WhereClause
         WITH i, item, $node_var WHERE $node_var IS NOT NULL
         $RelMerge
         ON CREATE SET $rel_var.firstseen = timestamp()
@@ -219,7 +245,7 @@ def _build_attach_additional_links_statement(
 
         additional_ref = additional_links_template.safe_substitute(
             AddlLabel=link.target_node_label,
-            MatchClause=_build_match_clause(link.target_node_matcher),
+            WhereClause=_build_where_clause_for_rel_match(node_var, link.target_node_matcher),
             node_var=node_var,
             rel_var=rel_var,
             RelMerge=rel_merge,
@@ -258,7 +284,6 @@ def _build_attach_relationships_statement(
         """
         WITH i, item
         CALL {
-            WITH i, item
             $attach_relationships_statement
         }
         """,
@@ -374,3 +399,55 @@ def build_ingestion_query(
         attach_relationships_statement=_build_attach_relationships_statement(sub_resource_rel, other_rels),
     )
     return ingest_query
+
+
+def build_create_index_queries(node_schema: CartographyNodeSchema) -> List[str]:
+    """
+    Generate queries to create indexes for the given CartographyNodeSchema and all node types attached to it via its
+    relationships.
+    :param node_schema: The Cartography node_schema object
+    :return: A list of queries of the form `CREATE INDEX IF NOT EXISTS FOR (n:$TargetNodeLabel) ON (n.$TargetAttribute)`
+    """
+    index_template = Template('CREATE INDEX IF NOT EXISTS FOR (n:$TargetNodeLabel) ON (n.$TargetAttribute);')
+
+    # First ensure an index exists for the node_schema and all extra labels on the `id` and `lastupdated` fields
+    result = [
+        index_template.safe_substitute(
+            TargetNodeLabel=node_schema.label,
+            TargetAttribute='id',
+        ),
+        index_template.safe_substitute(
+            TargetNodeLabel=node_schema.label,
+            TargetAttribute='lastupdated',
+        ),
+    ]
+    if node_schema.extra_node_labels:
+        result.extend([
+            index_template.safe_substitute(
+                TargetNodeLabel=label,
+                TargetAttribute='id',  # Precondition: 'id' is defined on all cartography node_schema objects.
+            ) for label in node_schema.extra_node_labels.labels
+        ])
+
+    # Next, for all relationships possible out of this node, ensure that indexes exist for all target nodes' properties
+    # as specified in their TargetNodeMatchers.
+    rel_schemas = []
+    if node_schema.sub_resource_relationship:
+        rel_schemas.extend([node_schema.sub_resource_relationship])
+    if node_schema.other_relationships:
+        rel_schemas.extend(node_schema.other_relationships.rels)
+    for rs in rel_schemas:
+        for target_key in asdict(rs.target_node_matcher).keys():
+            result.append(
+                index_template.safe_substitute(TargetNodeLabel=rs.target_node_label, TargetAttribute=target_key),
+            )
+
+    # Now, include extra indexes defined by the module author on the node schema's property refs.
+    node_props_as_dict: Dict[str, PropertyRef] = asdict(node_schema.properties)
+    result.extend([
+        index_template.safe_substitute(
+            TargetNodeLabel=node_schema.label,
+            TargetAttribute=prop_name,
+        ) for prop_name, prop_ref in node_props_as_dict.items() if prop_ref.extra_index
+    ])
+    return result

{cartography-0.73.1 → cartography-0.75.0}/cartography/graph/statement.py

@@ -4,6 +4,7 @@ import os
 from pathlib import Path
 from typing import Any
 from typing import Dict
+from typing import Optional
 from typing import Union
 
 import neo4j
@@ -40,8 +41,13 @@ class GraphStatement:
     """
 
     def __init__(
-        self, query: str, parameters: Dict = None, iterative: bool = False, iterationsize: int = 0,
-        parent_job_name: str = None, parent_job_sequence_num: int = None,
+            self,
+            query: str,
+            parameters: Optional[Dict[Any, Any]] = None,
+            iterative: bool = False,
+            iterationsize: int = 0,
+            parent_job_name: Optional[str] = None,
+            parent_job_sequence_num: Optional[int] = None,
     ):
         self.query = query
         self.parameters = parameters or {}
@@ -122,7 +128,12 @@ class GraphStatement:
             result.consume()
 
     @classmethod
-    def create_from_json(cls, json_obj: Dict, short_job_name: str = None, job_sequence_num: int = None):
+    def create_from_json(
+            cls,
+            json_obj: Dict[str, Any],
+            short_job_name: Optional[str] = None,
+            job_sequence_num: Optional[int] = None,
+    ):
         """
         Create a statement from a JSON blob.
         """

{cartography-0.73.1 → cartography-0.75.0}/cartography/intel/aws/apigateway.py

@@ -171,7 +171,7 @@ def _load_apigateway_policies(
 def _set_default_values(neo4j_session: neo4j.Session, aws_account_id: str) -> None:
     set_defaults = """
     MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(restApi:APIGatewayRestAPI)
-    where NOT EXISTS(restApi.anonymous_actions)
+    where restApi.anonymous_actions IS NULL
     SET restApi.anonymous_access = false, restApi.anonymous_actions = []
     """
 

{cartography-0.73.1 → cartography-0.75.0}/cartography/intel/aws/ecs.py

@@ -91,16 +91,16 @@ def get_ecs_services(cluster_arn: str, boto3_session: boto3.session.Session, reg
 
 @timeit
 @aws_handle_regions
-def get_ecs_task_definitions(boto3_session: boto3.session.Session, region: str) -> List[Dict[str, Any]]:
+def get_ecs_task_definitions(
+    boto3_session: boto3.session.Session,
+    region: str,
+    tasks: List[Dict[str, Any]],
+) -> List[Dict[str, Any]]:
     client = boto3_session.client('ecs', region_name=region)
-    paginator = client.get_paginator('list_task_definitions')
     task_definitions: List[Dict[str, Any]] = []
-    task_definition_arns: List[str] = []
-    for page in paginator.paginate():
-        task_definition_arns.extend(page.get('taskDefinitionArns', []))
-    for arn in task_definition_arns:
+    for task in tasks:
         task_definition = client.describe_task_definition(
-            taskDefinition=arn,
+            taskDefinition=task['taskDefinitionArn'],
         )
         task_definitions.append(task_definition['taskDefinition'])
     return task_definitions
@@ -294,7 +294,8 @@ def load_ecs_task_definitions(
     UNWIND $Definitions AS def
         MERGE (d:ECSTaskDefinition{id: def.taskDefinitionArn})
         ON CREATE SET d.firstseen = timestamp()
-        SET d.arn = def.taskDefinitionArn, d.region = $Region,
+        SET d.arn = def.taskDefinitionArn,
+            d.region = $Region,
             d.family = def.family,
             d.task_role_arn = def.taskRoleArn,
             d.execution_role_arn = def.executionRoleArn,
@@ -317,6 +318,11 @@ def load_ecs_task_definitions(
             d.ephemeral_storage_size_in_gib = def.ephemeralStorage.sizeInGiB,
             d.lastupdated = $aws_update_tag
         WITH d
+        MATCH (task:ECSTask{task_definition_arn: d.arn})
+        MERGE (task)-[r:HAS_TASK_DEFINITION]->(d)
+        ON CREATE SET r.firstseen = timestamp()
+        SET r.lastupdated = $aws_update_tag
+        WITH d
         MATCH (owner:AWSAccount{id: $AWS_ACCOUNT_ID})
         MERGE (owner)-[r:RESOURCE]->(d)
         ON CREATE SET r.firstseen = timestamp()
@@ -565,17 +571,6 @@ def sync(
                 current_aws_account_id,
                 update_tag,
             )
-            task_definitions = get_ecs_task_definitions(
-                boto3_session,
-                region,
-            )
-            load_ecs_task_definitions(
-                neo4j_session,
-                task_definitions,
-                region,
-                current_aws_account_id,
-                update_tag,
-            )
             services = get_ecs_services(
                 cluster_arn,
                 boto3_session,
@@ -602,4 +597,16 @@ def sync(
                 current_aws_account_id,
                 update_tag,
             )
+            task_definitions = get_ecs_task_definitions(
+                boto3_session,
+                region,
+                tasks,
+            )
+            load_ecs_task_definitions(
+                neo4j_session,
+                task_definitions,
+                region,
+                current_aws_account_id,
+                update_tag,
+            )
     cleanup_ecs(neo4j_session, common_job_parameters)