cartography 0.104.0rc2__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/data/jobs/cleanup/github_repos_cleanup.json

@@ -19,6 +19,7 @@
     "iterative": true,
     "iterationsize": 100
   },
+
   {
     "query": "MATCH (:GitHubBranch)-[r:BRANCH]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
     "iterative": true,
@@ -39,6 +40,7 @@
     "iterative": true,
     "iterationsize": 100
   },
+
   {
     "query": "MATCH (:GitHubUser)-[r:OUTSIDE_COLLAB_ADMIN]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
     "iterative": true,

cartography/driftdetect/cli.py

@@ -63,8 +63,9 @@ class CLI:
             default="bolt://localhost:7687",
             help=(
                 "A valid Neo4j URI to sync against. See "
-                "https://neo4j.com/docs/api/python-driver/current/driver.html#uri for complete documentation on the "
-                "structure of a Neo4j URI."
+                "https://neo4j.com/docs/browser-manual/current/operations/dbms-connection/#uri-scheme for "
+                "documentation on the structure of a Neo4j URI, and "
+                "https://neo4j.com/docs/api/python-driver/current/ for complete documentation on the Python driver."
             ),
         )
         parser_get_state.add_argument(

cartography/graph/cleanupbuilder.py

@@ -3,6 +3,7 @@ from string import Template
 from typing import Dict
 from typing import List
 
+from cartography.graph.querybuilder import _asdict_with_validate_relprops
 from cartography.graph.querybuilder import _build_match_clause
 from cartography.graph.querybuilder import rel_present_on_node_schema
 from cartography.models.core.common import PropertyRef
@@ -15,33 +16,40 @@ from cartography.models.core.relationships import TargetNodeMatcher
 def build_cleanup_queries(node_schema: CartographyNodeSchema) -> List[str]:
     """
     Generates queries to clean up stale nodes and relationships from the given CartographyNodeSchema.
+    Properly handles cases where a node schema has a scoped cleanup or not.
     Note that auto-cleanups for a node with no relationships is not currently supported.
-
-    Algorithm:
-    1. If node_schema has no relationships at all, return empty.
-
-    Otherwise,
-
-    1. If node_schema doesn't have a sub_resource relationship, generate queries only to clean up its other
-    relationships. No nodes will be cleaned up.
-
-    Otherwise,
-
-    1. First delete all stale nodes attached to the node_schema's sub resource
-    2. Delete all stale node to sub resource relationships
-        - We don't expect this to be very common (never for AWS resources, at least), but in case it is possible for an
-          asset to change sub resources, we want to handle it properly.
-    3. For all relationships defined on the node schema, delete all stale ones.
     :param node_schema: The given CartographyNodeSchema
     :return: A list of Neo4j queries to clean up nodes and relationships.
     """
+    # If the node has no relationships, do not delete the node. Leave this behind for the user to manage.
+    # Oftentimes these are SyncMetadata nodes.
     if (
         not node_schema.sub_resource_relationship
         and not node_schema.other_relationships
     ):
         return []
 
-    if not node_schema.sub_resource_relationship:
+    # Case 1 [Standard]: the node has a sub resource and scoped cleanup is true => clean up stale nodes
+    # of this type, scoped to the sub resource. Continue on to clean up the other_relationships too.
+    if node_schema.sub_resource_relationship and node_schema.scoped_cleanup:
+        queries = _build_cleanup_node_and_rel_queries(
+            node_schema,
+            node_schema.sub_resource_relationship,
+        )
+
+    # Case 2: The node has a sub resource but scoped cleanup is false => this does not make sense
+    # because if have a sub resource, we are implying that we are doing scoped cleanup.
+    elif node_schema.sub_resource_relationship and not node_schema.scoped_cleanup:
+        raise ValueError(
+            f"This is not expected: {node_schema.label} has a sub_resource_relationship but scoped_cleanup=False."
+            "Please check the class definition for this node schema. It doesn't make sense for a node to have a "
+            "sub resource relationship and an unscoped cleanup. Doing this will cause all stale nodes of this type "
+            "to be deleted regardless of the sub resource they are attached to."
+        )
+
+    # Case 3: The node has no sub resource but scoped cleanup is true => do not delete any nodes, but clean up stale relationships.
+    # Return early.
+    elif not node_schema.sub_resource_relationship and node_schema.scoped_cleanup:
         queries = []
         other_rels = (
             node_schema.other_relationships.rels
@@ -53,17 +61,20 @@ def build_cleanup_queries(node_schema: CartographyNodeSchema) -> List[str]:
             queries.append(query)
         return queries
 
-    result = _build_cleanup_node_and_rel_queries(
-        node_schema,
-        node_schema.sub_resource_relationship,
-    )
+    # Case 4: The node has no sub resource and scoped cleanup is false => clean up the stale nodes. Continue on to clean up the other_relationships too.
+    else:
+        queries = [_build_cleanup_node_query_unscoped(node_schema)]
+
     if node_schema.other_relationships:
         for rel in node_schema.other_relationships.rels:
-            # [0] is the delete node query, [1] is the delete relationship query. We only want the latter.
-            _, rel_query = _build_cleanup_node_and_rel_queries(node_schema, rel)
-            result.append(rel_query)
+            if node_schema.scoped_cleanup:
+                # [0] is the delete node query, [1] is the delete relationship query. We only want the latter.
+                _, rel_query = _build_cleanup_node_and_rel_queries(node_schema, rel)
+                queries.append(rel_query)
+            else:
+                queries.append(_build_cleanup_rel_queries_unscoped(node_schema, rel))
 
-    return result
+    return queries
 
 
 def _build_cleanup_rel_query_no_sub_resource(
@@ -94,6 +105,46 @@ def _build_cleanup_rel_query_no_sub_resource(
     )
 
 
+def _build_match_statement_for_cleanup(node_schema: CartographyNodeSchema) -> str:
+    """
+    Helper function to build a MATCH statement for a given node schema for cleanup.
+    """
+    if not node_schema.sub_resource_relationship and not node_schema.scoped_cleanup:
+        template = Template("MATCH (n:$node_label)")
+        return template.safe_substitute(
+            node_label=node_schema.label,
+        )
+
+    # if it has a sub resource relationship defined, we need to match on the sub resource to make sure we only delete
+    # nodes that are attached to the sub resource.
+    template = Template(
+        "MATCH (n:$node_label)$sub_resource_link(:$sub_resource_label{$match_sub_res_clause})"
+    )
+    sub_resource_link = ""
+    sub_resource_label = ""
+    match_sub_res_clause = ""
+
+    if node_schema.sub_resource_relationship:
+        # Draw sub resource rel with correct direction
+        if node_schema.sub_resource_relationship.direction == LinkDirection.INWARD:
+            sub_resource_link_template = Template("<-[s:$SubResourceRelLabel]-")
+        else:
+            sub_resource_link_template = Template("-[s:$SubResourceRelLabel]->")
+        sub_resource_link = sub_resource_link_template.safe_substitute(
+            SubResourceRelLabel=node_schema.sub_resource_relationship.rel_label,
+        )
+        sub_resource_label = node_schema.sub_resource_relationship.target_node_label
+        match_sub_res_clause = _build_match_clause(
+            node_schema.sub_resource_relationship.target_node_matcher,
+        )
+    return template.safe_substitute(
+        node_label=node_schema.label,
+        sub_resource_link=sub_resource_link,
+        sub_resource_label=sub_resource_label,
+        match_sub_res_clause=match_sub_res_clause,
+    )
+
+
 def _build_cleanup_node_and_rel_queries(
     node_schema: CartographyNodeSchema,
     selected_relationship: CartographyRelSchema,
@@ -120,15 +171,6 @@ def _build_cleanup_node_and_rel_queries(
             "verify the node class definition for the relationships that it has.",
         )
 
-    # Draw sub resource rel with correct direction
-    if node_schema.sub_resource_relationship.direction == LinkDirection.INWARD:
-        sub_resource_link_template = Template("<-[s:$SubResourceRelLabel]-")
-    else:
-        sub_resource_link_template = Template("-[s:$SubResourceRelLabel]->")
-    sub_resource_link = sub_resource_link_template.safe_substitute(
-        SubResourceRelLabel=node_schema.sub_resource_relationship.rel_label,
-    )
-
     # The cleanup node query must always be before the cleanup rel query
     delete_action_clauses = [
         """
@@ -161,19 +203,14 @@ def _build_cleanup_node_and_rel_queries(
     # Ensure the node is attached to the sub resource and delete the node
     query_template = Template(
         """
-        MATCH (n:$node_label)$sub_resource_link(:$sub_resource_label{$match_sub_res_clause})
+        $match_statement
         $selected_rel_clause
         $delete_action_clause
         """,
     )
     return [
         query_template.safe_substitute(
-            node_label=node_schema.label,
-            sub_resource_link=sub_resource_link,
-            sub_resource_label=node_schema.sub_resource_relationship.target_node_label,
-            match_sub_res_clause=_build_match_clause(
-                node_schema.sub_resource_relationship.target_node_matcher,
-            ),
+            match_statement=_build_match_statement_for_cleanup(node_schema),
             selected_rel_clause=(
                 ""
                 if selected_relationship == node_schema.sub_resource_relationship
@@ -185,6 +222,80 @@ def _build_cleanup_node_and_rel_queries(
     ]
 
 
+def _build_cleanup_node_query_unscoped(
+    node_schema: CartographyNodeSchema,
+) -> str:
+    """
+    Generates a cleanup query for a node_schema to allow unscoped cleanup.
+    """
+    if node_schema.scoped_cleanup:
+        raise ValueError(
+            f"_build_cleanup_node_query_for_unscoped_cleanup() failed: '{node_schema.label}' does not have "
+            "scoped_cleanup=False, so we cannot generate a query to clean it up. Please verify that the class "
+            "definition is what you expect.",
+        )
+
+    # The cleanup node query must always be before the cleanup rel query
+    delete_action_clause = """
+        WHERE n.lastupdated <> $UPDATE_TAG
+        WITH n LIMIT $LIMIT_SIZE
+        DETACH DELETE n;
+    """
+
+    # Ensure the node is attached to the sub resource and delete the node
+    query_template = Template(
+        """
+        $match_statement
+        $delete_action_clause
+        """,
+    )
+    return query_template.safe_substitute(
+        match_statement=_build_match_statement_for_cleanup(node_schema),
+        delete_action_clause=delete_action_clause,
+    )
+
+
+def _build_cleanup_rel_queries_unscoped(
+    node_schema: CartographyNodeSchema,
+    selected_relationship: CartographyRelSchema,
+) -> str:
+    """
+    Generates relationship cleanup query for a node_schema with scoped_cleanup=False.
+    """
+    if node_schema.scoped_cleanup:
+        raise ValueError(
+            f"_build_cleanup_node_and_rel_queries_unscoped() failed: '{node_schema.label}' does not have "
+            "scoped_cleanup=False, so we cannot generate a query to clean it up. Please verify that the class "
+            "definition is what you expect.",
+        )
+    if not rel_present_on_node_schema(node_schema, selected_relationship):
+        raise ValueError(
+            f"_build_cleanup_node_query(): Attempted to build cleanup query for node '{node_schema.label}' and "
+            f"relationship {selected_relationship.rel_label} but that relationship is not present on the node. Please "
+            "verify the node class definition for the relationships that it has.",
+        )
+
+    # The cleanup node query must always be before the cleanup rel query
+    delete_action_clause = """WHERE r.lastupdated <> $UPDATE_TAG
+        WITH r LIMIT $LIMIT_SIZE
+        DELETE r;
+        """
+
+    # Ensure the node is attached to the sub resource and delete the node
+    query_template = Template(
+        """
+        $match_statement
+        $selected_rel_clause
+        $delete_action_clause
+        """,
+    )
+    return query_template.safe_substitute(
+        match_statement=_build_match_statement_for_cleanup(node_schema),
+        selected_rel_clause=_build_selected_rel_clause(selected_relationship),
+        delete_action_clause=delete_action_clause,
+    )
+
+
 def _build_selected_rel_clause(selected_relationship: CartographyRelSchema) -> str:
     """
     Draw selected relationship with correct direction. Returns a string that looks like either
@@ -224,3 +335,49 @@ def _validate_target_node_matcher_for_cleanup_job(tgm: TargetNodeMatcher):
                 f"{key} has set_in_kwargs=False, please check by reviewing the full stack trace to know which object"
                 f"this message was raised from. Debug information: PropertyRef name = {prop_ref.name}.",
             )
+
+
+def build_cleanup_query_for_matchlink(rel_schema: CartographyRelSchema) -> str:
+    """
+    Generates a cleanup query for a matchlink relationship.
+    :param rel_schema: The CartographyRelSchema object to generate a query. This CartographyRelSchema object
+    - Must have a source_node_matcher and source_node_label defined
+    - Must have a CartographyRelProperties object where _sub_resource_label and _sub_resource_id are defined
+    :return: A Neo4j query used to clean up stale matchlink relationships.
+    """
+    if not rel_schema.source_node_matcher:
+        raise ValueError(
+            f"No source node matcher found for {rel_schema.rel_label}; returning empty list."
+        )
+
+    query_template = Template(
+        """
+        MATCH (from:$source_node_label)$rel_direction[r:$rel_label]$rel_direction_end(to:$target_node_label)
+        WHERE r.lastupdated <> $UPDATE_TAG
+            AND r._sub_resource_label = $sub_resource_label
+            AND r._sub_resource_id = $sub_resource_id
+        WITH r LIMIT $LIMIT_SIZE
+        DELETE r;
+        """
+    )
+
+    # Determine which way to point the arrow. INWARD is toward the source, otherwise we go toward the target.
+    if rel_schema.direction == LinkDirection.INWARD:
+        rel_direction = "<-"
+        rel_direction_end = "-"
+    else:
+        rel_direction = "-"
+        rel_direction_end = "->"
+
+    # Small hack: avoid type-checking errors by converting the rel_schema to a dict.
+    rel_props_as_dict = _asdict_with_validate_relprops(rel_schema)
+
+    return query_template.safe_substitute(
+        source_node_label=rel_schema.source_node_label,
+        target_node_label=rel_schema.target_node_label,
+        rel_label=rel_schema.rel_label,
+        rel_direction=rel_direction,
+        rel_direction_end=rel_direction_end,
+        sub_resource_label=rel_props_as_dict["_sub_resource_label"],
+        sub_resource_id=rel_props_as_dict["_sub_resource_id"],
+    )

cartography/graph/job.py

@@ -13,9 +13,11 @@ from typing import Union
 import neo4j
 
 from cartography.graph.cleanupbuilder import build_cleanup_queries
+from cartography.graph.cleanupbuilder import build_cleanup_query_for_matchlink
 from cartography.graph.statement import get_job_shortname
 from cartography.graph.statement import GraphStatement
 from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.relationships import CartographyRelSchema
 
 logger = logging.getLogger(__name__)
 
@@ -123,11 +125,13 @@ class GraphJob:
         }
 
     @classmethod
-    def from_json(cls, blob: str, short_name: Optional[str] = None) -> "GraphJob":
+    def from_json(
+        cls, blob: Union[str, dict], short_name: Optional[str] = None
+    ) -> "GraphJob":
         """
-        Create a job from a JSON blob.
+        Create a job from a JSON dict or blob.
         """
-        data: Dict = json.loads(blob)
+        data = json.loads(blob) if isinstance(blob, str) else blob
         statements = _get_statements_from_json(data, short_name)
         name = data["name"]
         return cls(name, statements, short_name)
@@ -137,11 +141,13 @@ class GraphJob:
         cls,
         node_schema: CartographyNodeSchema,
         parameters: Dict[str, Any],
+        iterationsize: int = 100,
     ) -> "GraphJob":
         """
         Create a cleanup job from a CartographyNodeSchema object.
         For a given node, the fields used in the node_schema.sub_resource_relationship.target_node_node_matcher.keys()
         must be provided as keys and values in the params dict.
+        :param iterationsize: The number of items to process in each iteration. Defaults to 100.
         """
         queries: List[str] = build_cleanup_queries(node_schema)
 
@@ -163,7 +169,7 @@ class GraphJob:
                 query,
                 parameters=parameters,
                 iterative=True,
-                iterationsize=100,
+                iterationsize=iterationsize,
                 parent_job_name=node_schema.label,
                 parent_job_sequence_num=idx,
             )
@@ -176,6 +182,48 @@ class GraphJob:
             node_schema.label,
         )
 
+    @classmethod
+    def from_matchlink(
+        cls,
+        rel_schema: CartographyRelSchema,
+        sub_resource_label: str,
+        sub_resource_id: str,
+        update_tag: int,
+        iterationsize: int = 100,
+    ) -> "GraphJob":
+        """
+        Create a cleanup job from a CartographyRelSchema object (specifically, a MatchLink).
+        This is used for cleaning up stale links between nodes created by load_rels(). Do not use for other purposes.
+
+        Other notes:
+        - For a given rel_schema, the fields used in the rel_schema.properties._sub_resource_label.name and
+        rel_schema.properties._sub_resource_id.name must be provided as keys and values in the params dict.
+        - The rel_schema must have a source_node_matcher and target_node_matcher.
+        :param iterationsize: The number of items to process in each iteration. Defaults to 100.
+        """
+        cleanup_link_query = build_cleanup_query_for_matchlink(rel_schema)
+        logger.debug(f"Cleanup query: {cleanup_link_query}")
+
+        parameters = {
+            "UPDATE_TAG": update_tag,
+            "_sub_resource_label": sub_resource_label,
+            "_sub_resource_id": sub_resource_id,
+        }
+
+        statement = GraphStatement(
+            cleanup_link_query,
+            parameters=parameters,
+            iterative=True,
+            iterationsize=iterationsize,
+            parent_job_name=rel_schema.rel_label,
+        )
+
+        return cls(
+            f"Cleanup {rel_schema.rel_label} between {rel_schema.source_node_label} and {rel_schema.target_node_label}",
+            [statement],
+            rel_schema.rel_label,
+        )
+
     @classmethod
     def from_json_file(cls, file_path: Union[str, Path]) -> "GraphJob":
         """
@@ -196,12 +244,12 @@ class GraphJob:
     def run_from_json(
         cls,
         neo4j_session: neo4j.Session,
-        blob: str,
+        blob: Union[str, dict],
         parameters: Dict,
         short_name: Optional[str] = None,
     ) -> None:
         """
-        Run a job from a JSON blob. This will deserialize the job and execute all statements sequentially.
+        Run a job from a JSON dict or blob. This will deserialize the job and execute all statements sequentially.
         """
         if not parameters:
             parameters = {}