cartography 0.105.0__py3-none-any.whl → 0.106.0rc2__py3-none-any.whl

cartography/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.105.0'
-__version_tuple__ = version_tuple = (0, 105, 0)
+__version__ = version = '0.106.0rc2'
+__version_tuple__ = version_tuple = (0, 106, 0, 'rc2')

cartography/client/core/tx.py

@@ -1,3 +1,4 @@
+import logging
 from typing import Any
 from typing import Dict
 from typing import List
@@ -8,10 +9,15 @@ from typing import Union
 import neo4j
 
 from cartography.graph.querybuilder import build_create_index_queries
+from cartography.graph.querybuilder import build_create_index_queries_for_matchlink
 from cartography.graph.querybuilder import build_ingestion_query
+from cartography.graph.querybuilder import build_matchlink_query
 from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.relationships import CartographyRelSchema
 from cartography.util import batch
 
+logger = logging.getLogger(__name__)
+
 
 def read_list_of_values_tx(
     tx: neo4j.Transaction,
@@ -255,6 +261,25 @@ def ensure_indexes(
         neo4j_session.run(query)
 
 
+def ensure_indexes_for_matchlinks(
+    neo4j_session: neo4j.Session,
+    rel_schema: CartographyRelSchema,
+) -> None:
+    """
+    Creates indexes for node fields if they don't exist for the given CartographyRelSchema object.
+    This is only used for load_rels() where we match on and connect existing nodes.
+    This is not used for CartographyNodeSchema objects.
+    """
+    queries = build_create_index_queries_for_matchlink(rel_schema)
+    logger.debug(f"CREATE INDEX queries for {rel_schema.rel_label}: {queries}")
+    for query in queries:
+        if not query.startswith("CREATE INDEX IF NOT EXISTS"):
+            raise ValueError(
+                'Query provided to `ensure_indexes_for_matchlinks()` does not start with "CREATE INDEX IF NOT EXISTS".',
+            )
+        neo4j_session.run(query)
+
+
 def load(
     neo4j_session: neo4j.Session,
     node_schema: CartographyNodeSchema,
@@ -276,3 +301,40 @@ def load(
     ensure_indexes(neo4j_session, node_schema)
     ingestion_query = build_ingestion_query(node_schema)
     load_graph_data(neo4j_session, ingestion_query, dict_list, **kwargs)
+
+
+def load_matchlinks(
+    neo4j_session: neo4j.Session,
+    rel_schema: CartographyRelSchema,
+    dict_list: list[dict[str, Any]],
+    **kwargs,
+) -> None:
+    """
+    Main entrypoint for intel modules to write relationships to the graph between two existing nodes.
+    :param neo4j_session: The Neo4j session
+    :param rel_schema: The CartographyRelSchema object to generate a query.
+    :param dict_list: The data to load to the graph represented as a list of dicts. The dicts must contain the source and
+    target node ids.
+    :param kwargs: Allows additional keyword args to be supplied to the Neo4j query.
+    :return: None
+    """
+    if len(dict_list) == 0:
+        # If there is no data to load, save some time.
+        return
+
+    # Validate that required kwargs are provided for cleanup queries
+    if "_sub_resource_label" not in kwargs:
+        raise ValueError(
+            f"Required kwarg '_sub_resource_label' not provided for {rel_schema.rel_label}. "
+            "This is needed for cleanup queries."
+        )
+    if "_sub_resource_id" not in kwargs:
+        raise ValueError(
+            f"Required kwarg '_sub_resource_id' not provided for {rel_schema.rel_label}. "
+            "This is needed for cleanup queries."
+        )
+
+    ensure_indexes_for_matchlinks(neo4j_session, rel_schema)
+    matchlink_query = build_matchlink_query(rel_schema)
+    logger.debug(f"Matchlink query: {matchlink_query}")
+    load_graph_data(neo4j_session, matchlink_query, dict_list, **kwargs)

cartography/data/indexes.cypher

@@ -99,21 +99,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:ECRRepositoryImage) ON (n.tag);
 CREATE INDEX IF NOT EXISTS FOR (n:ECRRepositoryImage) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:ECRScanFinding) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:ECRScanFinding) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSCluster) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSCluster) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSContainerInstance) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSContainerInstance) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSService) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSService) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSTaskDefinition) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSTaskDefinition) ON (n.arn);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSTaskDefinition) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSTask) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSTask) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSContainerDefinition) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSContainerDefinition) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSContainer) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:ECSContainer) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:ElasticacheCluster) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:ElasticacheCluster) ON (n.arn);
 CREATE INDEX IF NOT EXISTS FOR (n:ElasticacheCluster) ON (n.lastupdated);
@@ -375,22 +360,3 @@ CREATE INDEX IF NOT EXISTS FOR (n:AzureDisk) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:AzureDisk) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:AzureSnapshot) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:AzureSnapshot) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesCluster) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesCluster) ON (n.name);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesCluster) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesNamespace) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesNamespace) ON (n.name);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesNamespace) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesPod) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesPod) ON (n.name);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesPod) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesContainer) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesContainer) ON (n.name);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesContainer) ON (n.image);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesContainer) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesSecret) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesSecret) ON (n.name);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesSecret) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesService) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesService) ON (n.name);
-CREATE INDEX IF NOT EXISTS FOR (n:KubernetesService) ON (n.lastupdated);

cartography/graph/cleanupbuilder.py

@@ -3,6 +3,7 @@ from string import Template
 from typing import Dict
 from typing import List
 
+from cartography.graph.querybuilder import _asdict_with_validate_relprops
 from cartography.graph.querybuilder import _build_match_clause
 from cartography.graph.querybuilder import rel_present_on_node_schema
 from cartography.models.core.common import PropertyRef
@@ -334,3 +335,49 @@ def _validate_target_node_matcher_for_cleanup_job(tgm: TargetNodeMatcher):
                 f"{key} has set_in_kwargs=False, please check by reviewing the full stack trace to know which object"
                 f"this message was raised from. Debug information: PropertyRef name = {prop_ref.name}.",
             )
+
+
+def build_cleanup_query_for_matchlink(rel_schema: CartographyRelSchema) -> str:
+    """
+    Generates a cleanup query for a matchlink relationship.
+    :param rel_schema: The CartographyRelSchema object to generate a query. This CartographyRelSchema object
+    - Must have a source_node_matcher and source_node_label defined
+    - Must have a CartographyRelProperties object where _sub_resource_label and _sub_resource_id are defined
+    :return: A Neo4j query used to clean up stale matchlink relationships.
+    """
+    if not rel_schema.source_node_matcher:
+        raise ValueError(
+            f"No source node matcher found for {rel_schema.rel_label}; returning empty list."
+        )
+
+    query_template = Template(
+        """
+        MATCH (from:$source_node_label)$rel_direction[r:$rel_label]$rel_direction_end(to:$target_node_label)
+        WHERE r.lastupdated <> $UPDATE_TAG
+            AND r._sub_resource_label = $sub_resource_label
+            AND r._sub_resource_id = $sub_resource_id
+        WITH r LIMIT $LIMIT_SIZE
+        DELETE r;
+        """
+    )
+
+    # Determine which way to point the arrow. INWARD is toward the source, otherwise we go toward the target.
+    if rel_schema.direction == LinkDirection.INWARD:
+        rel_direction = "<-"
+        rel_direction_end = "-"
+    else:
+        rel_direction = "-"
+        rel_direction_end = "->"
+
+    # Small hack: avoid type-checking errors by converting the rel_schema to a dict.
+    rel_props_as_dict = _asdict_with_validate_relprops(rel_schema)
+
+    return query_template.safe_substitute(
+        source_node_label=rel_schema.source_node_label,
+        target_node_label=rel_schema.target_node_label,
+        rel_label=rel_schema.rel_label,
+        rel_direction=rel_direction,
+        rel_direction_end=rel_direction_end,
+        sub_resource_label=rel_props_as_dict["_sub_resource_label"],
+        sub_resource_id=rel_props_as_dict["_sub_resource_id"],
+    )

cartography/graph/job.py

@@ -13,9 +13,11 @@ from typing import Union
 import neo4j
 
 from cartography.graph.cleanupbuilder import build_cleanup_queries
+from cartography.graph.cleanupbuilder import build_cleanup_query_for_matchlink
 from cartography.graph.statement import get_job_shortname
 from cartography.graph.statement import GraphStatement
 from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.relationships import CartographyRelSchema
 
 logger = logging.getLogger(__name__)
 
@@ -176,6 +178,46 @@ class GraphJob:
             node_schema.label,
         )
 
+    @classmethod
+    def from_matchlink(
+        cls,
+        rel_schema: CartographyRelSchema,
+        sub_resource_label: str,
+        sub_resource_id: str,
+        update_tag: int,
+    ) -> "GraphJob":
+        """
+        Create a cleanup job from a CartographyRelSchema object (specifically, a MatchLink).
+        This is used for cleaning up stale links between nodes created by load_rels(). Do not use for other purposes.
+
+        Other notes:
+        - For a given rel_schema, the fields used in the rel_schema.properties._sub_resource_label.name and
+        rel_schema.properties._sub_resource_id.name must be provided as keys and values in the params dict.
+        - The rel_schema must have a source_node_matcher and target_node_matcher.
+        """
+        cleanup_link_query = build_cleanup_query_for_matchlink(rel_schema)
+        logger.debug(f"Cleanup query: {cleanup_link_query}")
+
+        parameters = {
+            "UPDATE_TAG": update_tag,
+            "_sub_resource_label": sub_resource_label,
+            "_sub_resource_id": sub_resource_id,
+        }
+
+        statement = GraphStatement(
+            cleanup_link_query,
+            parameters=parameters,
+            iterative=True,
+            iterationsize=100,
+            parent_job_name=rel_schema.rel_label,
+        )
+
+        return cls(
+            f"Cleanup {rel_schema.rel_label} between {rel_schema.source_node_label} and {rel_schema.target_node_label}",
+            [statement],
+            rel_schema.rel_label,
+        )
+
     @classmethod
     def from_json_file(cls, file_path: Union[str, Path]) -> "GraphJob":
         """

cartography/graph/querybuilder.py

@@ -14,6 +14,7 @@ from cartography.models.core.nodes import ExtraNodeLabels
 from cartography.models.core.relationships import CartographyRelSchema
 from cartography.models.core.relationships import LinkDirection
 from cartography.models.core.relationships import OtherRelationships
+from cartography.models.core.relationships import SourceNodeMatcher
 from cartography.models.core.relationships import TargetNodeMatcher
 
 logger = logging.getLogger(__name__)
@@ -109,10 +110,10 @@ def _build_rel_properties_statement(
     return set_clause
 
 
-def _build_match_clause(matcher: TargetNodeMatcher) -> str:
+def _build_match_clause(matcher: TargetNodeMatcher | SourceNodeMatcher) -> str:
     """
     Generate a Neo4j match statement on one or more keys and values for a given node.
-    :param matcher: A TargetNodeMatcher object
+    :param matcher: A TargetNodeMatcher or SourceNodeMatcher object
     :return: a Neo4j match clause
     """
     match = Template("$Key: $PropRef")
@@ -548,3 +549,136 @@ def build_create_index_queries(node_schema: CartographyNodeSchema) -> List[str]:
         ],
     )
     return result
+
+
+def build_create_index_queries_for_matchlink(
+    rel_schema: CartographyRelSchema,
+) -> list[str]:
+    """
+    Generate queries to create indexes for the given CartographyRelSchema and all node types attached to it via its
+    relationships.
+    :param rel_schema: The CartographyRelSchema object
+    :return: A list of queries of the form `CREATE INDEX IF NOT EXISTS FOR (n:$TargetNodeLabel) ON (n.$TargetAttribute)`
+    """
+    if not rel_schema.source_node_matcher:
+        logger.warning(
+            f"No source node matcher found for {rel_schema.rel_label}; returning empty list."
+            "Please note that build_create_index_queries_for_matchlink() is only used for load_matchlinks() where we match on "
+            "and connect existing nodes in the graph."
+        )
+        return []
+
+    index_template = Template(
+        "CREATE INDEX IF NOT EXISTS FOR (n:$NodeLabel) ON (n.$NodeAttribute);",
+    )
+
+    result = []
+    for source_key in asdict(rel_schema.source_node_matcher).keys():
+        result.append(
+            index_template.safe_substitute(
+                NodeLabel=rel_schema.source_node_label,
+                NodeAttribute=source_key,
+            ),
+        )
+    for target_key in asdict(rel_schema.target_node_matcher).keys():
+        result.append(
+            index_template.safe_substitute(
+                NodeLabel=rel_schema.target_node_label,
+                NodeAttribute=target_key,
+            ),
+        )
+
+    # Create a composite index for the relationship between the source and target nodes.
+    # https://neo4j.com/docs/cypher-manual/4.3/indexes-for-search-performance/#administration-indexes-create-a-composite-index-for-relationships
+    rel_index_template = Template(
+        "CREATE INDEX IF NOT EXISTS FOR ()$rel_direction[r:$RelLabel]$rel_direction_end() "
+        "ON (r.lastupdated, r._sub_resource_label, r._sub_resource_id);",
+    )
+    if rel_schema.direction == LinkDirection.INWARD:
+        result.append(
+            rel_index_template.safe_substitute(
+                RelLabel=rel_schema.rel_label,
+                rel_direction="<-",
+                rel_direction_end="-",
+            )
+        )
+    else:
+        result.append(
+            rel_index_template.safe_substitute(
+                RelLabel=rel_schema.rel_label,
+                rel_direction="-",
+                rel_direction_end="->",
+            )
+        )
+    return result
+
+
+def build_matchlink_query(rel_schema: CartographyRelSchema) -> str:
+    """
+    Generate a Neo4j query to link two existing nodes when given a CartographyRelSchema object.
+    This is only used for load_matchlinks().
+    :param rel_schema: The CartographyRelSchema object to generate a query. This CartographyRelSchema object
+    - Must have a source_node_matcher and source_node_label defined
+    - Must have a CartographyRelProperties object where _sub_resource_label and _sub_resource_id are defined
+    :return: A Neo4j query that can be used to link two existing nodes.
+    """
+    if not rel_schema.source_node_matcher or not rel_schema.source_node_label:
+        raise ValueError(
+            f"No source node matcher or source node label found for {rel_schema.rel_label}. "
+            "MatchLink relationships require a source_node_matcher and source_node_label to be defined."
+        )
+
+    rel_props_as_dict = _asdict_with_validate_relprops(rel_schema)
+
+    # These are needed for the cleanup query
+    if "_sub_resource_label" not in rel_props_as_dict:
+        raise ValueError(
+            f"Expected _sub_resource_label to be defined on {rel_schema.properties.__class__.__name__}"
+            "Please include `_sub_resource_label: PropertyRef = PropertyRef('_sub_resource_label', set_in_kwargs=True)`"
+        )
+    if "_sub_resource_id" not in rel_props_as_dict:
+        raise ValueError(
+            f"Expected _sub_resource_id to be defined on {rel_schema.properties.__class__.__name__}"
+            "Please include `_sub_resource_id: PropertyRef = PropertyRef('_sub_resource_id', set_in_kwargs=True)`"
+        )
+
+    matchlink_query_template = Template(
+        """
+        UNWIND $DictList as item
+            $source_match
+            $target_match
+            MERGE $rel
+            ON CREATE SET r.firstseen = timestamp()
+            SET
+                $set_rel_properties_statement;
+        """
+    )
+
+    source_match = Template(
+        "MATCH (from:$source_node_label{$match_clause})"
+    ).safe_substitute(
+        source_node_label=rel_schema.source_node_label,
+        match_clause=_build_match_clause(rel_schema.source_node_matcher),
+    )
+
+    target_match = Template(
+        "MATCH (to:$target_node_label{$match_clause})"
+    ).safe_substitute(
+        target_node_label=rel_schema.target_node_label,
+        match_clause=_build_match_clause(rel_schema.target_node_matcher),
+    )
+
+    if rel_schema.direction == LinkDirection.INWARD:
+        rel = f"(from)<-[r:{rel_schema.rel_label}]-(to)"
+    else:
+        rel = f"(from)-[r:{rel_schema.rel_label}]->(to)"
+
+    return matchlink_query_template.safe_substitute(
+        source_match=source_match,
+        target_match=target_match,
+        rel=rel,
+        set_rel_properties_statement=_build_rel_properties_statement(
+            "r",
+            rel_props_as_dict,
+        ),
+    )

cartography/graph/statement.py

@@ -56,7 +56,7 @@ class GraphStatement:
 
         self.parent_job_name = parent_job_name if parent_job_name else None
         self.parent_job_sequence_num = (
-            parent_job_sequence_num if parent_job_sequence_num else None
+            parent_job_sequence_num if parent_job_sequence_num else 1
         )
 
     def merge_parameters(self, parameters: Dict) -> None: