cartography 0.106.0rc1__py3-none-any.whl → 0.106.0rc2__py3-none-any.whl

cartography/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.106.0rc1'
-__version_tuple__ = version_tuple = (0, 106, 0, 'rc1')
+__version__ = version = '0.106.0rc2'
+__version_tuple__ = version_tuple = (0, 106, 0, 'rc2')

cartography/client/core/tx.py

@@ -1,3 +1,4 @@
+import logging
 from typing import Any
 from typing import Dict
 from typing import List
@@ -8,10 +9,15 @@ from typing import Union
 import neo4j
 
 from cartography.graph.querybuilder import build_create_index_queries
+from cartography.graph.querybuilder import build_create_index_queries_for_matchlink
 from cartography.graph.querybuilder import build_ingestion_query
+from cartography.graph.querybuilder import build_matchlink_query
 from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.relationships import CartographyRelSchema
 from cartography.util import batch
 
+logger = logging.getLogger(__name__)
+
 
 def read_list_of_values_tx(
     tx: neo4j.Transaction,
@@ -255,6 +261,25 @@ def ensure_indexes(
         neo4j_session.run(query)
 
 
+def ensure_indexes_for_matchlinks(
+    neo4j_session: neo4j.Session,
+    rel_schema: CartographyRelSchema,
+) -> None:
+    """
+    Creates indexes for node fields if they don't exist for the given CartographyRelSchema object.
+    This is only used for load_rels() where we match on and connect existing nodes.
+    This is not used for CartographyNodeSchema objects.
+    """
+    queries = build_create_index_queries_for_matchlink(rel_schema)
+    logger.debug(f"CREATE INDEX queries for {rel_schema.rel_label}: {queries}")
+    for query in queries:
+        if not query.startswith("CREATE INDEX IF NOT EXISTS"):
+            raise ValueError(
+                'Query provided to `ensure_indexes_for_matchlinks()` does not start with "CREATE INDEX IF NOT EXISTS".',
+            )
+        neo4j_session.run(query)
+
+
 def load(
     neo4j_session: neo4j.Session,
     node_schema: CartographyNodeSchema,
@@ -276,3 +301,40 @@ def load(
     ensure_indexes(neo4j_session, node_schema)
     ingestion_query = build_ingestion_query(node_schema)
     load_graph_data(neo4j_session, ingestion_query, dict_list, **kwargs)
+
+
+def load_matchlinks(
+    neo4j_session: neo4j.Session,
+    rel_schema: CartographyRelSchema,
+    dict_list: list[dict[str, Any]],
+    **kwargs,
+) -> None:
+    """
+    Main entrypoint for intel modules to write relationships to the graph between two existing nodes.
+    :param neo4j_session: The Neo4j session
+    :param rel_schema: The CartographyRelSchema object to generate a query.
+    :param dict_list: The data to load to the graph represented as a list of dicts. The dicts must contain the source and
+    target node ids.
+    :param kwargs: Allows additional keyword args to be supplied to the Neo4j query.
+    :return: None
+    """
+    if len(dict_list) == 0:
+        # If there is no data to load, save some time.
+        return
+
+    # Validate that required kwargs are provided for cleanup queries
+    if "_sub_resource_label" not in kwargs:
+        raise ValueError(
+            f"Required kwarg '_sub_resource_label' not provided for {rel_schema.rel_label}. "
+            "This is needed for cleanup queries."
+        )
+    if "_sub_resource_id" not in kwargs:
+        raise ValueError(
+            f"Required kwarg '_sub_resource_id' not provided for {rel_schema.rel_label}. "
+            "This is needed for cleanup queries."
+        )
+
+    ensure_indexes_for_matchlinks(neo4j_session, rel_schema)
+    matchlink_query = build_matchlink_query(rel_schema)
+    logger.debug(f"Matchlink query: {matchlink_query}")
+    load_graph_data(neo4j_session, matchlink_query, dict_list, **kwargs)

cartography/graph/cleanupbuilder.py

@@ -3,6 +3,7 @@ from string import Template
 from typing import Dict
 from typing import List
 
+from cartography.graph.querybuilder import _asdict_with_validate_relprops
 from cartography.graph.querybuilder import _build_match_clause
 from cartography.graph.querybuilder import rel_present_on_node_schema
 from cartography.models.core.common import PropertyRef
@@ -334,3 +335,49 @@ def _validate_target_node_matcher_for_cleanup_job(tgm: TargetNodeMatcher):
                 f"{key} has set_in_kwargs=False, please check by reviewing the full stack trace to know which object"
                 f"this message was raised from. Debug information: PropertyRef name = {prop_ref.name}.",
             )
+
+
+def build_cleanup_query_for_matchlink(rel_schema: CartographyRelSchema) -> str:
+    """
+    Generates a cleanup query for a matchlink relationship.
+    :param rel_schema: The CartographyRelSchema object to generate a query. This CartographyRelSchema object
+    - Must have a source_node_matcher and source_node_label defined
+    - Must have a CartographyRelProperties object where _sub_resource_label and _sub_resource_id are defined
+    :return: A Neo4j query used to clean up stale matchlink relationships.
+    """
+    if not rel_schema.source_node_matcher:
+        raise ValueError(
+            f"No source node matcher found for {rel_schema.rel_label}; returning empty list."
+        )
+
+    query_template = Template(
+        """
+        MATCH (from:$source_node_label)$rel_direction[r:$rel_label]$rel_direction_end(to:$target_node_label)
+        WHERE r.lastupdated <> $UPDATE_TAG
+            AND r._sub_resource_label = $sub_resource_label
+            AND r._sub_resource_id = $sub_resource_id
+        WITH r LIMIT $LIMIT_SIZE
+        DELETE r;
+        """
+    )
+
+    # Determine which way to point the arrow. INWARD is toward the source, otherwise we go toward the target.
+    if rel_schema.direction == LinkDirection.INWARD:
+        rel_direction = "<-"
+        rel_direction_end = "-"
+    else:
+        rel_direction = "-"
+        rel_direction_end = "->"
+
+    # Small hack: avoid type-checking errors by converting the rel_schema to a dict.
+    rel_props_as_dict = _asdict_with_validate_relprops(rel_schema)
+
+    return query_template.safe_substitute(
+        source_node_label=rel_schema.source_node_label,
+        target_node_label=rel_schema.target_node_label,
+        rel_label=rel_schema.rel_label,
+        rel_direction=rel_direction,
+        rel_direction_end=rel_direction_end,
+        sub_resource_label=rel_props_as_dict["_sub_resource_label"],
+        sub_resource_id=rel_props_as_dict["_sub_resource_id"],
+    )

cartography/graph/job.py

@@ -13,9 +13,11 @@ from typing import Union
 import neo4j
 
 from cartography.graph.cleanupbuilder import build_cleanup_queries
+from cartography.graph.cleanupbuilder import build_cleanup_query_for_matchlink
 from cartography.graph.statement import get_job_shortname
 from cartography.graph.statement import GraphStatement
 from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.relationships import CartographyRelSchema
 
 logger = logging.getLogger(__name__)
 
@@ -176,6 +178,46 @@ class GraphJob:
             node_schema.label,
         )
 
+    @classmethod
+    def from_matchlink(
+        cls,
+        rel_schema: CartographyRelSchema,
+        sub_resource_label: str,
+        sub_resource_id: str,
+        update_tag: int,
+    ) -> "GraphJob":
+        """
+        Create a cleanup job from a CartographyRelSchema object (specifically, a MatchLink).
+        This is used for cleaning up stale links between nodes created by load_rels(). Do not use for other purposes.
+
+        Other notes:
+        - For a given rel_schema, the fields used in the rel_schema.properties._sub_resource_label.name and
+        rel_schema.properties._sub_resource_id.name must be provided as keys and values in the params dict.
+        - The rel_schema must have a source_node_matcher and target_node_matcher.
+        """
+        cleanup_link_query = build_cleanup_query_for_matchlink(rel_schema)
+        logger.debug(f"Cleanup query: {cleanup_link_query}")
+
+        parameters = {
+            "UPDATE_TAG": update_tag,
+            "_sub_resource_label": sub_resource_label,
+            "_sub_resource_id": sub_resource_id,
+        }
+
+        statement = GraphStatement(
+            cleanup_link_query,
+            parameters=parameters,
+            iterative=True,
+            iterationsize=100,
+            parent_job_name=rel_schema.rel_label,
+        )
+
+        return cls(
+            f"Cleanup {rel_schema.rel_label} between {rel_schema.source_node_label} and {rel_schema.target_node_label}",
+            [statement],
+            rel_schema.rel_label,
+        )
+
     @classmethod
     def from_json_file(cls, file_path: Union[str, Path]) -> "GraphJob":
         """

cartography/graph/querybuilder.py

@@ -14,6 +14,7 @@ from cartography.models.core.nodes import ExtraNodeLabels
 from cartography.models.core.relationships import CartographyRelSchema
 from cartography.models.core.relationships import LinkDirection
 from cartography.models.core.relationships import OtherRelationships
+from cartography.models.core.relationships import SourceNodeMatcher
 from cartography.models.core.relationships import TargetNodeMatcher
 
 logger = logging.getLogger(__name__)
@@ -109,10 +110,10 @@ def _build_rel_properties_statement(
     return set_clause
 
 
-def _build_match_clause(matcher: TargetNodeMatcher) -> str:
+def _build_match_clause(matcher: TargetNodeMatcher | SourceNodeMatcher) -> str:
     """
     Generate a Neo4j match statement on one or more keys and values for a given node.
-    :param matcher: A TargetNodeMatcher object
+    :param matcher: A TargetNodeMatcher or SourceNodeMatcher object
     :return: a Neo4j match clause
     """
     match = Template("$Key: $PropRef")
@@ -548,3 +549,136 @@ def build_create_index_queries(node_schema: CartographyNodeSchema) -> List[str]:
         ],
     )
     return result
+
+
+def build_create_index_queries_for_matchlink(
+    rel_schema: CartographyRelSchema,
+) -> list[str]:
+    """
+    Generate queries to create indexes for the given CartographyRelSchema and all node types attached to it via its
+    relationships.
+    :param rel_schema: The CartographyRelSchema object
+    :return: A list of queries of the form `CREATE INDEX IF NOT EXISTS FOR (n:$TargetNodeLabel) ON (n.$TargetAttribute)`
+    """
+    if not rel_schema.source_node_matcher:
+        logger.warning(
+            f"No source node matcher found for {rel_schema.rel_label}; returning empty list."
+            "Please note that build_create_index_queries_for_matchlink() is only used for load_matchlinks() where we match on "
+            "and connect existing nodes in the graph."
+        )
+        return []
+
+    index_template = Template(
+        "CREATE INDEX IF NOT EXISTS FOR (n:$NodeLabel) ON (n.$NodeAttribute);",
+    )
+
+    result = []
+    for source_key in asdict(rel_schema.source_node_matcher).keys():
+        result.append(
+            index_template.safe_substitute(
+                NodeLabel=rel_schema.source_node_label,
+                NodeAttribute=source_key,
+            ),
+        )
+    for target_key in asdict(rel_schema.target_node_matcher).keys():
+        result.append(
+            index_template.safe_substitute(
+                NodeLabel=rel_schema.target_node_label,
+                NodeAttribute=target_key,
+            ),
+        )
+
+    # Create a composite index for the relationship between the source and target nodes.
+    # https://neo4j.com/docs/cypher-manual/4.3/indexes-for-search-performance/#administration-indexes-create-a-composite-index-for-relationships
+    rel_index_template = Template(
+        "CREATE INDEX IF NOT EXISTS FOR ()$rel_direction[r:$RelLabel]$rel_direction_end() "
+        "ON (r.lastupdated, r._sub_resource_label, r._sub_resource_id);",
+    )
+    if rel_schema.direction == LinkDirection.INWARD:
+        result.append(
+            rel_index_template.safe_substitute(
+                RelLabel=rel_schema.rel_label,
+                rel_direction="<-",
+                rel_direction_end="-",
+            )
+        )
+    else:
+        result.append(
+            rel_index_template.safe_substitute(
+                RelLabel=rel_schema.rel_label,
+                rel_direction="-",
+                rel_direction_end="->",
+            )
+        )
+    return result
+
+
+def build_matchlink_query(rel_schema: CartographyRelSchema) -> str:
+    """
+    Generate a Neo4j query to link two existing nodes when given a CartographyRelSchema object.
+    This is only used for load_matchlinks().
+    :param rel_schema: The CartographyRelSchema object to generate a query. This CartographyRelSchema object
+    - Must have a source_node_matcher and source_node_label defined
+    - Must have a CartographyRelProperties object where _sub_resource_label and _sub_resource_id are defined
+    :return: A Neo4j query that can be used to link two existing nodes.
+    """
+    if not rel_schema.source_node_matcher or not rel_schema.source_node_label:
+        raise ValueError(
+            f"No source node matcher or source node label found for {rel_schema.rel_label}. "
+            "MatchLink relationships require a source_node_matcher and source_node_label to be defined."
+        )
+
+    rel_props_as_dict = _asdict_with_validate_relprops(rel_schema)
+
+    # These are needed for the cleanup query
+    if "_sub_resource_label" not in rel_props_as_dict:
+        raise ValueError(
+            f"Expected _sub_resource_label to be defined on {rel_schema.properties.__class__.__name__}"
+            "Please include `_sub_resource_label: PropertyRef = PropertyRef('_sub_resource_label', set_in_kwargs=True)`"
+        )
+    if "_sub_resource_id" not in rel_props_as_dict:
+        raise ValueError(
+            f"Expected _sub_resource_id to be defined on {rel_schema.properties.__class__.__name__}"
+            "Please include `_sub_resource_id: PropertyRef = PropertyRef('_sub_resource_id', set_in_kwargs=True)`"
+        )
+
+    matchlink_query_template = Template(
+        """
+        UNWIND $DictList as item
+            $source_match
+            $target_match
+            MERGE $rel
+            ON CREATE SET r.firstseen = timestamp()
+            SET
+                $set_rel_properties_statement;
+        """
+    )
+
+    source_match = Template(
+        "MATCH (from:$source_node_label{$match_clause})"
+    ).safe_substitute(
+        source_node_label=rel_schema.source_node_label,
+        match_clause=_build_match_clause(rel_schema.source_node_matcher),
+    )
+
+    target_match = Template(
+        "MATCH (to:$target_node_label{$match_clause})"
+    ).safe_substitute(
+        target_node_label=rel_schema.target_node_label,
+        match_clause=_build_match_clause(rel_schema.target_node_matcher),
+    )
+
+    if rel_schema.direction == LinkDirection.INWARD:
+        rel = f"(from)<-[r:{rel_schema.rel_label}]-(to)"
+    else:
+        rel = f"(from)-[r:{rel_schema.rel_label}]->(to)"
+
+    return matchlink_query_template.safe_substitute(
+        source_match=source_match,
+        target_match=target_match,
+        rel=rel,
+        set_rel_properties_statement=_build_rel_properties_statement(
+            "r",
+            rel_props_as_dict,
+        ),
+    )

cartography/graph/statement.py

@@ -56,7 +56,7 @@ class GraphStatement:
 
         self.parent_job_name = parent_job_name if parent_job_name else None
         self.parent_job_sequence_num = (
-            parent_job_sequence_num if parent_job_sequence_num else None
+            parent_job_sequence_num if parent_job_sequence_num else 1
         )
 
     def merge_parameters(self, parameters: Dict) -> None:

cartography/intel/aws/efs.py

@@ -9,6 +9,7 @@ import neo4j
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
 from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.efs.access_point import EfsAccessPointSchema
 from cartography.models.aws.efs.file_system import EfsFileSystemSchema
 from cartography.models.aws.efs.mount_target import EfsMountTargetSchema
 from cartography.util import aws_handle_regions
@@ -44,6 +45,7 @@ def transform_efs_file_systems(
         transformed_file_system = {
             "FileSystemId": file_system["FileSystemId"],
             "FileSystemArn": file_system["FileSystemArn"],
+            "Region": region,
             "OwnerId": file_system.get("OwnerId"),
             "CreationToken": file_system.get("CreationToken"),
             "CreationTime": file_system.get("CreationTime"),
@@ -87,6 +89,49 @@ def get_efs_mount_targets(
     return mountTargets
 
 
+@timeit
+@aws_handle_regions
+def get_efs_access_points(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "efs", region_name=region, config=get_botocore_config()
+    )
+
+    paginator = client.get_paginator("describe_access_points")
+    accessPoints = []
+    for page in paginator.paginate():
+        accessPoints.extend(page.get("AccessPoints", []))
+
+    return accessPoints
+
+
+def transform_efs_access_points(
+    accessPoints: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    """
+    Transform Efs Access Points data for ingestion
+    """
+    transformed = []
+    for ap in accessPoints:
+        transformed.append(
+            {
+                "AccessPointArn": ap["AccessPointArn"],
+                "AccessPointId": ap["AccessPointId"],
+                "Region": region,
+                "FileSystemId": ap["FileSystemId"],
+                "Name": ap.get("Name"),
+                "LifeCycleState": ap.get("LifeCycleState"),
+                "OwnerId": ap.get("OwnerId"),
+                "Uid": ap.get("PosixUser", {}).get("Uid"),
+                "Gid": ap.get("PosixUser", {}).get("Gid"),
+                "RootDirectoryPath": ap.get("RootDirectory", {}).get("Path"),
+            }
+        )
+
+    return transformed
+
+
 @timeit
 def load_efs_mount_targets(
     neo4j_session: neo4j.Session,
@@ -129,6 +174,27 @@ def load_efs_file_systems(
     )
 
 
+@timeit
+def load_efs_access_points(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Efs {len(data)} access points for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        EfsAccessPointSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
 @timeit
 def cleanup(
     neo4j_session: neo4j.Session,
@@ -141,6 +207,9 @@ def cleanup(
     GraphJob.from_node_schema(EfsFileSystemSchema(), common_job_parameters).run(
         neo4j_session
     )
+    GraphJob.from_node_schema(EfsAccessPointSchema(), common_job_parameters).run(
+        neo4j_session
+    )
 
 
 @timeit
@@ -178,4 +247,15 @@ def sync(
             update_tag,
         )
 
+        accessPoints = get_efs_access_points(boto3_session, region)
+        accessPoints_transformed = transform_efs_access_points(accessPoints, region)
+
+        load_efs_access_points(
+            neo4j_session,
+            accessPoints_transformed,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
     cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/inspector.py

@@ -16,8 +16,6 @@ from cartography.util import aws_handle_regions
 from cartography.util import aws_paginate
 from cartography.util import batch
 from cartography.util import timeit
-from cartography.util import to_asynchronous
-from cartography.util import to_synchronous
 
 logger = logging.getLogger(__name__)
 
@@ -329,10 +327,9 @@ def sync(
         member_accounts = get_member_accounts(boto3_session, region)
         # the current host account may not be considered a "member", but we still fetch its findings
         member_accounts.append(current_aws_account_id)
-
-        async def async_ingest_findings_for_account(account_id: str) -> None:
-            await to_asynchronous(
-                _sync_findings_for_account,
+        logger.info(f"Member accounts to be synced: {member_accounts}")
+        for account_id in member_accounts:
+            _sync_findings_for_account(
                 neo4j_session,
                 boto3_session,
                 region,
@@ -341,11 +338,4 @@ def sync(
                 current_aws_account_id,
             )
 
-        to_synchronous(
-            *[
-                async_ingest_findings_for_account(account_id)
-                for account_id in member_accounts
-            ]
-        )
-
     cleanup(neo4j_session, common_job_parameters)

cartography/models/aws/efs/access_point.py

@@ -0,0 +1,77 @@
+from dataclasses import dataclass
+
+from cartography.models.core.common import PropertyRef
+from cartography.models.core.nodes import CartographyNodeProperties
+from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.relationships import CartographyRelProperties
+from cartography.models.core.relationships import CartographyRelSchema
+from cartography.models.core.relationships import LinkDirection
+from cartography.models.core.relationships import make_target_node_matcher
+from cartography.models.core.relationships import OtherRelationships
+from cartography.models.core.relationships import TargetNodeMatcher
+
+
+@dataclass(frozen=True)
+class EfsAccessPointNodeProperties(CartographyNodeProperties):
+    id: PropertyRef = PropertyRef("AccessPointArn")
+    arn: PropertyRef = PropertyRef("AccessPointArn", extra_index=True)
+    region: PropertyRef = PropertyRef("Region", set_in_kwargs=True)
+    access_point_id: PropertyRef = PropertyRef("AccessPointId")
+    file_system_id: PropertyRef = PropertyRef("FileSystemId")
+    lifecycle_state: PropertyRef = PropertyRef("LifeCycleState")
+    name: PropertyRef = PropertyRef("Name")
+    owner_id: PropertyRef = PropertyRef("OwnerId")
+    posix_gid: PropertyRef = PropertyRef("Gid")
+    posix_uid: PropertyRef = PropertyRef("Uid")
+    root_directory_path: PropertyRef = PropertyRef("RootDirectoryPath")
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class EfsAccessPointToAwsAccountRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class EfsAccessPointToAWSAccountRel(CartographyRelSchema):
+    target_node_label: str = "AWSAccount"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {"id": PropertyRef("AWS_ID", set_in_kwargs=True)},
+    )
+    direction: LinkDirection = LinkDirection.INWARD
+    rel_label: str = "RESOURCE"
+    properties: EfsAccessPointToAwsAccountRelProperties = (
+        EfsAccessPointToAwsAccountRelProperties()
+    )
+
+
+@dataclass(frozen=True)
+class EfsAccessPointToEfsFileSystemRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class EfsAccessPointToEfsFileSystemRel(CartographyRelSchema):
+    target_node_label: str = "EfsFileSystem"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {"id": PropertyRef("FileSystemId")},
+    )
+    direction: LinkDirection = LinkDirection.OUTWARD
+    rel_label: str = "ACCESS_POINT_OF"
+    properties: EfsAccessPointToEfsFileSystemRelProperties = (
+        EfsAccessPointToEfsFileSystemRelProperties()
+    )
+
+
+@dataclass(frozen=True)
+class EfsAccessPointSchema(CartographyNodeSchema):
+    label: str = "EfsAccessPoint"
+    properties: EfsAccessPointNodeProperties = EfsAccessPointNodeProperties()
+    sub_resource_relationship: EfsAccessPointToAWSAccountRel = (
+        EfsAccessPointToAWSAccountRel()
+    )
+    other_relationships: OtherRelationships = OtherRelationships(
+        [
+            EfsAccessPointToEfsFileSystemRel(),
+        ]
+    )

cartography/models/core/common.py

@@ -63,6 +63,7 @@ class PropertyRef:
             ```
             This means that as we create AWSInstanceProfile nodes, we will search for AWSRoles to attach to, and we do
             this by checking if each role's `arn` field is in the `Roles` list of the data dict.
+        Note that one_to_many has no effect on matchlinks.
         """
         self.name = name
         self.set_in_kwargs = set_in_kwargs

cartography/models/core/relationships.py

@@ -42,6 +42,11 @@ class CartographyRelProperties(abc.ABC):
     Abstract class that represents the properties on a CartographyRelSchema. This is intended to enforce that all
     subclasses will have a lastupdated field defined on their resulting relationships. These fields are assigned to the
     relationship in the `SET` clause.
+
+    If the CartographyRelSchema is used as a MatchLink, the following properties are required to be defined here:
+    - lastupdated: A PropertyRef to the update tag of the relationship.
+    - _sub_resource_label: A PropertyRef to the label of the sub-resource that the relationship is associated with.
+    - _sub_resource_id: A PropertyRef to the id of the sub-resource that the relationship is associated with.
     """
 
     lastupdated: PropertyRef = field(init=False)
@@ -90,6 +95,29 @@ def make_target_node_matcher(key_ref_dict: Dict[str, PropertyRef]) -> TargetNode
     return make_dataclass(TargetNodeMatcher.__name__, fields, frozen=True)()
 
 
+@dataclass(frozen=True)
+class SourceNodeMatcher:
+    """
+    Same as TargetNodeMatcher, but for the source node; see `make_source_node_matcher()`.
+    This object is used only for load_matchlinks() where we match on and connect existing nodes.
+    This has no effect on CartographyRelSchema objects that are included in CartographyNodeSchema.
+    """
+
+    pass
+
+
+def make_source_node_matcher(key_ref_dict: Dict[str, PropertyRef]) -> SourceNodeMatcher:
+    """
+    :param key_ref_dict: A Dict mapping keys present on the node to PropertyRef objects.
+    :return: A SourceNodeMatcher used for CartographyRelSchema to match with other nodes.
+    """
+    fields = [
+        (key, PropertyRef, field(default=prop_ref))
+        for key, prop_ref in key_ref_dict.items()
+    ]
+    return make_dataclass(SourceNodeMatcher.__name__, fields, frozen=True)()
+
+
 @dataclass(frozen=True)
 class CartographyRelSchema(abc.ABC):
     """
@@ -139,6 +167,22 @@ class CartographyRelSchema(abc.ABC):
         """
         pass
 
+    @property
+    def source_node_label(self) -> str | None:
+        """
+        :return: Optional. Only used for load_matchlinks(). The source node label to use for the relationship.
+        This does not affect CartographyRelSchema that are included in CartographyNodeSchema objects.
+        """
+        return None
+
+    @property
+    def source_node_matcher(self) -> SourceNodeMatcher | None:
+        """
+        :return: Optional. Only used for load_matchlinks(). A SourceNodeMatcher object used to find what node(s) to attach the relationship to.
+        This does not affect CartographyRelSchema that are included in CartographyNodeSchema objects.
+        """
+        return None
+
 
 @dataclass(frozen=True)
 class OtherRelationships:

{cartography-0.106.0rc1.dist-info → cartography-0.106.0rc2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cartography
-Version: 0.106.0rc1
+Version: 0.106.0rc2
 Summary: Explore assets and their relationships across your technical infrastructure.
 Maintainer: Cartography Contributors
 License: apache2

{cartography-0.106.0rc1.dist-info → cartography-0.106.0rc2.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cartography/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cartography/__main__.py,sha256=y5iqUrj4BmqZfjeDT-9balzpXeMARgHeIedRMRI1gr8,100
-cartography/_version.py,sha256=GMZa5N9JcxUAslbJ9f4PNbMORn40Ng2KlST6Ofx9W7I,525
+cartography/_version.py,sha256=axH1PjcS4osdPi0BaK-npCXNnQPC_1rYG4YGAn2aH60,525
 cartography/cli.py,sha256=LIH9uY6LyctL8VDsrllxFvU17f1vAAhXQVsQ83wGvRI,41072
 cartography/config.py,sha256=Magq4iNynKdIl3hYjrxlikXfVLxryf73t41L2bAyUPI,14298
 cartography/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -12,7 +12,7 @@ cartography/client/aws/__init__.py,sha256=Zj7nX21QQELwPLZlpldm_22IiXZ1VFsEFQbWX_
 cartography/client/aws/ecr.py,sha256=04IXnuEAauyO5Mx9Wmt79WdUnYDhYsk2QSOnwE5_BeM,1664
 cartography/client/aws/iam.py,sha256=dYsGikc36DEsSeR2XVOVFFUDwuU9yWj_EVkpgVYCFgM,1293
 cartography/client/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cartography/client/core/tx.py,sha256=1kvITtFseyG4C8h5xUxYEvhatejMs71oE0gCjjbDHkQ,10950
+cartography/client/core/tx.py,sha256=Xx6_a5u7PE5pyREuBL_J39ORcafqieFf4KdosaEv1c4,13530
 cartography/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cartography/data/indexes.cypher,sha256=vbKOyt87_MAxrDnKU8cbZBuiLnMyrbadSZq6aIaT6Zo,24112
 cartography/data/permission_relationships.yaml,sha256=RuKGGc_3ZUQ7ag0MssB8k_zaonCkVM5E8I_svBWTmGc,969
@@ -130,11 +130,11 @@ cartography/driftdetect/shortcut.py,sha256=0AvX9vZGNRWeLiRqxU7eOOo77gcfufHx4IHZv
 cartography/driftdetect/storage.py,sha256=xvDY4qn6jDhDpBvc8hFXXpbcYIbMNqkm3wX5LBf28u0,1549
 cartography/driftdetect/util.py,sha256=Lqxv8QoFn3_3Fz18qCOjkjJ6yBwgrHjrxXmArBAEdkc,929
 cartography/graph/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cartography/graph/cleanupbuilder.py,sha256=T71aR_IVp4oLu5SQb1miwn9Is8tqg25JJm4SkEdZ-80,14542
+cartography/graph/cleanupbuilder.py,sha256=uMSjMOnfkfTDrgMJwflNK8HAd4oioG8NIU0JeMN7ets,16552
 cartography/graph/context.py,sha256=RGxGb8EnxowcqjR0nFF86baNhgRHeUF9wjIoFUoG8LU,1230
-cartography/graph/job.py,sha256=5e1L6D3RCM2WgEgbm7CUeqEhMkgw-TEh2cl1-xmzlwQ,7739
-cartography/graph/querybuilder.py,sha256=Pd_WVJtZB_vh9VR7cm9PAS5Fl4qW783Hvq5njW5XMxA,21814
-cartography/graph/statement.py,sha256=bOCAKatvObyeEcqMiq0fUMnqUiZecLVaeWOIWZm7XWs,5386
+cartography/graph/job.py,sha256=5h4FsOV2tHeO5O2Gv-vglI5QUNE7V6p-RNmf9Fz5gvU,9396
+cartography/graph/querybuilder.py,sha256=b_dIScQ8MZfskM3CsY8vOgZjHMV1rzbJuywML1X72mw,27206
+cartography/graph/statement.py,sha256=VtvZFMqzzZk-gDeOnBx6oDj90XYOCM1EWw4d55bm7SU,5383
 cartography/intel/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cartography/intel/analysis.py,sha256=uWIpmrk2hezkqjfepqz1VV2BopT7VZ-alBj2YekfE_c,1546
 cartography/intel/create_indexes.py,sha256=6G43Rko8oXsAhHdSb2RbSVMvGMLi_YALlyUQE77jPVw,742
@@ -153,7 +153,7 @@ cartography/intel/aws/config.py,sha256=IIICicLQ0OTT3H3o8LDakIkA1BPUMwSyzpKonet-P
 cartography/intel/aws/dynamodb.py,sha256=VvvjeUgi1ZrqY9flXIQnfhhaSVSEqXXHW6T9917VLBk,5728
 cartography/intel/aws/ecr.py,sha256=7a9EHZru6AUIGE_6MJ4h4_ZmrvBxsXaerCHtGr59lJ0,8393
 cartography/intel/aws/ecs.py,sha256=-RPYdc6KC5fbaosaGsgWRkuLoHnE71euud4ZC1wGJI8,13464
-cartography/intel/aws/efs.py,sha256=gzxss1hUXs6C6omxCaFwI3QBu5JXX8G3CTduijrN7vk,5549
+cartography/intel/aws/efs.py,sha256=6ZvFmVHLfZlyo8xx2dlRsC1IoVOpBOtsij_AdFczkDo,7884
 cartography/intel/aws/eks.py,sha256=bPItyEj5q0nSDltJrr0S5MIrTPV0fK3xkqF6EV8fcqA,3759
 cartography/intel/aws/elasticache.py,sha256=dpRJCYxgUW2ImgGMt4L54xFil8apUoJxZq6hpewXxAE,4590
 cartography/intel/aws/elasticsearch.py,sha256=8X_Rp1zejkvXho0Zz_Cr4g-w9IpompdYRc-YS595Aso,8645
@@ -161,7 +161,7 @@ cartography/intel/aws/emr.py,sha256=EJoKjHQP7-F_A1trUNU05Sb42yNR1i0C9VIpGcCfAXw,
 cartography/intel/aws/iam.py,sha256=bkyIzWw2OC4MHxuoCvTiZ5eEGEQhz2asiUgY_tkX2GY,34322
 cartography/intel/aws/iam_instance_profiles.py,sha256=QUyu30xid60BFaemp-q0y9FgNsHaIQyQnQ8gHUzWC4k,2211
 cartography/intel/aws/identitycenter.py,sha256=C2EOfwQO1zBjePAXtreUcJIy7RU__ior3liRT4SpGO8,9544
-cartography/intel/aws/inspector.py,sha256=aqxNjnv4CJN-ZyQ16djpoQ6FmIK47vj6eAAQiwjQQv4,11436
+cartography/intel/aws/inspector.py,sha256=rD_O3DUGv5-GSMNqDzb11Ps5bX1sIo_JDq3UTAGvUpE,11168
 cartography/intel/aws/kms.py,sha256=RtCxNG-Den-f4Il-NO3YL_-BFUmg1qFt4lNucue9SQM,12130
 cartography/intel/aws/lambda_function.py,sha256=cutCsMnvyJB8vKUP0fHORrhijw944cXSoLKOHYdi6SM,10389
 cartography/intel/aws/organizations.py,sha256=m5qk60N6pe7iKLN-Rtfg9aYv7OozoloSkcsuePd-RGs,4680
@@ -373,6 +373,7 @@ cartography/models/aws/ecs/services.py,sha256=YPVTS0BB3X6tFCqIX3V1AZXcnHQEADusQk
 cartography/models/aws/ecs/task_definitions.py,sha256=zZh2SHXEAimp9YgYf1wRU-wNs_4LWNk7uNzDoms8Sy8,4182
 cartography/models/aws/ecs/tasks.py,sha256=6r2WZDc26LGR2j-U5s5v4d4PbMIvEDSkLAbTtpFdrUM,5013
 cartography/models/aws/efs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cartography/models/aws/efs/access_point.py,sha256=Auk8E0AoPjrCxch8bP8fhT3JZ2WxblTTfYSqamA_p8g,3185
 cartography/models/aws/efs/file_system.py,sha256=6BNzjQJKZ-rYlDLw1UvDBhVRKre07-9EkcW9xvR3Wh0,2842
 cartography/models/aws/efs/mount_target.py,sha256=Bdd2zgWp9HtsK9EmEAgoIYpT9AOTs5wzH3sgCq4HLMU,3347
 cartography/models/aws/eks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -409,9 +410,9 @@ cartography/models/cloudflare/member.py,sha256=s7S4OJjXAOq9vPc-TXFGbxKDjEHB5Y_Ek
 cartography/models/cloudflare/role.py,sha256=HRjDoLMSs0YJlJosZsXEZV58OE_3vGAWSDN_aGiK2Y8,1855
 cartography/models/cloudflare/zone.py,sha256=uMHTSHq32eWe47k3b08sbnQBLrbNWPKpL9Osc4jcFD0,2805
 cartography/models/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cartography/models/core/common.py,sha256=O8hF-kNdQ5-1NGKJuX5fcOcHf4kDYYc1b9tndPCcwUU,6202
+cartography/models/core/common.py,sha256=YPBpM0q2EBxTLQ-2OahGR1irMsCEFfu0eL-q5Uijxko,6261
 cartography/models/core/nodes.py,sha256=GFK-optqan5L57D36ZQ7AdylGRELUl9pMyBfZ7l-cz4,4367
-cartography/models/core/relationships.py,sha256=_ph52obhjeqc67IZ_rwuAtIbBqMHtKb04ceCP6g4Q04,5151
+cartography/models/core/relationships.py,sha256=TfcU01lIgBxoINjEwTWngIXkbFuHnE-UHGQ9h-V6ZDE,7100
 cartography/models/crowdstrike/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cartography/models/crowdstrike/hosts.py,sha256=J4UpdsVUNiQXqxnvmWdVkwsFueVagPV-7sTV3D7Lkts,2686
 cartography/models/cve/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -481,9 +482,9 @@ cartography/models/trivy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZ
 cartography/models/trivy/findings.py,sha256=SgI_h1aRyR20uAHvuXIZ1T6r4IZJt6SVhxRaF2bTsm0,3085
 cartography/models/trivy/fix.py,sha256=ho9ENVl9HSXqyggyCwR6ilkOBKDxpQ7rGibo_j21NA4,2587
 cartography/models/trivy/package.py,sha256=IwO1RZZ-MFRtNbt8Cq6YFl6fdNJMFmULnJkkK8Q4rL4,2809
-cartography-0.106.0rc1.dist-info/licenses/LICENSE,sha256=kvLEBRYaQ1RvUni6y7Ti9uHeooqnjPoo6n_-0JO1ETc,11351
-cartography-0.106.0rc1.dist-info/METADATA,sha256=9XcrmhSHjFUjYenD-w_9n7_q-vZTVf5XLlvVVfh6zYE,12409
-cartography-0.106.0rc1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-cartography-0.106.0rc1.dist-info/entry_points.txt,sha256=GVIAWD0o0_K077qMA_k1oZU4v-M0a8GLKGJR8tZ-qH8,112
-cartography-0.106.0rc1.dist-info/top_level.txt,sha256=BHqsNJQiI6Q72DeypC1IINQJE59SLhU4nllbQjgJi9g,12
-cartography-0.106.0rc1.dist-info/RECORD,,
+cartography-0.106.0rc2.dist-info/licenses/LICENSE,sha256=kvLEBRYaQ1RvUni6y7Ti9uHeooqnjPoo6n_-0JO1ETc,11351
+cartography-0.106.0rc2.dist-info/METADATA,sha256=Y1-Ir7t90DLnZrn2Tq-ee9j6akBX_Ezyc8YeZejQFWY,12409
+cartography-0.106.0rc2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+cartography-0.106.0rc2.dist-info/entry_points.txt,sha256=GVIAWD0o0_K077qMA_k1oZU4v-M0a8GLKGJR8tZ-qH8,112
+cartography-0.106.0rc2.dist-info/top_level.txt,sha256=BHqsNJQiI6Q72DeypC1IINQJE59SLhU4nllbQjgJi9g,12
+cartography-0.106.0rc2.dist-info/RECORD,,