cartography 0.110.0rc1__py3-none-any.whl → 0.111.0rc1__py3-none-any.whl

cartography/_version.py

@@ -1,7 +1,14 @@
 # file generated by setuptools-scm
 # don't change, don't track in version control
 
-__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
 
 TYPE_CHECKING = False
 if TYPE_CHECKING:
@@ -9,13 +16,19 @@ if TYPE_CHECKING:
     from typing import Union
 
     VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
 else:
     VERSION_TUPLE = object
+    COMMIT_ID = object
 
 version: str
 __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
 
-__version__ = version = '0.110.0rc1'
-__version_tuple__ = version_tuple = (0, 110, 0, 'rc1')
+__version__ = version = '0.111.0rc1'
+__version_tuple__ = version_tuple = (0, 111, 0, 'rc1')
+
+__commit_id__ = commit_id = None

cartography/cli.py

@@ -254,14 +254,6 @@ class CLI:
                 "The name of environment variable containing Entra Client Secret for Service Principal Authentication."
             ),
         )
-        parser.add_argument(
-            "--entra-best-effort-mode",
-            action="store_true",
-            help=(
-                "Enable Entra ID sync best effort mode. This will allow cartography to continue "
-                "syncing other Entra ID entities and delay raising an exception until the end of the sync."
-            ),
-        )
         parser.add_argument(
             "--aws-requested-syncs",
             type=str,

cartography/config.py

@@ -51,9 +51,6 @@ class Config:
     :param entra_client_id: Client Id for connecting in a Service Principal Authentication approach. Optional.
     :type entra_client_secret: str
     :param entra_client_secret: Client Secret for connecting in a Service Principal Authentication approach. Optional.
-    :type entra_best_effort_mode: bool
-    :param entra_best_effort_mode: If True, Entra ID sync will continue on errors and raise an aggregated
-        exception at the end of the sync. If False (default), exceptions will be raised immediately.
     :type aws_requested_syncs: str
     :param aws_requested_syncs: Comma-separated list of AWS resources to sync. Optional.
     :type aws_guardduty_severity_threshold: str
@@ -167,8 +164,6 @@ class Config:
     :param sentinelone_api_url: SentinelOne API URL. Optional.
     :type sentinelone_api_token: string
     :param sentinelone_api_token: SentinelOne API token for authentication. Optional.
-    :type sentinelone_api_token_env_var: string
-    :param sentinelone_api_token_env_var: The name of an environment variable containing the SentinelOne API token. Optional.
     :type sentinelone_account_ids: list[str]
     :param sentinelone_account_ids: List of SentinelOne account IDs to sync. Optional.
     """
@@ -194,7 +189,6 @@ class Config:
         entra_tenant_id=None,
         entra_client_id=None,
         entra_client_secret=None,
-        entra_best_effort_mode=False,
         aws_requested_syncs=None,
         aws_guardduty_severity_threshold=None,
         analysis_job_directory=None,
@@ -257,7 +251,6 @@ class Config:
         scaleway_org=None,
         sentinelone_api_url=None,
         sentinelone_api_token=None,
-        sentinelone_api_token_env_var=None,
         sentinelone_account_ids=None,
     ):
         self.neo4j_uri = neo4j_uri
@@ -281,7 +274,6 @@ class Config:
         self.entra_tenant_id = entra_tenant_id
         self.entra_client_id = entra_client_id
         self.entra_client_secret = entra_client_secret
-        self.entra_best_effort_mode = entra_best_effort_mode
         self.aws_requested_syncs = aws_requested_syncs
         self.aws_guardduty_severity_threshold = aws_guardduty_severity_threshold
         self.analysis_job_directory = analysis_job_directory
@@ -344,5 +336,4 @@ class Config:
         self.scaleway_org = scaleway_org
         self.sentinelone_api_url = sentinelone_api_url
         self.sentinelone_api_token = sentinelone_api_token
-        self.sentinelone_api_token_env_var = sentinelone_api_token_env_var
         self.sentinelone_account_ids = sentinelone_account_ids

cartography/data/indexes.cypher

@@ -51,8 +51,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:AWSTransitGatewayAttachment) ON (n.lastupdated
 CREATE INDEX IF NOT EXISTS FOR (n:AWSUser) ON (n.arn);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSUser) ON (n.name);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSUser) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSVpc) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSVpc) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:AccountAccessKey) ON (n.accesskeyid);
 CREATE INDEX IF NOT EXISTS FOR (n:AccountAccessKey) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:AutoScalingGroup) ON (n.arn);

cartography/data/jobs/analysis/aws_ec2_keypair_analysis.json

@@ -22,8 +22,8 @@
             "iterative": false
         },
         {
-            "__comment__": "Attach EC2KeyPairs with matching fingerprints to eachother and set duplicate_keyfingerprint = True",
-            "query": "MATCH (k1:EC2KeyPair), (k2:EC2KeyPair) WHERE k1.id <> k2.id AND k1.keyfingerprint = k2.keyfingerprint SET k1.duplicate_keyfingerprint = True, k2.duplicate_keyfingerprint = True MERGE (k1)-[r:MATCHING_FINGERPRINT]-(k2) ON CREATE SET r.firstseen = $UPDATE_TAG SET r.lastupdated = $UPDATE_TAG return COUNT(*) as TotalCompleted",
+            "__comment__": "Attach EC2KeyPairs with matching fingerprints to each other and set duplicate_keyfingerprint = True. Use id(k1) < id(k2) to avoid Cartesian product warning and ensure O(1) comparison.",
+            "query": "MATCH (k1:EC2KeyPair) MATCH (k2:EC2KeyPair) WHERE id(k1) < id(k2) AND k1.keyfingerprint = k2.keyfingerprint SET k1.duplicate_keyfingerprint = True, k2.duplicate_keyfingerprint = True MERGE (k1)-[r:MATCHING_FINGERPRINT]-(k2) ON CREATE SET r.firstseen = $UPDATE_TAG SET r.lastupdated = $UPDATE_TAG RETURN COUNT(*) as TotalCompleted",
             "iterative": false
         }
     ]

cartography/graph/querybuilder.py

@@ -1,5 +1,7 @@
 import logging
 from dataclasses import asdict
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version
 from string import Template
 from typing import Dict
 from typing import List
@@ -223,6 +225,8 @@ def _build_attach_sub_resource_statement(
         $RelMergeClause
         ON CREATE SET r.firstseen = timestamp()
         SET
+            r._module_name = "$module_name",
+            r._module_version = "$module_version",
             $set_rel_properties_statement
         """,
     )
@@ -244,6 +248,8 @@ def _build_attach_sub_resource_statement(
         SubResourceLabel=sub_resource_link.target_node_label,
         MatchClause=_build_match_clause(sub_resource_link.target_node_matcher),
         RelMergeClause=rel_merge_clause,
+        module_name=_get_module_from_schema(sub_resource_link),
+        module_version=_get_cartography_version(),
         SubResourceRelLabel=sub_resource_link.rel_label,
         set_rel_properties_statement=_build_rel_properties_statement(
             "r",
@@ -278,6 +284,8 @@ def _build_attach_additional_links_statement(
         $RelMerge
         ON CREATE SET $rel_var.firstseen = timestamp()
         SET
+            $rel_var._module_name = "$module_name",
+            $rel_var._module_version = "$module_version",
             $set_rel_properties_statement
         """,
     )
@@ -312,6 +320,8 @@ def _build_attach_additional_links_statement(
             node_var=node_var,
             rel_var=rel_var,
             RelMerge=rel_merge,
+            module_name=_get_module_from_schema(link),
+            module_version=_get_cartography_version(),
             set_rel_properties_statement=_build_rel_properties_statement(
                 rel_var,
                 rel_props_as_dict,
@@ -453,6 +463,8 @@ def build_ingestion_query(
             MERGE (i:$node_label{id: $dict_id_field})
             ON CREATE SET i.firstseen = timestamp()
             SET
+                i._module_name = "$module_name",
+                i._module_version = "$module_version",
                 $set_node_properties_statement
             $attach_relationships_statement
         """,
@@ -475,6 +487,8 @@ def build_ingestion_query(
     ingest_query = query_template.safe_substitute(
         node_label=node_schema.label,
         dict_id_field=node_props.id,
+        module_name=_get_module_from_schema(node_schema),
+        module_version=_get_cartography_version(),
         set_node_properties_statement=_build_node_properties_statement(
             node_props_as_dict,
             node_schema.extra_node_labels,
@@ -650,6 +664,8 @@ def build_matchlink_query(rel_schema: CartographyRelSchema) -> str:
             MERGE $rel
             ON CREATE SET r.firstseen = timestamp()
             SET
+                r._module_name = "$module_name",
+                r._module_version = "$module_version",
                 $set_rel_properties_statement;
         """
     )
@@ -677,8 +693,62 @@ def build_matchlink_query(rel_schema: CartographyRelSchema) -> str:
         source_match=source_match,
         target_match=target_match,
         rel=rel,
+        module_name=_get_module_from_schema(rel_schema),
+        module_version=_get_cartography_version(),
         set_rel_properties_statement=_build_rel_properties_statement(
             "r",
             rel_props_as_dict,
         ),
     )
+
+
+def _get_cartography_version() -> str:
+    """
+    Get the current version of the cartography package.
+
+    This function attempts to retrieve the version of the installed cartography package
+    using importlib.metadata. If the package is not found (typically in development
+    or testing environments), it returns 'dev' as a fallback.
+
+    Returns:
+        The version string of the cartography package, or 'dev' if not found
+    """
+    try:
+        return version("cartography")
+    except PackageNotFoundError:
+        # This can occured if the cartography package is not installed in the environment, typically in development or testing environments.
+        logger.warning("cartography package not found. Returning 'dev' version.")
+        # Fallback to reading the VERSION file if the package is not found
+        return "dev"
+
+
+def _get_module_from_schema(
+    schema,  #: "CartographyNodeSchema" | "CartographyRelSchema",
+) -> str:
+    """
+    Extract the module name from a Cartography schema object.
+
+    This function extracts and formats the module name from a CartographyNodeSchema
+    or CartographyRelSchema object. It expects schemas to be part of the official
+    cartography.models package hierarchy and returns a formatted string indicating
+    the specific cartography module.
+
+    Args:
+        schema: A CartographyNodeSchema or CartographyRelSchema object
+
+    Returns:
+        A formatted module name string in the format 'cartography:<module_name>'
+        or 'unknown:<full_module_path>' if the schema is not from cartography.models
+    """
+    # If the entity schema does not belong to the cartography.models package,
+    # we log a warning and return the full module path.
+    if not schema.__module__.startswith("cartography.models."):
+        logger.warning(
+            "The schema %s does not start with 'cartography.models.'. "
+            "This may indicate that the schema is not part of the official cartography models.",
+            schema.__module__,
+        )
+        return f"unknown:{schema.__module__}"
+    # Otherwise, we return the module path as a string.
+    parts = schema.__module__.split(".")
+    return f"cartography:{parts[2]}"

cartography/intel/aws/apigateway.py

@@ -14,12 +14,18 @@ from policyuniverse.policy import Policy
 
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
-from cartography.models.aws.apigateway import APIGatewayRestAPISchema
-from cartography.models.aws.apigatewaycertificate import (
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.apigateway.apigateway import APIGatewayRestAPISchema
+from cartography.models.aws.apigateway.apigatewaycertificate import (
     APIGatewayClientCertificateSchema,
 )
-from cartography.models.aws.apigatewayresource import APIGatewayResourceSchema
-from cartography.models.aws.apigatewaystage import APIGatewayStageSchema
+from cartography.models.aws.apigateway.apigatewaydeployment import (
+    APIGatewayDeploymentSchema,
+)
+from cartography.models.aws.apigateway.apigatewayresource import (
+    APIGatewayResourceSchema,
+)
+from cartography.models.aws.apigateway.apigatewaystage import APIGatewayStageSchema
 from cartography.util import aws_handle_regions
 from cartography.util import timeit
 
@@ -40,6 +46,38 @@ def get_apigateway_rest_apis(
     return apis
 
 
+def get_rest_api_ids(
+    rest_apis: List[Dict],
+) -> List[str]:
+    """
+    Extracts the IDs of the REST APIs from the provided list.
+    """
+    return [api["id"] for api in rest_apis if "id" in api]
+
+
+@timeit
+@aws_handle_regions
+def get_rest_api_deployments(
+    boto3_session: boto3.session.Session,
+    rest_api_ids: List[str],
+    region: str,
+) -> List[Dict[str, Any]]:
+    """
+    Retrieves the deployments for each REST API in the provided list.
+    """
+    client = boto3_session.client(
+        "apigateway", region_name=region, config=get_botocore_config()
+    )
+    deployments: List[Dict[str, Any]] = []
+    for api_id in rest_api_ids:
+        paginator = client.get_paginator("get_deployments")
+        for page in paginator.paginate(restApiId=api_id):
+            for deployment in page.get("items", []):
+                deployment["api_id"] = api_id
+                deployments.append(deployment)
+    return deployments
+
+
 @timeit
 @aws_handle_regions
 def get_rest_api_details(
@@ -244,6 +282,25 @@ def transform_rest_api_details(
     return stages, certificates, resources
 
 
+def transform_apigateway_deployments(
+    deployments: List[Dict[str, Any]],
+    region: str,
+) -> List[Dict[str, Any]]:
+    """
+    Transform API Gateway Deployment data for ingestion
+    """
+    transformed_deployments = []
+    for deployment in deployments:
+        transformed_deployment = {
+            "id": f"{deployment['api_id']}/{deployment['id']}",
+            "api_id": deployment["api_id"],
+            "description": deployment.get("description"),
+            "region": region,
+        }
+        transformed_deployments.append(transformed_deployment)
+    return transformed_deployments
+
+
 @timeit
 def load_rest_api_details(
     neo4j_session: neo4j.Session,
@@ -283,6 +340,30 @@ def load_rest_api_details(
     )
 
 
+@timeit
+def load_apigateway_deployments(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    """
+    Load API Gateway Deployment data into neo4j.
+    """
+    logger.info(
+        f"Loading API Gateway {len(data)} deployments for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        APIGatewayDeploymentSchema(),
+        data,
+        region=region,
+        lastupdated=aws_update_tag,
+        AWS_ID=current_aws_account_id,
+    )
+
+
 @timeit
 def parse_policy(api_id: str, policy: Policy) -> Optional[Dict[Any, Any]]:
     """
@@ -345,6 +426,12 @@ def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
     )
     cleanup_job.run(neo4j_session)
 
+    cleanup_job = GraphJob.from_node_schema(
+        APIGatewayDeploymentSchema(),
+        common_job_parameters,
+    )
+    cleanup_job.run(neo4j_session)
+
 
 @timeit
 def sync_apigateway_rest_apis(
@@ -375,6 +462,19 @@ def sync_apigateway_rest_apis(
         current_aws_account_id,
         aws_update_tag,
     )
+
+    api_ids = get_rest_api_ids(rest_apis)
+    deployments = get_rest_api_deployments(
+        boto3_session,
+        api_ids,
+        region,
+    )
+
+    transformed_deployments = transform_apigateway_deployments(
+        deployments,
+        region,
+    )
+
     load_apigateway_rest_apis(
         neo4j_session,
         transformed_apis,
@@ -388,6 +488,13 @@ def sync_apigateway_rest_apis(
         current_aws_account_id,
         aws_update_tag,
     )
+    load_apigateway_deployments(
+        neo4j_session,
+        transformed_deployments,
+        region,
+        current_aws_account_id,
+        aws_update_tag,
+    )
 
 
 @timeit

cartography/intel/aws/cognito.py

@@ -0,0 +1,201 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.cognito.identity_pool import CognitoIdentityPoolSchema
+from cartography.models.aws.cognito.user_pool import CognitoUserPoolSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_identity_pools(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "cognito-identity", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("list_identity_pools")
+
+    all_identity_pools = []
+
+    for page in paginator.paginate(MaxResults=50):
+        identity_pools = page.get("IdentityPools", [])
+        all_identity_pools.extend(identity_pools)
+    return all_identity_pools
+
+
+@timeit
+@aws_handle_regions
+def get_identity_pool_roles(
+    boto3_session: boto3.Session, identity_pools: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "cognito-identity", region_name=region, config=get_botocore_config()
+    )
+    all_identity_pool_details = []
+    for identity_pool in identity_pools:
+        response = client.get_identity_pool_roles(
+            IdentityPoolId=identity_pool["IdentityPoolId"]
+        )
+        all_identity_pool_details.append(response)
+    return all_identity_pool_details
+
+
+@timeit
+@aws_handle_regions
+def get_user_pools(boto3_session: boto3.Session, region: str) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "cognito-idp", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("list_user_pools")
+    all_user_pools = []
+
+    for page in paginator.paginate(MaxResults=50):
+        user_pools = page.get("UserPools", [])
+        all_user_pools.extend(user_pools)
+    return all_user_pools
+
+
+def transform_identity_pools(
+    identity_pools: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    transformed_identity_pools = []
+    for pool in identity_pools:
+        transformed_pool = {
+            "IdentityPoolId": pool["IdentityPoolId"],
+            "Region": region,
+            "Roles": list(pool.get("Roles", {}).values()),
+        }
+        transformed_identity_pools.append(transformed_pool)
+    return transformed_identity_pools
+
+
+def transform_user_pools(
+    user_pools: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    transformed_user_pools = []
+    for pool in user_pools:
+        transformed_pool = {
+            "Id": pool["Id"],
+            "Region": region,
+            "Name": pool["Name"],
+            "Status": pool.get("Status"),
+        }
+        transformed_user_pools.append(transformed_pool)
+    return transformed_user_pools
+
+
+@timeit
+def load_identity_pools(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Cognito Identity Pools {len(data)} for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        CognitoIdentityPoolSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def load_user_pools(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Cognito User Pools {len(data)} for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        CognitoUserPoolSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.debug("Running Efs cleanup job.")
+    GraphJob.from_node_schema(CognitoIdentityPoolSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(CognitoUserPoolSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing Cognito Identity Pools for region '{region}' in account '{current_aws_account_id}'.",
+        )
+
+        identity_pools = get_identity_pools(boto3_session, region)
+        if not identity_pools:
+            logger.info(
+                f"No Cognito Identity Pools found in region '{region}'. Skipping sync."
+            )
+        else:
+            identity_pool_roles = get_identity_pool_roles(
+                boto3_session, identity_pools, region
+            )
+            transformed_identity_pools = transform_identity_pools(
+                identity_pool_roles, region
+            )
+
+            load_identity_pools(
+                neo4j_session,
+                transformed_identity_pools,
+                region,
+                current_aws_account_id,
+                update_tag,
+            )
+
+            user_pools = get_user_pools(boto3_session, region)
+            transformed_user_pools = transform_user_pools(user_pools, region)
+
+            load_user_pools(
+                neo4j_session,
+                transformed_user_pools,
+                region,
+                current_aws_account_id,
+                update_tag,
+            )
+
+    cleanup(neo4j_session, common_job_parameters)