cartography 0.110.0rc1__py3-none-any.whl → 0.111.0__py3-none-any.whl

cartography/_version.py

@@ -1,7 +1,14 @@
 # file generated by setuptools-scm
 # don't change, don't track in version control
 
-__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
 
 TYPE_CHECKING = False
 if TYPE_CHECKING:
@@ -9,13 +16,19 @@ if TYPE_CHECKING:
     from typing import Union
 
     VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
 else:
     VERSION_TUPLE = object
+    COMMIT_ID = object
 
 version: str
 __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
 
-__version__ = version = '0.110.0rc1'
-__version_tuple__ = version_tuple = (0, 110, 0, 'rc1')
+__version__ = version = '0.111.0'
+__version_tuple__ = version_tuple = (0, 111, 0)
+
+__commit_id__ = commit_id = None

cartography/cli.py

@@ -254,14 +254,6 @@ class CLI:
                 "The name of environment variable containing Entra Client Secret for Service Principal Authentication."
             ),
         )
-        parser.add_argument(
-            "--entra-best-effort-mode",
-            action="store_true",
-            help=(
-                "Enable Entra ID sync best effort mode. This will allow cartography to continue "
-                "syncing other Entra ID entities and delay raising an exception until the end of the sync."
-            ),
-        )
         parser.add_argument(
             "--aws-requested-syncs",
             type=str,
@@ -770,6 +762,42 @@ class CLI:
                 "Required if you are using the SentinelOne intel module. Ignored otherwise."
             ),
         )
+        parser.add_argument(
+            "--keycloak-client-id",
+            type=str,
+            default=None,
+            help=(
+                "The Keycloak client ID to sync. "
+                "Required if you are using the Keycloak intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--keycloak-client-secret-env-var",
+            type=str,
+            default="KEYCLOAK_CLIENT_SECRET",
+            help=(
+                "The name of an environment variable containing the Keycloak client secret. "
+                "Required if you are using the Keycloak intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--keycloak-url",
+            type=str,
+            help=(
+                "The base URL for the Keycloak instance. "
+                "Required if you are using the Keycloak intel module. Ignored otherwise. "
+            ),
+        )
+        parser.add_argument(
+            "--keycloak-realm",
+            type=str,
+            default="master",
+            help=(
+                "The Keycloak realm used for authentication (note: all available realms will be synced). "
+                "Should be `master` (default value) in most of the cases. "
+                "Required if you are using the Keycloak intel module. Ignored otherwise. "
+            ),
+        )
 
         return parser
 
@@ -1141,6 +1169,16 @@ class CLI:
         else:
             config.sentinelone_api_token = None
 
+        if config.keycloak_client_secret_env_var:
+            logger.debug(
+                f"Reading Client Secret for Keycloak from environment variable {config.keycloak_client_secret_env_var}",
+            )
+            config.keycloak_client_secret = os.environ.get(
+                config.keycloak_client_secret_env_var
+            )
+        else:
+            config.keycloak_client_secret = None
+
         # Run cartography
         try:
             return cartography.sync.run_with_config(self.sync, config)

cartography/config.py

@@ -51,9 +51,6 @@ class Config:
     :param entra_client_id: Client Id for connecting in a Service Principal Authentication approach. Optional.
     :type entra_client_secret: str
     :param entra_client_secret: Client Secret for connecting in a Service Principal Authentication approach. Optional.
-    :type entra_best_effort_mode: bool
-    :param entra_best_effort_mode: If True, Entra ID sync will continue on errors and raise an aggregated
-        exception at the end of the sync. If False (default), exceptions will be raised immediately.
     :type aws_requested_syncs: str
     :param aws_requested_syncs: Comma-separated list of AWS resources to sync. Optional.
     :type aws_guardduty_severity_threshold: str
@@ -167,10 +164,16 @@ class Config:
     :param sentinelone_api_url: SentinelOne API URL. Optional.
     :type sentinelone_api_token: string
     :param sentinelone_api_token: SentinelOne API token for authentication. Optional.
-    :type sentinelone_api_token_env_var: string
-    :param sentinelone_api_token_env_var: The name of an environment variable containing the SentinelOne API token. Optional.
     :type sentinelone_account_ids: list[str]
     :param sentinelone_account_ids: List of SentinelOne account IDs to sync. Optional.
+    :type keycloak_client_id: str
+    :param keycloak_client_id: Keycloak client ID for API authentication. Optional.
+    :type keycloak_client_secret: str
+    :param keycloak_client_secret: Keycloak client secret for API authentication. Optional.
+    :type keycloak_realm: str
+    :param keycloak_realm: Keycloak realm for authentication (all realms will be synced). Optional.
+    :type keycloak_url: str
+    :param keycloak_url: Keycloak base URL, e.g. https://keycloak.example.com. Optional.
     """
 
     def __init__(
@@ -194,7 +197,6 @@ class Config:
         entra_tenant_id=None,
         entra_client_id=None,
         entra_client_secret=None,
-        entra_best_effort_mode=False,
         aws_requested_syncs=None,
         aws_guardduty_severity_threshold=None,
         analysis_job_directory=None,
@@ -257,8 +259,11 @@ class Config:
         scaleway_org=None,
         sentinelone_api_url=None,
         sentinelone_api_token=None,
-        sentinelone_api_token_env_var=None,
         sentinelone_account_ids=None,
+        keycloak_client_id=None,
+        keycloak_client_secret=None,
+        keycloak_realm=None,
+        keycloak_url=None,
     ):
         self.neo4j_uri = neo4j_uri
         self.neo4j_user = neo4j_user
@@ -281,7 +286,6 @@ class Config:
         self.entra_tenant_id = entra_tenant_id
         self.entra_client_id = entra_client_id
         self.entra_client_secret = entra_client_secret
-        self.entra_best_effort_mode = entra_best_effort_mode
         self.aws_requested_syncs = aws_requested_syncs
         self.aws_guardduty_severity_threshold = aws_guardduty_severity_threshold
         self.analysis_job_directory = analysis_job_directory
@@ -344,5 +348,8 @@ class Config:
         self.scaleway_org = scaleway_org
         self.sentinelone_api_url = sentinelone_api_url
         self.sentinelone_api_token = sentinelone_api_token
-        self.sentinelone_api_token_env_var = sentinelone_api_token_env_var
         self.sentinelone_account_ids = sentinelone_account_ids
+        self.keycloak_client_id = keycloak_client_id
+        self.keycloak_client_secret = keycloak_client_secret
+        self.keycloak_realm = keycloak_realm
+        self.keycloak_url = keycloak_url

cartography/data/indexes.cypher

@@ -51,8 +51,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:AWSTransitGatewayAttachment) ON (n.lastupdated
 CREATE INDEX IF NOT EXISTS FOR (n:AWSUser) ON (n.arn);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSUser) ON (n.name);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSUser) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSVpc) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSVpc) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:AccountAccessKey) ON (n.accesskeyid);
 CREATE INDEX IF NOT EXISTS FOR (n:AccountAccessKey) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:AutoScalingGroup) ON (n.arn);

cartography/data/jobs/analysis/aws_ec2_keypair_analysis.json

@@ -22,8 +22,8 @@
             "iterative": false
         },
         {
-            "__comment__": "Attach EC2KeyPairs with matching fingerprints to eachother and set duplicate_keyfingerprint = True",
-            "query": "MATCH (k1:EC2KeyPair), (k2:EC2KeyPair) WHERE k1.id <> k2.id AND k1.keyfingerprint = k2.keyfingerprint SET k1.duplicate_keyfingerprint = True, k2.duplicate_keyfingerprint = True MERGE (k1)-[r:MATCHING_FINGERPRINT]-(k2) ON CREATE SET r.firstseen = $UPDATE_TAG SET r.lastupdated = $UPDATE_TAG return COUNT(*) as TotalCompleted",
+            "__comment__": "Attach EC2KeyPairs with matching fingerprints to each other and set duplicate_keyfingerprint = True. Use id(k1) < id(k2) to avoid Cartesian product warning and ensure O(1) comparison.",
+            "query": "MATCH (k1:EC2KeyPair) MATCH (k2:EC2KeyPair) WHERE id(k1) < id(k2) AND k1.keyfingerprint = k2.keyfingerprint SET k1.duplicate_keyfingerprint = True, k2.duplicate_keyfingerprint = True MERGE (k1)-[r:MATCHING_FINGERPRINT]-(k2) ON CREATE SET r.firstseen = $UPDATE_TAG SET r.lastupdated = $UPDATE_TAG RETURN COUNT(*) as TotalCompleted",
             "iterative": false
         }
     ]

cartography/data/jobs/analysis/keycloak_inheritance.json

@@ -0,0 +1,30 @@
+{
+  "statements": [
+    {
+        "__comment__": "Inherit group memberships from subgroups to parent groups",
+        "query": "MATCH (u:KeycloakUser)-[:MEMBER_OF]->(g:KeycloakGroup)-[:SUBGROUP_OF*1..5]->(pg:KeycloakGroup) MERGE (u)-[r:INHERITED_MEMBER_OF]->(pg) ON CREATE SET r.firstseen = $UPDATE_TAG SET r.lastupdated = $UPDATE_TAG",
+        "iterative": false
+    },
+    {
+        "__comment__": "Assign roles to users based on group memberships",
+        "query": "MATCH (u:KeycloakUser)-[:MEMBER_OF|INHERITED_MEMBER_OR]->(g:KeycloakGroup)-[:GRANTS]->(r:KeycloakRole) MERGE (u)-[r0:ASSUME_ROLE]-(r) ON CREATE SET r0.firstseen = $UPDATE_TAG SET r0.lastupdated = $UPDATE_TAG",
+        "iterative": false
+    },
+    {
+        "__comment__": "Propagate role grants to composite roles",
+        "query": "MATCH (r:KeycloakRole)-[:INCLUDES*1..5]->(c:KeycloakRole)-[:GRANTS]->(s:KeycloakScope) MERGE (r)-[r0:INDIRECT_GRANTS]-(s) ON CREATE SET r0.firstseen = $UPDATE_TAG SET r0.lastupdated = $UPDATE_TAG",
+        "iterative": false
+    },
+    {
+        "__comment__": "Identify legitimate scopes for users based on assumed roles",
+        "query": "MATCH (u:KeycloakUser)-[:ASSUME_ROLE]-(:KeycloakRole)-[:GRANTS|INDIRECT_GRANTS]->(s:KeycloakScope) MERGE (u)-[r:ASSUME_SCOPE]->(s) ON CREATE SET r.firstseen = $UPDATE_TAG SET r.lastupdated = $UPDATE_TAG",
+        "iterative": false
+    },
+    {
+        "__comment__": "Assign assumed scopes to users for orphan scopes (scopes not granted by any role)",
+        "query": "MATCH (s:KeycloakScope)<-[:RESOURCE]-(r:KeycloakRealm) MATCH (u:KeycloakUser)<-[:RESOURCE]-(r) WHERE NOT (s)<-[:GRANTS|INDIRECT_GRANTS]-(:KeycloakRole) MERGE (u)-[r0:ASSUME_SCOPE]->(s) SET r0.firstseen = $UPDATE_TAG SET r0.lastupdated = $UPDATE_TAG",
+        "iterative": false
+    }
+  ],
+  "name": "Keycloak inheritance analysis"
+}

cartography/graph/querybuilder.py

@@ -1,5 +1,7 @@
 import logging
 from dataclasses import asdict
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version
 from string import Template
 from typing import Dict
 from typing import List
@@ -223,6 +225,8 @@ def _build_attach_sub_resource_statement(
         $RelMergeClause
         ON CREATE SET r.firstseen = timestamp()
         SET
+            r._module_name = "$module_name",
+            r._module_version = "$module_version",
             $set_rel_properties_statement
         """,
     )
@@ -244,6 +248,8 @@ def _build_attach_sub_resource_statement(
         SubResourceLabel=sub_resource_link.target_node_label,
         MatchClause=_build_match_clause(sub_resource_link.target_node_matcher),
         RelMergeClause=rel_merge_clause,
+        module_name=_get_module_from_schema(sub_resource_link),
+        module_version=_get_cartography_version(),
         SubResourceRelLabel=sub_resource_link.rel_label,
         set_rel_properties_statement=_build_rel_properties_statement(
             "r",
@@ -278,6 +284,8 @@ def _build_attach_additional_links_statement(
         $RelMerge
         ON CREATE SET $rel_var.firstseen = timestamp()
         SET
+            $rel_var._module_name = "$module_name",
+            $rel_var._module_version = "$module_version",
             $set_rel_properties_statement
         """,
     )
@@ -312,6 +320,8 @@ def _build_attach_additional_links_statement(
             node_var=node_var,
             rel_var=rel_var,
             RelMerge=rel_merge,
+            module_name=_get_module_from_schema(link),
+            module_version=_get_cartography_version(),
             set_rel_properties_statement=_build_rel_properties_statement(
                 rel_var,
                 rel_props_as_dict,
@@ -453,6 +463,8 @@ def build_ingestion_query(
             MERGE (i:$node_label{id: $dict_id_field})
             ON CREATE SET i.firstseen = timestamp()
             SET
+                i._module_name = "$module_name",
+                i._module_version = "$module_version",
                 $set_node_properties_statement
             $attach_relationships_statement
         """,
@@ -475,6 +487,8 @@ def build_ingestion_query(
     ingest_query = query_template.safe_substitute(
         node_label=node_schema.label,
         dict_id_field=node_props.id,
+        module_name=_get_module_from_schema(node_schema),
+        module_version=_get_cartography_version(),
         set_node_properties_statement=_build_node_properties_statement(
             node_props_as_dict,
             node_schema.extra_node_labels,
@@ -650,6 +664,8 @@ def build_matchlink_query(rel_schema: CartographyRelSchema) -> str:
             MERGE $rel
             ON CREATE SET r.firstseen = timestamp()
             SET
+                r._module_name = "$module_name",
+                r._module_version = "$module_version",
                 $set_rel_properties_statement;
         """
     )
@@ -677,8 +693,62 @@ def build_matchlink_query(rel_schema: CartographyRelSchema) -> str:
         source_match=source_match,
         target_match=target_match,
         rel=rel,
+        module_name=_get_module_from_schema(rel_schema),
+        module_version=_get_cartography_version(),
         set_rel_properties_statement=_build_rel_properties_statement(
             "r",
             rel_props_as_dict,
         ),
     )
+
+
+def _get_cartography_version() -> str:
+    """
+    Get the current version of the cartography package.
+
+    This function attempts to retrieve the version of the installed cartography package
+    using importlib.metadata. If the package is not found (typically in development
+    or testing environments), it returns 'dev' as a fallback.
+
+    Returns:
+        The version string of the cartography package, or 'dev' if not found
+    """
+    try:
+        return version("cartography")
+    except PackageNotFoundError:
+        # This can occured if the cartography package is not installed in the environment, typically in development or testing environments.
+        logger.warning("cartography package not found. Returning 'dev' version.")
+        # Fallback to reading the VERSION file if the package is not found
+        return "dev"
+
+
+def _get_module_from_schema(
+    schema,  #: "CartographyNodeSchema" | "CartographyRelSchema",
+) -> str:
+    """
+    Extract the module name from a Cartography schema object.
+
+    This function extracts and formats the module name from a CartographyNodeSchema
+    or CartographyRelSchema object. It expects schemas to be part of the official
+    cartography.models package hierarchy and returns a formatted string indicating
+    the specific cartography module.
+
+    Args:
+        schema: A CartographyNodeSchema or CartographyRelSchema object
+
+    Returns:
+        A formatted module name string in the format 'cartography:<module_name>'
+        or 'unknown:<full_module_path>' if the schema is not from cartography.models
+    """
+    # If the entity schema does not belong to the cartography.models package,
+    # we log a warning and return the full module path.
+    if not schema.__module__.startswith("cartography.models."):
+        logger.warning(
+            "The schema %s does not start with 'cartography.models.'. "
+            "This may indicate that the schema is not part of the official cartography models.",
+            schema.__module__,
+        )
+        return f"unknown:{schema.__module__}"
+    # Otherwise, we return the module path as a string.
+    parts = schema.__module__.split(".")
+    return f"cartography:{parts[2]}"

cartography/intel/aws/apigateway.py

@@ -14,12 +14,18 @@ from policyuniverse.policy import Policy
 
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
-from cartography.models.aws.apigateway import APIGatewayRestAPISchema
-from cartography.models.aws.apigatewaycertificate import (
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.apigateway.apigateway import APIGatewayRestAPISchema
+from cartography.models.aws.apigateway.apigatewaycertificate import (
     APIGatewayClientCertificateSchema,
 )
-from cartography.models.aws.apigatewayresource import APIGatewayResourceSchema
-from cartography.models.aws.apigatewaystage import APIGatewayStageSchema
+from cartography.models.aws.apigateway.apigatewaydeployment import (
+    APIGatewayDeploymentSchema,
+)
+from cartography.models.aws.apigateway.apigatewayresource import (
+    APIGatewayResourceSchema,
+)
+from cartography.models.aws.apigateway.apigatewaystage import APIGatewayStageSchema
 from cartography.util import aws_handle_regions
 from cartography.util import timeit
 
@@ -40,6 +46,38 @@ def get_apigateway_rest_apis(
     return apis
 
 
+def get_rest_api_ids(
+    rest_apis: List[Dict],
+) -> List[str]:
+    """
+    Extracts the IDs of the REST APIs from the provided list.
+    """
+    return [api["id"] for api in rest_apis if "id" in api]
+
+
+@timeit
+@aws_handle_regions
+def get_rest_api_deployments(
+    boto3_session: boto3.session.Session,
+    rest_api_ids: List[str],
+    region: str,
+) -> List[Dict[str, Any]]:
+    """
+    Retrieves the deployments for each REST API in the provided list.
+    """
+    client = boto3_session.client(
+        "apigateway", region_name=region, config=get_botocore_config()
+    )
+    deployments: List[Dict[str, Any]] = []
+    for api_id in rest_api_ids:
+        paginator = client.get_paginator("get_deployments")
+        for page in paginator.paginate(restApiId=api_id):
+            for deployment in page.get("items", []):
+                deployment["api_id"] = api_id
+                deployments.append(deployment)
+    return deployments
+
+
 @timeit
 @aws_handle_regions
 def get_rest_api_details(
@@ -63,6 +101,7 @@ def get_rest_api_details(
 
 
 @timeit
+@aws_handle_regions
 def get_rest_api_stages(api: Dict, client: botocore.client.BaseClient) -> Any:
     """
     Gets the REST API Stage Resources.
@@ -104,6 +143,7 @@ def get_rest_api_client_certificate(
 
 
 @timeit
+@aws_handle_regions
 def get_rest_api_resources(api: Dict, client: botocore.client.BaseClient) -> List[Any]:
     """
     Gets the collection of Resource resources.
@@ -244,6 +284,25 @@ def transform_rest_api_details(
     return stages, certificates, resources
 
 
+def transform_apigateway_deployments(
+    deployments: List[Dict[str, Any]],
+    region: str,
+) -> List[Dict[str, Any]]:
+    """
+    Transform API Gateway Deployment data for ingestion
+    """
+    transformed_deployments = []
+    for deployment in deployments:
+        transformed_deployment = {
+            "id": f"{deployment['api_id']}/{deployment['id']}",
+            "api_id": deployment["api_id"],
+            "description": deployment.get("description"),
+            "region": region,
+        }
+        transformed_deployments.append(transformed_deployment)
+    return transformed_deployments
+
+
 @timeit
 def load_rest_api_details(
     neo4j_session: neo4j.Session,
@@ -283,6 +342,30 @@ def load_rest_api_details(
     )
 
 
+@timeit
+def load_apigateway_deployments(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    """
+    Load API Gateway Deployment data into neo4j.
+    """
+    logger.info(
+        f"Loading API Gateway {len(data)} deployments for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        APIGatewayDeploymentSchema(),
+        data,
+        region=region,
+        lastupdated=aws_update_tag,
+        AWS_ID=current_aws_account_id,
+    )
+
+
 @timeit
 def parse_policy(api_id: str, policy: Policy) -> Optional[Dict[Any, Any]]:
     """
@@ -345,6 +428,12 @@ def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
     )
     cleanup_job.run(neo4j_session)
 
+    cleanup_job = GraphJob.from_node_schema(
+        APIGatewayDeploymentSchema(),
+        common_job_parameters,
+    )
+    cleanup_job.run(neo4j_session)
+
 
 @timeit
 def sync_apigateway_rest_apis(
@@ -375,6 +464,19 @@ def sync_apigateway_rest_apis(
         current_aws_account_id,
         aws_update_tag,
     )
+
+    api_ids = get_rest_api_ids(rest_apis)
+    deployments = get_rest_api_deployments(
+        boto3_session,
+        api_ids,
+        region,
+    )
+
+    transformed_deployments = transform_apigateway_deployments(
+        deployments,
+        region,
+    )
+
     load_apigateway_rest_apis(
         neo4j_session,
         transformed_apis,
@@ -388,6 +490,13 @@ def sync_apigateway_rest_apis(
         current_aws_account_id,
         aws_update_tag,
     )
+    load_apigateway_deployments(
+        neo4j_session,
+        transformed_deployments,
+        region,
+        current_aws_account_id,
+        aws_update_tag,
+    )
 
 
 @timeit