cartography 0.93.0rc1__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/aws/organizations.py

@@ -5,6 +5,8 @@ import boto3
 import botocore.exceptions
 import neo4j
 
+from cartography.client.core.tx import run_write_query
+from cartography.intel.aws.iam import sync_root_principal
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -16,19 +18,22 @@ def get_account_from_arn(arn: str) -> str:
 
 
 def get_caller_identity(boto3_session: boto3.session.Session) -> Dict:
-    client = boto3_session.client('sts')
+    client = boto3_session.client("sts")
     return client.get_caller_identity()
 
 
 def get_current_aws_account_id(boto3_session: boto3.session.Session) -> Dict:
-    return get_caller_identity(boto3_session)['Account']
+    return get_caller_identity(boto3_session)["Account"]
 
 
 def get_aws_account_default(boto3_session: boto3.session.Session) -> Dict:
     try:
         return {boto3_session.profile_name: get_current_aws_account_id(boto3_session)}
     except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
-        logger.debug("Error occurred getting default AWS account number.", exc_info=True)
+        logger.debug(
+            "Error occurred getting default AWS account number.",
+            exc_info=True,
+        )
         logger.error(
             (
                 "Unable to get AWS account number, an error occurred: '%s'. Make sure your AWS credentials are "
@@ -43,13 +48,20 @@ def get_aws_account_default(boto3_session: boto3.session.Session) -> Dict:
 def get_aws_accounts_from_botocore_config(boto3_session: boto3.session.Session) -> Dict:
     d = {}
     for profile_name in boto3_session.available_profiles:
-        if profile_name == 'default':
+        if profile_name == "default":
             logger.debug("Skipping AWS profile 'default'.")
             continue
         try:
             profile_boto3_session = boto3.Session(profile_name=profile_name)
-        except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
-            logger.debug("Error occurred calling boto3.Session() with profile_name '%s'.", profile_name, exc_info=True)
+        except (
+            botocore.exceptions.BotoCoreError,
+            botocore.exceptions.ClientError,
+        ) as e:
+            logger.debug(
+                "Error occurred calling boto3.Session() with profile_name '%s'.",
+                profile_name,
+                exc_info=True,
+            )
             logger.error(
                 (
                     "Unable to initialize an AWS session using profile '%s', an error occurred: '%s'. Make sure your "
@@ -62,7 +74,10 @@ def get_aws_accounts_from_botocore_config(boto3_session: boto3.session.Session)
             continue
         try:
             d[profile_name] = get_current_aws_account_id(profile_boto3_session)
-        except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
+        except (
+            botocore.exceptions.BotoCoreError,
+            botocore.exceptions.ClientError,
+        ) as e:
             logger.debug(
                 "Error occurred getting AWS account number with profile_name '%s'.",
                 profile_name,
@@ -87,7 +102,9 @@ def get_aws_accounts_from_botocore_config(boto3_session: boto3.session.Session)
 
 
 def load_aws_accounts(
-    neo4j_session: neo4j.Session, aws_accounts: Dict, aws_update_tag: int,
+    neo4j_session: neo4j.Session,
+    aws_accounts: Dict,
+    aws_update_tag: int,
     common_job_parameters: Dict,
 ) -> None:
     query = """
@@ -95,26 +112,30 @@ def load_aws_accounts(
     ON CREATE SET aa.firstseen = timestamp()
     SET aa.lastupdated = $aws_update_tag, aa.name = $ACCOUNT_NAME, aa.inscope=true
     REMOVE aa.foreign
-    WITH aa
-    MERGE (root:AWSPrincipal{arn: $RootArn})
-    ON CREATE SET root.firstseen = timestamp(), root.type = 'AWS'
-    SET root.lastupdated = $aws_update_tag
-    WITH aa, root
-    MERGE (aa)-[r:RESOURCE]->(root)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $aws_update_tag;
     """
     for account_name, account_id in aws_accounts.items():
-        root_arn = f'arn:aws:iam::{account_id}:root'
-        neo4j_session.run(
+        root_arn = f"arn:aws:iam::{account_id}:root"
+        run_write_query(
+            neo4j_session,
             query,
             ACCOUNT_ID=account_id,
             ACCOUNT_NAME=account_name,
             RootArn=root_arn,
             aws_update_tag=aws_update_tag,
         )
+        # Every AWS account has a root principal
+        sync_root_principal(
+            neo4j_session,
+            account_id,
+            aws_update_tag,
+        )
 
 
 @timeit
-def sync(neo4j_session: neo4j.Session, accounts: Dict, update_tag: int, common_job_parameters: Dict) -> None:
+def sync(
+    neo4j_session: neo4j.Session,
+    accounts: Dict,
+    update_tag: int,
+    common_job_parameters: Dict,
+) -> None:
     load_aws_accounts(neo4j_session, accounts, update_tag, common_job_parameters)

cartography/intel/aws/permission_relationships.py

@@ -12,6 +12,9 @@ import boto3
 import neo4j
 import yaml
 
+from cartography.client.core.tx import read_list_of_dicts_tx
+from cartography.client.core.tx import read_list_of_values_tx
+from cartography.client.core.tx import run_write_query
 from cartography.graph.statement import GraphStatement
 from cartography.util import timeit
 
@@ -19,7 +22,7 @@ logger = logging.getLogger(__name__)
 
 
 def evaluate_clause(clause: str, match: str) -> bool:
-    """ Evaluates the a clause in IAM. Clauses can be AWS [not]actions and [not]resources
+    """Evaluates the a clause in IAM. Clauses can be AWS [not]actions and [not]resources
 
     Arguments:
         clause {str, re.Pattern} -- The clause you are evaluating against. Clauses can use
@@ -36,9 +39,9 @@ def evaluate_clause(clause: str, match: str) -> bool:
 
 def evaluate_notaction_for_permission(statement: Dict, permission: str) -> bool:
     """Return whether an IAM 'notaction' clause in the given statement applies to the item"""
-    if 'notaction' not in statement:
+    if "notaction" not in statement:
         return False
-    for clause in statement['notaction']:
+    for clause in statement["notaction"]:
         if evaluate_clause(clause, permission):
             return True
     return False
@@ -46,9 +49,9 @@ def evaluate_notaction_for_permission(statement: Dict, permission: str) -> bool:
 
 def evaluate_action_for_permission(statement: Dict, permission: str) -> bool:
     """Return whether an IAM 'action' clause in the given statement applies to the permission"""
-    if 'action' not in statement:
+    if "action" not in statement:
         return True
-    for clause in statement['action']:
+    for clause in statement["action"]:
         if evaluate_clause(clause, permission):
             return True
     return False
@@ -56,9 +59,9 @@ def evaluate_action_for_permission(statement: Dict, permission: str) -> bool:
 
 def evaluate_resource_for_permission(statement: Dict, resource_arn: str) -> bool:
     """Return whether the given IAM 'resource' statement applies to the resource_arn"""
-    if 'resource' not in statement:
+    if "resource" not in statement:
         return False
-    for clause in statement['resource']:
+    for clause in statement["resource"]:
         if evaluate_clause(clause, resource_arn):
             return True
     return False
@@ -66,16 +69,20 @@ def evaluate_resource_for_permission(statement: Dict, resource_arn: str) -> bool
 
 def evaluate_notresource_for_permission(statement: Dict, resource_arn: str) -> bool:
     """Return whether an IAM 'notresource' clause in the given statement applies to the resource_arn"""
-    if 'notresource' not in statement:
+    if "notresource" not in statement:
         return False
-    for clause in statement['notresource']:
+    for clause in statement["notresource"]:
         if evaluate_clause(clause, resource_arn):
             return True
     return False
 
 
-def evaluate_statements_for_permission(statements: List[Dict], permission: str, resource_arn: str) -> bool:
-    """ Evaluate an entire statement for a specific permission against a resource
+def evaluate_statements_for_permission(
+    statements: List[Dict],
+    permission: str,
+    resource_arn: str,
+) -> bool:
+    """Evaluate an entire statement for a specific permission against a resource
 
     Arguments:
         statements {[dict]} -- The list of statements to be evaluated
@@ -97,9 +104,11 @@ def evaluate_statements_for_permission(statements: List[Dict], permission: str,
 
 
 def evaluate_policy_for_permissions(
-    statements: List[Dict], permissions: List[str], resource_arn: str,
+    statements: List[Dict],
+    permissions: List[str],
+    resource_arn: str,
 ) -> Tuple[bool, bool]:
-    """ Evaluates an entire policy for specific permissions to a resource.
+    """Evaluates an entire policy for specific permissions to a resource.
     AWS Policy evaluation reference
     https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_evaluation-logic.html
 
@@ -118,19 +127,31 @@ def evaluate_policy_for_permissions(
     allow_statements = [s for s in statements if s["effect"] == "Allow"]
     deny_statements = [s for s in statements if s["effect"] == "Deny"]
     for permission in permissions:
-        if evaluate_statements_for_permission(deny_statements, permission, resource_arn):
+        if evaluate_statements_for_permission(
+            deny_statements,
+            permission,
+            resource_arn,
+        ):
             # The action explicitly denied then no other policy can override it
             return False, True
         else:
-            if evaluate_statements_for_permission(allow_statements, permission, resource_arn):
+            if evaluate_statements_for_permission(
+                allow_statements,
+                permission,
+                resource_arn,
+            ):
                 # The action is allowed by this policy
                 return True, False
     # The action is not allowed by this policy, but not specifically denied either
     return False, False
 
 
-def principal_allowed_on_resource(policies: Dict, resource_arn: str, permissions: List[str]) -> bool:
-    """ Evaluates an enture set of policies for a specific resource for a specific permission.
+def principal_allowed_on_resource(
+    policies: Dict,
+    resource_arn: str,
+    permissions: List[str],
+) -> bool:
+    """Evaluates an enture set of policies for a specific resource for a specific permission.
 
 
     Arguments:
@@ -145,7 +166,11 @@ def principal_allowed_on_resource(policies: Dict, resource_arn: str, permissions
         raise ValueError("permissions is not a list")
     granted = False
     for _, statements in policies.items():
-        allowed, explicit_deny = evaluate_policy_for_permissions(statements, permissions, resource_arn)
+        allowed, explicit_deny = evaluate_policy_for_permissions(
+            statements,
+            permissions,
+            resource_arn,
+        )
 
         if explicit_deny:
 
@@ -157,9 +182,11 @@ def principal_allowed_on_resource(policies: Dict, resource_arn: str, permissions
 
 
 def calculate_permission_relationships(
-    principals: Dict, resource_arns: List[str], permissions: List[str],
+    principals: Dict,
+    resource_arns: List[str],
+    permissions: List[str],
 ) -> List[Dict]:
-    """ Evaluate principals permissions to resources
+    """Evaluate principals permissions to resources
     This currently only evaluates policies on IAM principals. It does not take into account
     Resource Policies - Policies attached to the resource instead of the IAM principal
     Permission Boundaries - Boundaries for an IAM principal
@@ -180,24 +207,14 @@ def calculate_permission_relationships(
     for resource_arn in resource_arns:
         for principal_arn, policies in principals.items():
             if principal_allowed_on_resource(policies, resource_arn, permissions):
-                allowed_mappings.append({"principal_arn": principal_arn, "resource_arn": resource_arn})
+                allowed_mappings.append(
+                    {"principal_arn": principal_arn, "resource_arn": resource_arn},
+                )
     return allowed_mappings
 
 
-def parse_statement_node(node_group: List[Any]) -> List[Any]:
-    """ Parse a dict from group of Neo4J node
-
-    Arguments:
-        node_group {[Neo4j.Node]} -- the node to parse
-
-    Returns:
-        [list] -- A list of statements from the node
-    """
-    return [n._properties for n in node_group]
-
-
 def compile_regex(item: str) -> Pattern:
-    r""" Compile a clause into a regex. Clause checking in AWS is case insensitive
+    r"""Compile a clause into a regex. Clause checking in AWS is case insensitive
     The following regex symbols will be replaced to make AWS * and ? matching a regex
     * -> .* (wildcard)
     ? -> .? (single character wildcard)
@@ -225,7 +242,7 @@ def compile_regex(item: str) -> Pattern:
 
 
 def compile_statement(statements: List[Any]) -> List[Any]:
-    """ Compile a statement by precompiling the regex for the relevant clauses. This is done to boost
+    """Compile a statement by precompiling the regex for the relevant clauses. This is done to boost
     performance by not recompiling the regex over and over again.
 
     Arguments:
@@ -234,11 +251,13 @@ def compile_statement(statements: List[Any]) -> List[Any]:
     Returns:
         [dict] -- the compiled statement
     """
-    properties = ['action', 'resource', 'notresource', 'notaction']
+    properties = ["action", "resource", "notresource", "notaction"]
     for statement in statements:
         for statement_property in properties:
             if statement_property in statement:
-                statement[statement_property] = [compile_regex(item) for item in statement[statement_property]]
+                statement[statement_property] = [
+                    compile_regex(item) for item in statement[statement_property]
+                ]
     return statements
 
 
@@ -252,7 +271,8 @@ def get_principals_for_account(neo4j_session: neo4j.Session, account_id: str) ->
     RETURN
     DISTINCT principal.arn as principal_arn, policy.id as policy_id, collect(statements) as statements
     """
-    results = neo4j_session.run(
+    results = neo4j_session.execute_read(
+        read_list_of_dicts_tx,
         get_policy_query,
         AccountId=account_id,
     )
@@ -263,42 +283,55 @@ def get_principals_for_account(neo4j_session: neo4j.Session, account_id: str) ->
         statements = r["statements"]
         if principal_arn not in principals:
             principals[principal_arn] = {}
-        principals[principal_arn][policy_id] = compile_statement(parse_statement_node(statements))
+        principals[principal_arn][policy_id] = compile_statement(statements)
     return principals
 
 
-def get_resource_arns(neo4j_session: neo4j.Session, account_id: str, node_label: str) -> List[Any]:
-    get_resource_query = Template("""
+def get_resource_arns(
+    neo4j_session: neo4j.Session,
+    account_id: str,
+    node_label: str,
+) -> List[Any]:
+    get_resource_query = Template(
+        """
     MATCH (acc:AWSAccount{id:$AccountId})-[:RESOURCE]->(resource:$node_label)
     return resource.arn as arn
-    """)
-    get_resource_query_template = get_resource_query.safe_substitute(node_label=node_label)
-    results = neo4j_session.run(
+    """,
+    )
+    get_resource_query_template = get_resource_query.safe_substitute(
+        node_label=node_label,
+    )
+    return neo4j_session.execute_read(
+        read_list_of_values_tx,
         get_resource_query_template,
         AccountId=account_id,
     )
-    arns = [r["arn"] for r in results]
-    return arns
 
 
 def load_principal_mappings(
-    neo4j_session: neo4j.Session, principal_mappings: List[Dict], node_label: str,
-    relationship_name: str, update_tag: int,
+    neo4j_session: neo4j.Session,
+    principal_mappings: List[Dict],
+    node_label: str,
+    relationship_name: str,
+    update_tag: int,
 ) -> None:
-    map_policy_query = Template("""
+    map_policy_query = Template(
+        """
     UNWIND $Mapping as mapping
     MATCH (principal:AWSPrincipal{arn:mapping.principal_arn})
     MATCH (resource:$node_label{arn:mapping.resource_arn})
     MERGE (principal)-[r:$relationship_name]->(resource)
     SET r.lastupdated = $aws_update_tag
-    """)
+    """,
+    )
     if not principal_mappings:
         return
     map_policy_query_template = map_policy_query.safe_substitute(
         node_label=node_label,
         relationship_name=relationship_name,
     )
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         map_policy_query_template,
         Mapping=principal_mappings,
         aws_update_tag=update_tag,
@@ -306,24 +339,37 @@ def load_principal_mappings(
 
 
 def cleanup_rpr(
-    neo4j_session: neo4j.Session, node_label: str, relationship_name: str, update_tag: int,
+    neo4j_session: neo4j.Session,
+    node_label: str,
+    relationship_name: str,
+    update_tag: int,
     current_aws_id: str,
 ) -> None:
-    logger.info("Cleaning up relationship '%s' for node label '%s'", relationship_name, node_label)
-    cleanup_rpr_query = Template("""
+    logger.info(
+        "Cleaning up relationship '%s' for node label '%s'",
+        relationship_name,
+        node_label,
+    )
+    cleanup_rpr_query = Template(
+        """
         MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(principal:AWSPrincipal)-[r:$relationship_name]->
         (resource:$node_label)
         WHERE r.lastupdated <> $UPDATE_TAG
         WITH r LIMIT $LIMIT_SIZE  DELETE (r) return COUNT(*) as TotalCompleted
-    """)
+    """,
+    )
     cleanup_rpr_query_template = cleanup_rpr_query.safe_substitute(
         node_label=node_label,
         relationship_name=relationship_name,
     )
 
     statement = GraphStatement(
-        cleanup_rpr_query_template, {'UPDATE_TAG': update_tag, 'AWS_ID': current_aws_id},
-        True, 1000,
+        cleanup_rpr_query_template,
+        {"UPDATE_TAG": update_tag, "AWS_ID": current_aws_id},
+        True,
+        1000,
+        parent_job_name=f"{relationship_name}:{node_label}",
+        parent_job_sequence_num=1,
     )
     statement.run(neo4j_session)
 
@@ -355,10 +401,17 @@ def is_valid_rpr(rpr: Dict) -> bool:
 
 @timeit
 def sync(
-    neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str], current_aws_account_id: str,
-    update_tag: int, common_job_parameters: Dict,
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict,
 ) -> None:
-    logger.info("Syncing Permission Relationships for account '%s'.", current_aws_account_id)
+    logger.info(
+        "Syncing Permission Relationships for account '%s'.",
+        current_aws_account_id,
+    )
     principals = get_principals_for_account(neo4j_session, current_aws_account_id)
     pr_file = common_job_parameters["permission_relationships_file"]
     if not pr_file:
@@ -370,18 +423,41 @@ def sync(
     relationship_mapping = parse_permission_relationships_file(pr_file)
     for rpr in relationship_mapping:
         if not is_valid_rpr(rpr):
-            raise ValueError("""
+            raise ValueError(
+                """
         Resource permission relationship is missing fields.
         Required fields: permissions, relationship_name, target_label"
-        """)
+        """,
+            )
         permissions = rpr["permissions"]
         relationship_name = rpr["relationship_name"]
         target_label = rpr["target_label"]
-        resource_arns = get_resource_arns(neo4j_session, current_aws_account_id, target_label)
-        logger.info("Syncing relationship '%s' for node label '%s'", relationship_name, target_label)
-        allowed_mappings = calculate_permission_relationships(principals, resource_arns, permissions)
+        resource_arns = get_resource_arns(
+            neo4j_session,
+            current_aws_account_id,
+            target_label,
+        )
+        logger.info(
+            "Syncing relationship '%s' for node label '%s'",
+            relationship_name,
+            target_label,
+        )
+        allowed_mappings = calculate_permission_relationships(
+            principals,
+            resource_arns,
+            permissions,
+        )
         load_principal_mappings(
-            neo4j_session, allowed_mappings,
-            target_label, relationship_name, update_tag,
+            neo4j_session,
+            allowed_mappings,
+            target_label,
+            relationship_name,
+            update_tag,
+        )
+        cleanup_rpr(
+            neo4j_session,
+            target_label,
+            relationship_name,
+            update_tag,
+            current_aws_account_id,
         )
-        cleanup_rpr(neo4j_session, target_label, relationship_name, update_tag, current_aws_account_id)