cartography 0.110.0rc2__py3-none-any.whl → 0.111.0__py3-none-any.whl

cartography/intel/aws/ec2/vpc.py

@@ -1,13 +1,15 @@
 import logging
-from string import Template
-from typing import Dict
-from typing import List
+from typing import Any
 
 import boto3
 import neo4j
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.ec2.vpc import AWSVpcSchema
+from cartography.models.aws.ec2.vpc_cidr import AWSIPv4CidrBlockSchema
+from cartography.models.aws.ec2.vpc_cidr import AWSIPv6CidrBlockSchema
 from cartography.util import aws_handle_regions
-from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 from .util import get_botocore_config
@@ -17,87 +19,78 @@ logger = logging.getLogger(__name__)
 
 @timeit
 @aws_handle_regions
-def get_ec2_vpcs(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
+def get_ec2_vpcs(
+    boto3_session: boto3.session.Session,
+    region: str,
+) -> list[dict[str, Any]]:
     client = boto3_session.client(
         "ec2",
         region_name=region,
         config=get_botocore_config(),
     )
-    return client.describe_vpcs()["Vpcs"]
-
-
-def _get_cidr_association_statement(block_type: str) -> str:
-    INGEST_CIDR_TEMPLATE = Template(
-        """
-    MATCH (vpc:AWSVpc{id: $VpcId})
-    WITH vpc
-    UNWIND $CidrBlock as block_data
-        MERGE (new_block:$block_label{id: $VpcId + '|' + block_data.$block_cidr})
-        ON CREATE SET new_block.firstseen = timestamp()
-        SET new_block.association_id = block_data.AssociationId,
-        new_block.cidr_block = block_data.$block_cidr,
-        new_block.block_state = block_data.$state_name.State,
-        new_block.block_state_message = block_data.$state_name.StatusMessage,
-        new_block.lastupdated = $update_tag
-        WITH vpc, new_block
-        MERGE (vpc)-[r:BLOCK_ASSOCIATION]->(new_block)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $update_tag""",
-    )
-
-    BLOCK_CIDR = "CidrBlock"
-    STATE_NAME = "CidrBlockState"
-
-    # base label type. We add the AWS ipv4 or 6 depending on block type
-    BLOCK_TYPE = "AWSCidrBlock"
-
-    if block_type == "ipv6":
-        BLOCK_CIDR = "Ipv6" + BLOCK_CIDR
-        STATE_NAME = "Ipv6" + STATE_NAME
-        BLOCK_TYPE = BLOCK_TYPE + ":AWSIpv6CidrBlock"
-    elif block_type == "ipv4":
-        BLOCK_TYPE = BLOCK_TYPE + ":AWSIpv4CidrBlock"
-    else:
-        raise ValueError(f"Unsupported block type specified - {block_type}")
-
-    return INGEST_CIDR_TEMPLATE.safe_substitute(
-        block_label=BLOCK_TYPE,
-        block_cidr=BLOCK_CIDR,
-        state_name=STATE_NAME,
-    )
-
-
-@timeit
-def load_cidr_association_set(
-    neo4j_session: neo4j.Session,
-    vpc_id: str,
-    vpc_data: Dict,
-    block_type: str,
-    update_tag: int,
-) -> None:
-    ingest_statement = _get_cidr_association_statement(block_type)
-
-    if block_type == "ipv6":
-        data = vpc_data.get("Ipv6CidrBlockAssociationSet", [])
-    else:
-        data = vpc_data.get("CidrBlockAssociationSet", [])
-
-    neo4j_session.run(
-        ingest_statement,
-        VpcId=vpc_id,
-        CidrBlock=data,
-        update_tag=update_tag,
-    )
+    return client.describe_vpcs().get("Vpcs", [])
+
+
+def transform_vpc_data(
+    vpc_list: list[dict[str, Any]], region: str
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]:
+
+    vpc_data: list[dict[str, Any]] = []
+    ipv4_cidr_blocks: list[dict[str, Any]] = []
+    ipv6_cidr_blocks: list[dict[str, Any]] = []
+
+    for vpc in vpc_list:
+        vpc_record = {
+            "VpcId": vpc.get("VpcId"),
+            "InstanceTenancy": vpc.get("InstanceTenancy"),
+            "State": vpc.get("State"),
+            "IsDefault": vpc.get("IsDefault"),
+            "PrimaryCIDRBlock": vpc.get("CidrBlock"),
+            "DhcpOptionsId": vpc.get("DhcpOptionsId"),
+            "lastupdated": vpc.get("lastupdated"),
+        }
+        vpc_data.append(vpc_record)
+
+        ipv4_associations = vpc.get("CidrBlockAssociationSet", [])
+        for association in ipv4_associations:
+            ipv4_block = {
+                "Id": vpc["VpcId"] + "|" + association.get("CidrBlock"),
+                "VpcId": vpc["VpcId"],
+                "AssociationId": association.get("AssociationId"),
+                "CidrBlock": association.get("CidrBlock"),
+                "BlockState": association.get("CidrBlockState", {}).get("State"),
+                "BlockStateMessage": association.get("CidrBlockState", {}).get(
+                    "StatusMessage"
+                ),
+            }
+            ipv4_cidr_blocks.append(ipv4_block)
+
+        ipv6_associations = vpc.get("Ipv6CidrBlockAssociationSet", [])
+        for association in ipv6_associations:
+            ipv6_block = {
+                "Id": vpc["VpcId"] + "|" + association.get("Ipv6CidrBlock"),
+                "VpcId": vpc["VpcId"],
+                "AssociationId": association.get("AssociationId"),
+                "CidrBlock": association.get("Ipv6CidrBlock"),
+                "BlockState": association.get("Ipv6CidrBlockState", {}).get("State"),
+                "BlockStateMessage": association.get("Ipv6CidrBlockState", {}).get(
+                    "StatusMessage"
+                ),
+            }
+            ipv6_cidr_blocks.append(ipv6_block)
+
+    return vpc_data, ipv4_cidr_blocks, ipv6_cidr_blocks
 
 
 @timeit
 def load_ec2_vpcs(
     neo4j_session: neo4j.Session,
-    data: List[Dict],
+    vpcs: list[dict[str, Any]],
     region: str,
-    current_aws_account_id: str,
+    aws_account_id: str,
     update_tag: int,
 ) -> None:
+    logger.info(f"Loading {len(vpcs)} EC2 VPCs for region '{region}' into graph.")
     # https://docs.aws.amazon.com/cli/latest/reference/ec2/describe-vpcs.html
     # {
     #     "Vpcs": [
@@ -126,69 +119,69 @@ def load_ec2_vpcs(
     #         }
     #     ]
     # }
+    load(
+        neo4j_session,
+        AWSVpcSchema(),
+        vpcs,
+        lastupdated=update_tag,
+        Region=region,
+        AWS_ID=aws_account_id,
+    )
 
-    ingest_vpc = """
-    MERGE (new_vpc:AWSVpc{id: $VpcId})
-    ON CREATE SET new_vpc.firstseen = timestamp(), new_vpc.vpcid =$VpcId
-    SET new_vpc.instance_tenancy = $InstanceTenancy,
-    new_vpc.state = $State,
-    new_vpc.is_default = $IsDefault,
-    new_vpc.primary_cidr_block = $PrimaryCIDRBlock,
-    new_vpc.dhcp_options_id = $DhcpOptionsId,
-    new_vpc.region = $Region,
-    new_vpc.lastupdated = $update_tag
-    WITH new_vpc
-    MATCH (awsAccount:AWSAccount{id: $AWS_ACCOUNT_ID})
-    MERGE (awsAccount)-[r:RESOURCE]->(new_vpc)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $update_tag"""
-
-    for vpc in data:
-        vpc_id = vpc["VpcId"]  # fail if not present
-
-        neo4j_session.run(
-            ingest_vpc,
-            VpcId=vpc_id,
-            InstanceTenancy=vpc.get("InstanceTenancy", None),
-            State=vpc.get("State", None),
-            IsDefault=vpc.get("IsDefault", None),
-            PrimaryCIDRBlock=vpc.get("CidrBlock", None),
-            DhcpOptionsId=vpc.get("DhcpOptionsId", None),
-            Region=region,
-            AWS_ACCOUNT_ID=current_aws_account_id,
-            update_tag=update_tag,
-        )
 
-        load_cidr_association_set(
-            neo4j_session,
-            vpc_id=vpc_id,
-            block_type="ipv4",
-            vpc_data=vpc,
-            update_tag=update_tag,
-        )
+@timeit
+def load_ipv4_cidr_blocks(
+    neo4j_session: neo4j.Session,
+    ipv4_cidr_blocks: list[dict[str, Any]],
+    region: str,
+    aws_account_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        AWSIPv4CidrBlockSchema(),
+        ipv4_cidr_blocks,
+        lastupdated=update_tag,
+    )
 
-        load_cidr_association_set(
-            neo4j_session,
-            vpc_id=vpc_id,
-            block_type="ipv6",
-            vpc_data=vpc,
-            update_tag=update_tag,
-        )
+
+@timeit
+def load_ipv6_cidr_blocks(
+    neo4j_session: neo4j.Session,
+    ipv6_cidr_blocks: list[dict[str, Any]],
+    region: str,
+    aws_account_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        AWSIPv6CidrBlockSchema(),
+        ipv6_cidr_blocks,
+        lastupdated=update_tag,
+    )
 
 
 @timeit
-def cleanup_ec2_vpcs(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
-    run_cleanup_job("aws_import_vpc_cleanup.json", neo4j_session, common_job_parameters)
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(AWSIPv6CidrBlockSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(AWSIPv4CidrBlockSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(AWSVpcSchema(), common_job_parameters).run(neo4j_session)
 
 
 @timeit
 def sync_vpc(
     neo4j_session: neo4j.Session,
     boto3_session: boto3.session.Session,
-    regions: List[str],
+    regions: list[str],
     current_aws_account_id: str,
     update_tag: int,
-    common_job_parameters: Dict,
+    common_job_parameters: dict[str, Any],
 ) -> None:
     for region in regions:
         logger.info(
@@ -196,6 +189,29 @@ def sync_vpc(
             region,
             current_aws_account_id,
         )
-        data = get_ec2_vpcs(boto3_session, region)
-        load_ec2_vpcs(neo4j_session, data, region, current_aws_account_id, update_tag)
-    cleanup_ec2_vpcs(neo4j_session, common_job_parameters)
+        raw_vpc_data = get_ec2_vpcs(boto3_session, region)
+        vpc_data, ipv4_cidr_blocks, ipv6_cidr_blocks = transform_vpc_data(
+            raw_vpc_data, region
+        )
+        load_ec2_vpcs(
+            neo4j_session,
+            vpc_data,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+        load_ipv4_cidr_blocks(
+            neo4j_session,
+            ipv4_cidr_blocks,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+        load_ipv6_cidr_blocks(
+            neo4j_session,
+            ipv6_cidr_blocks,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/eventbridge.py

@@ -10,6 +10,7 @@ from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
 from cartography.intel.aws.ec2.util import get_botocore_config
 from cartography.models.aws.eventbridge.rule import EventBridgeRuleSchema
+from cartography.models.aws.eventbridge.target import EventBridgeTargetSchema
 from cartography.util import aws_handle_regions
 from cartography.util import timeit
 
@@ -33,6 +34,44 @@ def get_eventbridge_rules(
     return rules
 
 
+@timeit
+@aws_handle_regions
+def get_eventbridge_targets(
+    boto3_session: boto3.Session, region: str, rules: List[Dict[str, Any]]
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "events", region_name=region, config=get_botocore_config()
+    )
+    targets = []
+    for rule in rules:
+        paginator = client.get_paginator("list_targets_by_rule")
+        for page in paginator.paginate(Rule=rule["Name"]):
+            for target in page.get("Targets", []):
+                target["RuleArn"] = rule["Arn"]
+                targets.append(target)
+    return targets
+
+
+def transform_eventbridge_targets(
+    targets: List[Dict[str, Any]],
+    region: str,
+) -> List[Dict[str, Any]]:
+    """
+    Transform EventBridge target data for ingestion into Neo4j.
+    """
+    transformed_data = []
+    for target in targets:
+        transformed_target = {
+            "Id": target["Arn"],
+            "Arn": target["Arn"],
+            "RuleArn": target["RuleArn"],
+            "RoleArn": target.get("RoleArn"),
+            "Region": region,
+        }
+        transformed_data.append(transformed_target)
+    return transformed_data
+
+
 @timeit
 def load_eventbridge_rules(
     neo4j_session: neo4j.Session,
@@ -54,6 +93,27 @@ def load_eventbridge_rules(
     )
 
 
+@timeit
+def load_eventbridge_targets(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading EventBridge {len(data)} targets for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        EventBridgeTargetSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
 @timeit
 def cleanup(
     neo4j_session: neo4j.Session,
@@ -63,6 +123,9 @@ def cleanup(
     GraphJob.from_node_schema(EventBridgeRuleSchema(), common_job_parameters).run(
         neo4j_session
     )
+    GraphJob.from_node_schema(EventBridgeTargetSchema(), common_job_parameters).run(
+        neo4j_session
+    )
 
 
 @timeit
@@ -88,4 +151,14 @@ def sync(
             update_tag,
         )
 
+        targets = get_eventbridge_targets(boto3_session, region, rules)
+        transformed_targets = transform_eventbridge_targets(targets, region)
+        load_eventbridge_targets(
+            neo4j_session,
+            transformed_targets,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
     cleanup(neo4j_session, common_job_parameters)

cartography/intel/github/repos.py

@@ -41,12 +41,12 @@ UserAffiliationAndRepoPermission = namedtuple(
 
 
 GITHUB_ORG_REPOS_PAGINATED_GRAPHQL = """
-    query($login: String!, $cursor: String) {
+    query($login: String!, $cursor: String, $count: Int!) {
     organization(login: $login)
         {
             url
             login
-            repositories(first: 50, after: $cursor){
+            repositories(first: $count, after: $cursor){
                 pageInfo{
                     endCursor
                     hasNextPage
@@ -168,14 +168,22 @@ def _get_repo_collaborators_inner_func(
         repo_name = repo["name"]
         repo_url = repo["url"]
 
-        if (
-            affiliation == "OUTSIDE" and repo["outsideCollaborators"]["totalCount"] == 0
-        ) or (
-            affiliation == "DIRECT" and repo["directCollaborators"]["totalCount"] == 0
-        ):
-            # repo has no collabs of the affiliation type we're looking for, so don't waste time making an API call
-            result[repo_url] = []
-            continue
+        # Guard against None when collaborator fields are not accessible due to permissions.
+        direct_info = repo.get("directCollaborators")
+        outside_info = repo.get("outsideCollaborators")
+
+        if affiliation == "OUTSIDE":
+            total_outside = 0 if not outside_info else outside_info.get("totalCount", 0)
+            if total_outside == 0:
+                # No outside collaborators or not permitted to view; skip API calls for this repo.
+                result[repo_url] = []
+                continue
+        else:  # DIRECT
+            total_direct = 0 if not direct_info else direct_info.get("totalCount", 0)
+            if total_direct == 0:
+                # No direct collaborators or not permitted to view; skip API calls for this repo.
+                result[repo_url] = []
+                continue
 
         logger.info(f"Loading {affiliation} collaborators for repo {repo_name}.")
         collaborators = _get_repo_collaborators(
@@ -290,6 +298,7 @@ def get(token: str, api_url: str, organization: str) -> List[Dict]:
         organization,
         GITHUB_ORG_REPOS_PAGINATED_GRAPHQL,
         "repositories",
+        count=50,
     )
     return repos.nodes
 
@@ -405,9 +414,16 @@ def _create_default_branch_id(repo_url: str, default_branch_ref_id: str) -> str:
 
 def _create_git_url_from_ssh_url(ssh_url: str) -> str:
     """
-    Return a git:// URL from the given ssh_url
+    Convert SSH URL to git:// URL.
+    Example:
+        git@github.com:cartography-cncf/cartography.git
+        -> git://github.com/cartography-cncf/cartography.git
     """
-    return ssh_url.replace("/", ":").replace("git@", "git://")
+    # Remove the user part (e.g., "git@")
+    _, host_and_path = ssh_url.split("@", 1)
+    # Replace first ':' (separating host and repo) with '/'
+    host, path = host_and_path.split(":", 1)
+    return f"git://{host}/{path}"
 
 
 def _transform_repo_objects(input_repo_object: Dict, out_repo_list: List[Dict]) -> None:

cartography/intel/github/util.py

@@ -157,6 +157,18 @@ def fetch_all(
             retry += 1
             exc = err
         except requests.exceptions.HTTPError as err:
+            if (
+                err.response is not None
+                and err.response.status_code == 502
+                and kwargs.get("count")
+                and kwargs["count"] > 1
+            ):
+                kwargs["count"] = max(1, kwargs["count"] // 2)
+                logger.warning(
+                    "GitHub: Received 502 response. Reducing page size to %s and retrying.",
+                    kwargs["count"],
+                )
+                continue
             retry += 1
             exc = err
         except requests.exceptions.ChunkedEncodingError as err:

cartography/intel/keycloak/__init__.py

@@ -0,0 +1,153 @@
+import logging
+
+import neo4j
+import requests
+
+import cartography.intel.keycloak.authenticationexecutions
+import cartography.intel.keycloak.authenticationflows
+import cartography.intel.keycloak.clients
+import cartography.intel.keycloak.groups
+import cartography.intel.keycloak.identityproviders
+import cartography.intel.keycloak.organizations
+import cartography.intel.keycloak.realms
+import cartography.intel.keycloak.roles
+import cartography.intel.keycloak.scopes
+import cartography.intel.keycloak.users
+from cartography.config import Config
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def start_keycloak_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of Keycloak data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+    if (
+        not config.keycloak_client_id
+        or not config.keycloak_client_secret
+        or not config.keycloak_url
+        or not config.keycloak_realm
+    ):
+        logger.info(
+            "Keycloak import is not configured - skipping this module. "
+            "See docs to configure.",
+        )
+        return
+
+    # Create requests sessions
+    with requests.session() as api_session:
+        payload = {
+            "grant_type": "client_credentials",
+            "client_id": config.keycloak_client_id,
+            "client_secret": config.keycloak_client_secret,
+        }
+        req = api_session.post(
+            f"{config.keycloak_url}/realms/{config.keycloak_realm}/protocol/openid-connect/token",
+            data=payload,
+            timeout=_TIMEOUT,
+        )
+        req.raise_for_status()
+        api_session.headers.update(
+            {"Authorization": f'Bearer {req.json()["access_token"]}'}
+        )
+
+        common_job_parameters = {
+            "UPDATE_TAG": config.update_tag,
+        }
+
+        for realm in cartography.intel.keycloak.realms.sync(
+            neo4j_session, api_session, config.keycloak_url, common_job_parameters
+        ):
+            realm_scopped_job_parameters = {
+                "UPDATE_TAG": config.update_tag,
+                "REALM": realm["realm"],
+                "REALM_ID": realm["id"],
+            }
+            cartography.intel.keycloak.users.sync(
+                neo4j_session,
+                api_session,
+                config.keycloak_url,
+                realm_scopped_job_parameters,
+            )
+            cartography.intel.keycloak.identityproviders.sync(
+                neo4j_session,
+                api_session,
+                config.keycloak_url,
+                realm_scopped_job_parameters,
+            )
+            scopes = cartography.intel.keycloak.scopes.sync(
+                neo4j_session,
+                api_session,
+                config.keycloak_url,
+                realm_scopped_job_parameters,
+            )
+            scope_ids = [s["id"] for s in scopes]
+            flows = cartography.intel.keycloak.authenticationflows.sync(
+                neo4j_session,
+                api_session,
+                config.keycloak_url,
+                realm_scopped_job_parameters,
+            )
+            flow_aliases_to_id = {f["alias"]: f["id"] for f in flows}
+            cartography.intel.keycloak.authenticationexecutions.sync(
+                neo4j_session,
+                api_session,
+                config.keycloak_url,
+                realm_scopped_job_parameters,
+                list(flow_aliases_to_id.keys()),
+            )
+            realm_default_flows = {
+                "browser": flow_aliases_to_id.get(realm.get("browserFlow")),
+                "registration": flow_aliases_to_id.get(realm.get("registrationFlow")),
+                "direct_grant": flow_aliases_to_id.get(realm.get("directGrantFlow")),
+                "reset_credentials": flow_aliases_to_id.get(
+                    realm.get("resetCredentialsFlow")
+                ),
+                "client_authentication": flow_aliases_to_id.get(
+                    realm.get("clientAuthenticationFlow")
+                ),
+                "docker_authentication": flow_aliases_to_id.get(
+                    realm.get("dockerAuthenticationFlow")
+                ),
+                "first_broker_login": flow_aliases_to_id.get(
+                    realm.get("firstBrokerLoginFlow")
+                ),
+            }
+
+            clients = cartography.intel.keycloak.clients.sync(
+                neo4j_session,
+                api_session,
+                config.keycloak_url,
+                realm_scopped_job_parameters,
+                realm_default_flows,
+            )
+            client_ids = [c["id"] for c in clients]
+            cartography.intel.keycloak.roles.sync(
+                neo4j_session,
+                api_session,
+                config.keycloak_url,
+                realm_scopped_job_parameters,
+                client_ids,
+                scope_ids,
+            )
+            cartography.intel.keycloak.groups.sync(
+                neo4j_session,
+                api_session,
+                config.keycloak_url,
+                realm_scopped_job_parameters,
+            )
+
+            # Organizations if they are enabled
+            if realm.get("organizationsEnabled", False):
+                cartography.intel.keycloak.organizations.sync(
+                    neo4j_session,
+                    api_session,
+                    config.keycloak_url,
+                    realm_scopped_job_parameters,
+                )