cartography 0.103.0rc1__py3-none-any.whl → 0.104.0rc2__py3-none-any.whl

cartography/intel/aws/efs.py

@@ -0,0 +1,93 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.efs.mount_target import EfsMountTargetSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_efs_mount_targets(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "efs", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("describe_mount_targets")
+    mountTargets = []
+    for page in paginator.paginate():
+        mountTargets.extend(page["MountTargets"])
+    return mountTargets
+
+
+@timeit
+def load_efs_mount_targets(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Efs {len(data)} mount targets for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        EfsMountTargetSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.debug("Running Efs cleanup job.")
+    cleanup_job = GraphJob.from_node_schema(
+        EfsMountTargetSchema(), common_job_parameters
+    )
+    cleanup_job.run(neo4j_session)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing Efs for region '{region}' in account '{current_aws_account_id}'.",
+        )
+        mountTargets = get_efs_mount_targets(boto3_session, region)
+        mount_target_data: List[Dict[str, Any]] = []
+        for mountTarget in mountTargets:
+            mount_target_data.append(mountTarget)
+
+        load_efs_mount_targets(
+            neo4j_session,
+            mount_target_data,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/resources.py

@@ -5,10 +5,12 @@ from cartography.intel.aws.ec2.route_tables import sync_route_tables
 
 from . import apigateway
 from . import cloudtrail
+from . import cloudwatch
 from . import config
 from . import dynamodb
 from . import ecr
 from . import ecs
+from . import efs
 from . import eks
 from . import elasticache
 from . import elasticsearch
@@ -102,4 +104,6 @@ RESOURCE_FUNCTIONS: Dict[str, Callable[..., None]] = {
     "config": config.sync,
     "identitycenter": identitycenter.sync_identity_center_instances,
     "cloudtrail": cloudtrail.sync,
+    "cloudwatch": cloudwatch.sync,
+    "efs": efs.sync,
 }

cartography/intel/aws/secretsmanager.py

@@ -5,12 +5,20 @@ from typing import List
 import boto3
 import neo4j
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.secretsmanager.secret_version import (
+    SecretsManagerSecretVersionSchema,
+)
+from cartography.stats import get_stats_client
 from cartography.util import aws_handle_regions
 from cartography.util import dict_date_to_epoch
+from cartography.util import merge_module_sync_metadata
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
+stat_handler = get_stats_client(__name__)
 
 
 @timeit
@@ -76,6 +84,93 @@ def cleanup_secrets(neo4j_session: neo4j.Session, common_job_parameters: Dict) -
     )
 
 
+@timeit
+@aws_handle_regions
+def get_secret_versions(
+    boto3_session: boto3.session.Session, region: str, secret_arn: str
+) -> List[Dict]:
+    """
+    Get all versions of a secret from AWS Secrets Manager.
+    """
+    client = boto3_session.client("secretsmanager", region_name=region)
+    paginator = client.get_paginator("list_secret_version_ids")
+    versions = []
+
+    for page in paginator.paginate(SecretId=secret_arn, IncludeDeprecated=True):
+        for version in page["Versions"]:
+            version["SecretId"] = secret_arn
+            version["ARN"] = f"{secret_arn}:version:{version['VersionId']}"
+        versions.extend(page["Versions"])
+
+    return versions
+
+
+def transform_secret_versions(
+    versions: List[Dict],
+    region: str,
+    aws_account_id: str,
+) -> List[Dict]:
+    """
+    Transform AWS Secrets Manager Secret Versions to match the data model.
+    """
+    transformed_data = []
+    for version in versions:
+        transformed = {
+            "ARN": version["ARN"],
+            "SecretId": version["SecretId"],
+            "VersionId": version["VersionId"],
+            "VersionStages": version["VersionStages"],
+            "CreatedDate": dict_date_to_epoch(version, "CreatedDate"),
+        }
+
+        if "KmsKeyId" in version and version["KmsKeyId"]:
+            transformed["KmsKeyId"] = version["KmsKeyId"]
+
+        if "Tags" in version and version["Tags"]:
+            transformed["Tags"] = version["Tags"]
+
+        transformed_data.append(transformed)
+
+    return transformed_data
+
+
+@timeit
+def load_secret_versions(
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    region: str,
+    aws_account_id: str,
+    update_tag: int,
+) -> None:
+    """
+    Load secret versions into Neo4j using the data model.
+    """
+    logger.info(f"Loading {len(data)} Secret Versions for region {region} into graph.")
+
+    load(
+        neo4j_session,
+        SecretsManagerSecretVersionSchema(),
+        data,
+        lastupdated=update_tag,
+        Region=region,
+        AWS_ID=aws_account_id,
+    )
+
+
+@timeit
+def cleanup_secret_versions(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict
+) -> None:
+    """
+    Run Secret Versions cleanup job.
+    """
+    logger.debug("Running Secret Versions cleanup job.")
+    cleanup_job = GraphJob.from_node_schema(
+        SecretsManagerSecretVersionSchema(), common_job_parameters
+    )
+    cleanup_job.run(neo4j_session)
+
+
 @timeit
 def sync(
     neo4j_session: neo4j.Session,
@@ -85,12 +180,50 @@ def sync(
     update_tag: int,
     common_job_parameters: Dict,
 ) -> None:
+    """
+    Sync AWS Secrets Manager resources.
+    """
     for region in regions:
         logger.info(
-            "Syncing Secrets Manager for region '%s' in account '%s'.",
-            region,
-            current_aws_account_id,
+            f"Syncing Secrets Manager for region '{region}' in account '{current_aws_account_id}'."
         )
         secrets = get_secret_list(boto3_session, region)
+
         load_secrets(neo4j_session, secrets, region, current_aws_account_id, update_tag)
+
+        all_versions = []
+        for secret in secrets:
+            logger.info(
+                f"Getting versions for secret {secret.get('Name', 'unnamed')} ({secret['ARN']})"
+            )
+            versions = get_secret_versions(boto3_session, region, secret["ARN"])
+            logger.info(
+                f"Found {len(versions)} versions for secret {secret.get('Name', 'unnamed')}"
+            )
+            all_versions.extend(versions)
+
+        transformed_data = transform_secret_versions(
+            all_versions,
+            region,
+            current_aws_account_id,
+        )
+
+        load_secret_versions(
+            neo4j_session,
+            transformed_data,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
     cleanup_secrets(neo4j_session, common_job_parameters)
+    cleanup_secret_versions(neo4j_session, common_job_parameters)
+
+    merge_module_sync_metadata(
+        neo4j_session,
+        group_type="AWSAccount",
+        group_id=current_aws_account_id,
+        synced_type="SecretsManagerSecretVersion",
+        update_tag=update_tag,
+        stat_handler=stat_handler,
+    )

cartography/intel/aws/ssm.py

@@ -1,4 +1,6 @@
+import json
 import logging
+import re
 from typing import Any
 from typing import Dict
 from typing import List
@@ -10,6 +12,7 @@ from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
 from cartography.models.aws.ssm.instance_information import SSMInstanceInformationSchema
 from cartography.models.aws.ssm.instance_patch import SSMInstancePatchSchema
+from cartography.models.aws.ssm.parameters import SSMParameterSchema
 from cartography.util import aws_handle_regions
 from cartography.util import dict_date_to_epoch
 from cartography.util import timeit
@@ -107,6 +110,42 @@ def transform_instance_patches(data_list: List[Dict[str, Any]]) -> List[Dict[str
     return data_list
 
 
+@timeit
+@aws_handle_regions
+def get_ssm_parameters(
+    boto3_session: boto3.session.Session,
+    region: str,
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client("ssm", region_name=region)
+    paginator = client.get_paginator("describe_parameters")
+    ssm_parameters_data: List[Dict[str, Any]] = []
+    for page in paginator.paginate(PaginationConfig={"PageSize": 50}):
+        ssm_parameters_data.extend(page.get("Parameters", []))
+    return ssm_parameters_data
+
+
+def transform_ssm_parameters(
+    raw_parameters_data: List[Dict[str, Any]],
+) -> List[Dict[str, Any]]:
+    transformed_list: List[Dict[str, Any]] = []
+    for param in raw_parameters_data:
+        param["LastModifiedDate"] = dict_date_to_epoch(param, "LastModifiedDate")
+        param["PoliciesJson"] = json.dumps(param.get("Policies", []))
+        # KMSKey uses shorter UUID as their primary id
+        # SSM Parameters, when encrypted, reference KMS keys using their full ARNs in the KeyId field
+        # Adding a param to match on the id property of the target node
+        if param.get("Type") == "SecureString" and param.get("KeyId") is not None:
+            match = re.match(r".*key/(.*)$", param["KeyId"])
+            if match:
+                param["KMSKeyIdShort"] = match.group(1)
+            else:
+                param["KMSKeyIdShort"] = None
+        else:
+            param["KMSKeyIdShort"] = None
+        transformed_list.append(param)
+    return transformed_list
+
+
 @timeit
 def load_instance_information(
     neo4j_session: neo4j.Session,
@@ -143,6 +182,24 @@ def load_instance_patches(
     )
 
 
+@timeit
+def load_ssm_parameters(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        SSMParameterSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
 @timeit
 def cleanup_ssm(
     neo4j_session: neo4j.Session,
@@ -156,6 +213,9 @@ def cleanup_ssm(
     GraphJob.from_node_schema(SSMInstancePatchSchema(), common_job_parameters).run(
         neo4j_session,
     )
+    GraphJob.from_node_schema(SSMParameterSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
 
 
 @timeit
@@ -193,4 +253,15 @@ def sync(
             current_aws_account_id,
             update_tag,
         )
+
+        data = get_ssm_parameters(boto3_session, region)
+        data = transform_ssm_parameters(data)
+        load_ssm_parameters(
+            neo4j_session,
+            data,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
     cleanup_ssm(neo4j_session, common_job_parameters)

cartography/intel/cloudflare/__init__.py

@@ -0,0 +1,74 @@
+import logging
+
+import neo4j
+from cloudflare import Cloudflare
+
+import cartography.intel.cloudflare.accounts
+import cartography.intel.cloudflare.dnsrecords
+import cartography.intel.cloudflare.members
+import cartography.intel.cloudflare.roles
+import cartography.intel.cloudflare.zones
+from cartography.config import Config
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def start_cloudflare_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of Cloudflare data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+
+    if not config.cloudflare_token:
+        logger.info(
+            "Cloudflare import is not configured - skipping this module. "
+            "See docs to configure.",
+        )
+        return
+
+    # Create client
+    client = Cloudflare(api_token=config.cloudflare_token)
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+    }
+
+    for account in cartography.intel.cloudflare.accounts.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+    ):
+        account_job_parameters = common_job_parameters.copy()
+        account_job_parameters["account_id"] = account["id"]
+        cartography.intel.cloudflare.roles.sync(
+            neo4j_session,
+            client,
+            account_job_parameters,
+            account_id=account["id"],
+        )
+
+        cartography.intel.cloudflare.members.sync(
+            neo4j_session,
+            client,
+            account_job_parameters,
+            account_id=account["id"],
+        )
+
+        for zone in cartography.intel.cloudflare.zones.sync(
+            neo4j_session,
+            client,
+            account_job_parameters,
+            account_id=account["id"],
+        ):
+            zone_job_parameters = account_job_parameters.copy()
+            zone_job_parameters["zone_id"] = zone["id"]
+            cartography.intel.cloudflare.dnsrecords.sync(
+                neo4j_session,
+                client,
+                zone_job_parameters,
+                zone_id=zone["id"],
+            )

cartography/intel/cloudflare/accounts.py

@@ -0,0 +1,57 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+from cloudflare import Cloudflare
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.cloudflare.account import CloudflareAccountSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: Cloudflare,
+    common_job_parameters: Dict[str, Any],
+) -> List[Dict]:
+    accounts = get(client)
+    load_accounts(
+        neo4j_session,
+        accounts,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+    return accounts
+
+
+@timeit
+def get(client: Cloudflare) -> List[Dict[str, Any]]:
+    return [account.to_dict() for account in client.accounts.list()]
+
+
+def load_accounts(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Cloudflare accounts into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        CloudflareAccountSchema(),
+        data,
+        lastupdated=update_tag,
+    )
+
+
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(CloudflareAccountSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/cloudflare/dnsrecords.py

@@ -0,0 +1,64 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+from cloudflare import Cloudflare
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.cloudflare.dnsrecord import CloudflareDNSRecordSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: Cloudflare,
+    common_job_parameters: Dict[str, Any],
+    zone_id: str,
+) -> None:
+    dnsrecords = get(client, zone_id)
+    load_dnsrecords(
+        neo4j_session,
+        dnsrecords,
+        zone_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(client: Cloudflare, zone_id: str) -> List[Dict[str, Any]]:
+    return [
+        dnsrecord.to_dict() for dnsrecord in client.dns.records.list(zone_id=zone_id)
+    ]
+
+
+def load_dnsrecords(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    zone_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Cloudflare DNSRecords into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        CloudflareDNSRecordSchema(),
+        data,
+        lastupdated=update_tag,
+        zone_id=zone_id,
+    )
+
+
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(CloudflareDNSRecordSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/cloudflare/members.py

@@ -0,0 +1,75 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+from cloudflare import Cloudflare
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.cloudflare.member import CloudflareMemberSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: Cloudflare,
+    common_job_parameters: Dict[str, Any],
+    account_id: str,
+) -> None:
+    members = get(client, account_id)
+    transformed_members = transform_members(members)
+    load_members(
+        neo4j_session,
+        transformed_members,
+        account_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(client: Cloudflare, account_id: str) -> List[Dict[str, Any]]:
+    return [
+        member.to_dict()
+        for member in client.accounts.members.list(account_id=account_id)
+    ]
+
+
+def load_members(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    account_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Cloudflare members into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        CloudflareMemberSchema(),
+        data,
+        lastupdated=update_tag,
+        account_id=account_id,
+    )
+
+
+def transform_members(data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    result: List[Dict[str, Any]] = []
+    for member in data:
+        member["roles_ids"] = [role["id"] for role in member.get("roles", [])]
+        member["policies_ids"] = [policy["id"] for policy in member.get("policies", [])]
+        result.append(member)
+    return result
+
+
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(CloudflareMemberSchema(), common_job_parameters).run(
+        neo4j_session
+    )