cartography 0.103.0rc1__py3-none-any.whl → 0.104.0__py3-none-any.whl

cartography/intel/anthropic/workspaces.py

@@ -0,0 +1,95 @@
+import logging
+from typing import Any
+from typing import Tuple
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.anthropic.util import paginated_get
+from cartography.models.anthropic.workspace import AnthropicWorkspaceSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: dict[str, Any],
+) -> list[dict]:
+    org_id, workspaces = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+    )
+    common_job_parameters["ORG_ID"] = org_id
+    for workspace in workspaces:
+        workspace["users"] = []
+        workspace["admins"] = []
+        for user in get_workspace_users(
+            api_session,
+            common_job_parameters["BASE_URL"],
+            workspace["id"],
+        ):
+            workspace["users"].append(user["user_id"])
+            if user["workspace_role"] == "workspace_admin":
+                workspace["admins"].append(user["user_id"])
+    load_workspaces(
+        neo4j_session, workspaces, org_id, common_job_parameters["UPDATE_TAG"]
+    )
+    cleanup(neo4j_session, common_job_parameters)
+    return workspaces
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+) -> Tuple[str, list[dict[str, Any]]]:
+    return paginated_get(
+        api_session, f"{base_url}/organizations/workspaces", timeout=_TIMEOUT
+    )
+
+
+@timeit
+def get_workspace_users(
+    api_session: requests.Session,
+    base_url: str,
+    workspace_id: str,
+) -> list[dict[str, Any]]:
+    _, result = paginated_get(
+        api_session,
+        f"{base_url}/organizations/workspaces/{workspace_id}/members",
+        timeout=_TIMEOUT,
+    )
+    return result
+
+
+@timeit
+def load_workspaces(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    ORG_ID: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Anthropic workspaces into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        AnthropicWorkspaceSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=ORG_ID,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(AnthropicWorkspaceSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/aws/cloudtrail.py

@@ -4,7 +4,6 @@ from typing import Dict
 from typing import List
 
 import boto3
-import botocore.exceptions
 import neo4j
 
 from cartography.client.core.tx import load
@@ -25,10 +24,8 @@ def get_cloudtrail_trails(
     client = boto3_session.client(
         "cloudtrail", region_name=region, config=get_botocore_config()
     )
-    paginator = client.get_paginator("list_trails")
-    trails = []
-    for page in paginator.paginate():
-        trails.extend(page["Trails"])
+
+    trails = client.describe_trails()["trailList"]
 
     # CloudTrail multi-region trails are shown in list_trails,
     # but the get_trail call only works in the home region
@@ -36,28 +33,6 @@ def get_cloudtrail_trails(
     return trails_filtered
 
 
-@timeit
-def get_cloudtrail_trail(
-    boto3_session: boto3.Session,
-    region: str,
-    trail_name: str,
-) -> Dict[str, Any]:
-    client = boto3_session.client(
-        "cloudtrail", region_name=region, config=get_botocore_config()
-    )
-    trail_details: Dict[str, Any] = {}
-    try:
-        response = client.get_trail(Name=trail_name)
-        trail_details = response["Trail"]
-    except botocore.exceptions.ClientError as e:
-        code = e.response["Error"]["Code"]
-        msg = e.response["Error"]["Message"]
-        logger.warning(
-            f"Could not run CloudTrail get_trail due to boto3 error {code}: {msg}. Skipping.",
-        )
-    return trail_details
-
-
 @timeit
 def load_cloudtrail_trails(
     neo4j_session: neo4j.Session,
@@ -105,20 +80,10 @@ def sync(
             f"Syncing CloudTrail for region '{region}' in account '{current_aws_account_id}'.",
         )
         trails = get_cloudtrail_trails(boto3_session, region)
-        trail_data: List[Dict[str, Any]] = []
-        for trail in trails:
-            trail_name = trail["Name"]
-            trail_details = get_cloudtrail_trail(
-                boto3_session,
-                region,
-                trail_name,
-            )
-            if trail_details:
-                trail_data.append(trail_details)
 
         load_cloudtrail_trails(
             neo4j_session,
-            trail_data,
+            trails,
             region,
             current_aws_account_id,
             update_tag,

cartography/intel/aws/cloudwatch.py

@@ -0,0 +1,93 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.cloudwatch.loggroup import CloudWatchLogGroupSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_cloudwatch_log_groups(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "logs", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("describe_log_groups")
+    logGroups = []
+    for page in paginator.paginate():
+        logGroups.extend(page["logGroups"])
+    return logGroups
+
+
+@timeit
+def load_cloudwatch_log_groups(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading CloudWatch {len(data)} log groups for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        CloudWatchLogGroupSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.debug("Running CloudWatch cleanup job.")
+    cleanup_job = GraphJob.from_node_schema(
+        CloudWatchLogGroupSchema(), common_job_parameters
+    )
+    cleanup_job.run(neo4j_session)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing CloudWatch for region '{region}' in account '{current_aws_account_id}'.",
+        )
+        logGroups = get_cloudwatch_log_groups(boto3_session, region)
+        group_data: List[Dict[str, Any]] = []
+        for logGroup in logGroups:
+            group_data.append(logGroup)
+
+        load_cloudwatch_log_groups(
+            neo4j_session,
+            group_data,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/ec2/load_balancer_v2s.py

@@ -99,7 +99,10 @@ def load_load_balancer_v2s(
     SET r.lastupdated = $update_tag
     """
     for lb in data:
-        load_balancer_id = lb["DNSName"]
+        load_balancer_id = lb.get("DNSName")
+        if not load_balancer_id:
+            logger.warning("Skipping load balancer entry with missing DNSName: %r", lb)
+            continue
 
         neo4j_session.run(
             ingest_load_balancer_v2,

cartography/intel/aws/resources.py

@@ -5,6 +5,7 @@ from cartography.intel.aws.ec2.route_tables import sync_route_tables
 
 from . import apigateway
 from . import cloudtrail
+from . import cloudwatch
 from . import config
 from . import dynamodb
 from . import ecr
@@ -102,4 +103,5 @@ RESOURCE_FUNCTIONS: Dict[str, Callable[..., None]] = {
     "config": config.sync,
     "identitycenter": identitycenter.sync_identity_center_instances,
     "cloudtrail": cloudtrail.sync,
+    "cloudwatch": cloudwatch.sync,
 }

cartography/intel/aws/secretsmanager.py

@@ -5,12 +5,20 @@ from typing import List
 import boto3
 import neo4j
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.secretsmanager.secret_version import (
+    SecretsManagerSecretVersionSchema,
+)
+from cartography.stats import get_stats_client
 from cartography.util import aws_handle_regions
 from cartography.util import dict_date_to_epoch
+from cartography.util import merge_module_sync_metadata
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
+stat_handler = get_stats_client(__name__)
 
 
 @timeit
@@ -76,6 +84,107 @@ def cleanup_secrets(neo4j_session: neo4j.Session, common_job_parameters: Dict) -
     )
 
 
+@timeit
+@aws_handle_regions
+def get_secret_versions(
+    boto3_session: boto3.session.Session, region: str, secret_arn: str
+) -> List[Dict]:
+    """
+    Get all versions of a secret from AWS Secrets Manager.
+
+    Note: list_secret_version_ids is not paginatable through boto3's paginator,
+    so we implement manual pagination.
+    """
+    client = boto3_session.client("secretsmanager", region_name=region)
+    next_token = None
+    versions = []
+
+    while True:
+        params = {"SecretId": secret_arn, "IncludeDeprecated": True}
+        if next_token:
+            params["NextToken"] = next_token
+
+        response = client.list_secret_version_ids(**params)
+
+        for version in response.get("Versions", []):
+            version["SecretId"] = secret_arn
+            version["ARN"] = f"{secret_arn}:version:{version['VersionId']}"
+
+        versions.extend(response.get("Versions", []))
+
+        next_token = response.get("NextToken")
+        if not next_token:
+            break
+
+    return versions
+
+
+def transform_secret_versions(
+    versions: List[Dict],
+    region: str,
+    aws_account_id: str,
+) -> List[Dict]:
+    """
+    Transform AWS Secrets Manager Secret Versions to match the data model.
+    """
+    transformed_data = []
+    for version in versions:
+        transformed = {
+            "ARN": version["ARN"],
+            "SecretId": version["SecretId"],
+            "VersionId": version["VersionId"],
+            "VersionStages": version.get("VersionStages"),
+            "CreatedDate": dict_date_to_epoch(version, "CreatedDate"),
+        }
+
+        if "KmsKeyId" in version and version["KmsKeyId"]:
+            transformed["KmsKeyId"] = version["KmsKeyId"]
+
+        if "Tags" in version and version["Tags"]:
+            transformed["Tags"] = version["Tags"]
+
+        transformed_data.append(transformed)
+
+    return transformed_data
+
+
+@timeit
+def load_secret_versions(
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    region: str,
+    aws_account_id: str,
+    update_tag: int,
+) -> None:
+    """
+    Load secret versions into Neo4j using the data model.
+    """
+    logger.info(f"Loading {len(data)} Secret Versions for region {region} into graph.")
+
+    load(
+        neo4j_session,
+        SecretsManagerSecretVersionSchema(),
+        data,
+        lastupdated=update_tag,
+        Region=region,
+        AWS_ID=aws_account_id,
+    )
+
+
+@timeit
+def cleanup_secret_versions(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict
+) -> None:
+    """
+    Run Secret Versions cleanup job.
+    """
+    logger.debug("Running Secret Versions cleanup job.")
+    cleanup_job = GraphJob.from_node_schema(
+        SecretsManagerSecretVersionSchema(), common_job_parameters
+    )
+    cleanup_job.run(neo4j_session)
+
+
 @timeit
 def sync(
     neo4j_session: neo4j.Session,
@@ -85,12 +194,50 @@ def sync(
     update_tag: int,
     common_job_parameters: Dict,
 ) -> None:
+    """
+    Sync AWS Secrets Manager resources.
+    """
     for region in regions:
         logger.info(
-            "Syncing Secrets Manager for region '%s' in account '%s'.",
-            region,
-            current_aws_account_id,
+            f"Syncing Secrets Manager for region '{region}' in account '{current_aws_account_id}'."
         )
         secrets = get_secret_list(boto3_session, region)
+
         load_secrets(neo4j_session, secrets, region, current_aws_account_id, update_tag)
+
+        all_versions = []
+        for secret in secrets:
+            logger.info(
+                f"Getting versions for secret {secret.get('Name', 'unnamed')} ({secret['ARN']})"
+            )
+            versions = get_secret_versions(boto3_session, region, secret["ARN"])
+            logger.info(
+                f"Found {len(versions)} versions for secret {secret.get('Name', 'unnamed')}"
+            )
+            all_versions.extend(versions)
+
+        transformed_data = transform_secret_versions(
+            all_versions,
+            region,
+            current_aws_account_id,
+        )
+
+        load_secret_versions(
+            neo4j_session,
+            transformed_data,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
     cleanup_secrets(neo4j_session, common_job_parameters)
+    cleanup_secret_versions(neo4j_session, common_job_parameters)
+
+    merge_module_sync_metadata(
+        neo4j_session,
+        group_type="AWSAccount",
+        group_id=current_aws_account_id,
+        synced_type="SecretsManagerSecretVersion",
+        update_tag=update_tag,
+        stat_handler=stat_handler,
+    )

cartography/intel/aws/ssm.py

@@ -1,4 +1,6 @@
+import json
 import logging
+import re
 from typing import Any
 from typing import Dict
 from typing import List
@@ -10,6 +12,7 @@ from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
 from cartography.models.aws.ssm.instance_information import SSMInstanceInformationSchema
 from cartography.models.aws.ssm.instance_patch import SSMInstancePatchSchema
+from cartography.models.aws.ssm.parameters import SSMParameterSchema
 from cartography.util import aws_handle_regions
 from cartography.util import dict_date_to_epoch
 from cartography.util import timeit
@@ -107,6 +110,42 @@ def transform_instance_patches(data_list: List[Dict[str, Any]]) -> List[Dict[str
     return data_list
 
 
+@timeit
+@aws_handle_regions
+def get_ssm_parameters(
+    boto3_session: boto3.session.Session,
+    region: str,
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client("ssm", region_name=region)
+    paginator = client.get_paginator("describe_parameters")
+    ssm_parameters_data: List[Dict[str, Any]] = []
+    for page in paginator.paginate(PaginationConfig={"PageSize": 50}):
+        ssm_parameters_data.extend(page.get("Parameters", []))
+    return ssm_parameters_data
+
+
+def transform_ssm_parameters(
+    raw_parameters_data: List[Dict[str, Any]],
+) -> List[Dict[str, Any]]:
+    transformed_list: List[Dict[str, Any]] = []
+    for param in raw_parameters_data:
+        param["LastModifiedDate"] = dict_date_to_epoch(param, "LastModifiedDate")
+        param["PoliciesJson"] = json.dumps(param.get("Policies", []))
+        # KMSKey uses shorter UUID as their primary id
+        # SSM Parameters, when encrypted, reference KMS keys using their full ARNs in the KeyId field
+        # Adding a param to match on the id property of the target node
+        if param.get("Type") == "SecureString" and param.get("KeyId") is not None:
+            match = re.match(r".*key/(.*)$", param["KeyId"])
+            if match:
+                param["KMSKeyIdShort"] = match.group(1)
+            else:
+                param["KMSKeyIdShort"] = None
+        else:
+            param["KMSKeyIdShort"] = None
+        transformed_list.append(param)
+    return transformed_list
+
+
 @timeit
 def load_instance_information(
     neo4j_session: neo4j.Session,
@@ -143,6 +182,24 @@ def load_instance_patches(
     )
 
 
+@timeit
+def load_ssm_parameters(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        SSMParameterSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
 @timeit
 def cleanup_ssm(
     neo4j_session: neo4j.Session,
@@ -156,6 +213,9 @@ def cleanup_ssm(
     GraphJob.from_node_schema(SSMInstancePatchSchema(), common_job_parameters).run(
         neo4j_session,
     )
+    GraphJob.from_node_schema(SSMParameterSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
 
 
 @timeit
@@ -193,4 +253,15 @@ def sync(
             current_aws_account_id,
             update_tag,
         )
+
+        data = get_ssm_parameters(boto3_session, region)
+        data = transform_ssm_parameters(data)
+        load_ssm_parameters(
+            neo4j_session,
+            data,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
     cleanup_ssm(neo4j_session, common_job_parameters)

cartography/intel/cloudflare/__init__.py

@@ -0,0 +1,74 @@
+import logging
+
+import neo4j
+from cloudflare import Cloudflare
+
+import cartography.intel.cloudflare.accounts
+import cartography.intel.cloudflare.dnsrecords
+import cartography.intel.cloudflare.members
+import cartography.intel.cloudflare.roles
+import cartography.intel.cloudflare.zones
+from cartography.config import Config
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def start_cloudflare_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of Cloudflare data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+
+    if not config.cloudflare_token:
+        logger.info(
+            "Cloudflare import is not configured - skipping this module. "
+            "See docs to configure.",
+        )
+        return
+
+    # Create client
+    client = Cloudflare(api_token=config.cloudflare_token)
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+    }
+
+    for account in cartography.intel.cloudflare.accounts.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+    ):
+        account_job_parameters = common_job_parameters.copy()
+        account_job_parameters["account_id"] = account["id"]
+        cartography.intel.cloudflare.roles.sync(
+            neo4j_session,
+            client,
+            account_job_parameters,
+            account_id=account["id"],
+        )
+
+        cartography.intel.cloudflare.members.sync(
+            neo4j_session,
+            client,
+            account_job_parameters,
+            account_id=account["id"],
+        )
+
+        for zone in cartography.intel.cloudflare.zones.sync(
+            neo4j_session,
+            client,
+            account_job_parameters,
+            account_id=account["id"],
+        ):
+            zone_job_parameters = account_job_parameters.copy()
+            zone_job_parameters["zone_id"] = zone["id"]
+            cartography.intel.cloudflare.dnsrecords.sync(
+                neo4j_session,
+                client,
+                zone_job_parameters,
+                zone_id=zone["id"],
+            )

cartography/intel/cloudflare/accounts.py

@@ -0,0 +1,57 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+from cloudflare import Cloudflare
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.cloudflare.account import CloudflareAccountSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: Cloudflare,
+    common_job_parameters: Dict[str, Any],
+) -> List[Dict]:
+    accounts = get(client)
+    load_accounts(
+        neo4j_session,
+        accounts,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+    return accounts
+
+
+@timeit
+def get(client: Cloudflare) -> List[Dict[str, Any]]:
+    return [account.to_dict() for account in client.accounts.list()]
+
+
+def load_accounts(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Cloudflare accounts into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        CloudflareAccountSchema(),
+        data,
+        lastupdated=update_tag,
+    )
+
+
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(CloudflareAccountSchema(), common_job_parameters).run(
+        neo4j_session
+    )