cartography 0.93.0rc1__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/spacelift/__init__.py

@@ -0,0 +1,161 @@
+import logging
+
+import boto3
+import neo4j
+import requests
+
+from cartography.config import Config
+from cartography.intel.spacelift.account import sync_account
+from cartography.intel.spacelift.ec2_ownership import sync_ec2_ownership
+from cartography.intel.spacelift.runs import sync_runs
+from cartography.intel.spacelift.spaces import sync_spaces
+from cartography.intel.spacelift.stacks import sync_stacks
+from cartography.intel.spacelift.util import get_spacelift_token
+from cartography.intel.spacelift.workerpools import sync_worker_pools
+from cartography.intel.spacelift.workers import sync_workers
+from cartography.stats import get_stats_client
+from cartography.util import merge_module_sync_metadata
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+stat_handler = get_stats_client(__name__)
+
+
+@timeit
+def start_spacelift_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    Perform ingestion of Spacelift data.
+    :param neo4j_session: Neo4j session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+    # Validate configuration
+    if not config.spacelift_api_endpoint:
+        logger.info("Spacelift API endpoint not configured - skipping this module.")
+        return
+
+    # Determine authentication method and obtain token
+    token = None
+    if config.spacelift_api_token:
+        # Method 1: Use provided token directly
+        logger.info("Using provided Spacelift API token for authentication")
+        token = config.spacelift_api_token
+    elif config.spacelift_api_key_id and config.spacelift_api_key_secret:
+        # Method 2: Exchange API key ID and secret for a token
+        logger.info("Exchanging Spacelift API key for authentication token")
+        try:
+            token = get_spacelift_token(
+                config.spacelift_api_endpoint,
+                config.spacelift_api_key_id,
+                config.spacelift_api_key_secret,
+            )
+        except Exception as e:
+            logger.error(f"Failed to obtain Spacelift authentication token: {e}")
+            logger.info("Skipping Spacelift module due to authentication failure.")
+            return
+    else:
+        logger.info(
+            "Spacelift authentication not configured. "
+            "Provide either --spacelift-api-token-env-var or both "
+            "--spacelift-api-key-id-env-var and --spacelift-api-key-secret-env-var. "
+            "Skipping this module."
+        )
+        return
+
+    logger.info("Starting Spacelift ingestion")
+
+    # Set up common job parameters
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+    }
+
+    # Create authenticated session for Spacelift API
+    spacelift_session = requests.Session()
+    spacelift_session.headers.update(
+        {
+            "Authorization": f"Bearer {token}",
+        }
+    )
+
+    account_id = sync_account(
+        neo4j_session,
+        config.spacelift_api_endpoint,
+        common_job_parameters,
+    )
+
+    common_job_parameters["spacelift_account_id"] = account_id
+
+    sync_spaces(
+        neo4j_session,
+        spacelift_session,
+        config.spacelift_api_endpoint,
+        account_id,
+        common_job_parameters,
+    )
+
+    sync_stacks(
+        neo4j_session,
+        spacelift_session,
+        config.spacelift_api_endpoint,
+        account_id,
+        common_job_parameters,
+    )
+
+    sync_worker_pools(
+        neo4j_session,
+        spacelift_session,
+        config.spacelift_api_endpoint,
+        account_id,
+        common_job_parameters,
+    )
+
+    sync_workers(
+        neo4j_session,
+        spacelift_session,
+        config.spacelift_api_endpoint,
+        account_id,
+        common_job_parameters,
+    )
+
+    # Note: Users are synced as part of runs sync (derived from run.triggeredBy)
+    sync_runs(
+        neo4j_session,
+        spacelift_session,
+        config.spacelift_api_endpoint,
+        account_id,
+        common_job_parameters,
+    )
+
+    # Sync EC2 ownership relationships from CloudTrail data (optional)
+    if all(
+        hasattr(config, attr)
+        for attr in [
+            "spacelift_ec2_ownership_s3_bucket",
+            "spacelift_ec2_ownership_s3_prefix",
+        ]
+    ):
+        if hasattr(config, "spacelift_ec2_ownership_aws_profile"):
+            aws_session = boto3.Session(
+                profile_name=config.spacelift_ec2_ownership_aws_profile
+            )
+        else:
+            aws_session = boto3.Session()
+        sync_ec2_ownership(
+            neo4j_session,
+            aws_session,
+            config.spacelift_ec2_ownership_s3_bucket,
+            config.spacelift_ec2_ownership_s3_prefix,
+            config.update_tag,
+            account_id,
+        )
+
+    merge_module_sync_metadata(
+        neo4j_session,
+        group_type="Spacelift",
+        group_id=account_id,
+        synced_type="SpaceliftData",
+        update_tag=config.update_tag,
+        stat_handler=stat_handler,
+    )
+
+    logger.info("Completed Spacelift ingestion")

cartography/intel/spacelift/account.py

@@ -0,0 +1,73 @@
+import logging
+from typing import Any
+from urllib.parse import urlparse
+
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.spacelift.spaceliftaccount import SpaceliftAccountSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_account(api_endpoint: str) -> str:
+    # Parse URL to extract subdomain (account ID)
+    parsed = urlparse(api_endpoint)
+    hostname = parsed.hostname or ""
+    # Extract subdomain (everything before .app.spacelift.io)
+    account_id = hostname.split(".")[0] if hostname else ""
+
+    return account_id
+
+
+def load_account(
+    neo4j_session: neo4j.Session,
+    account_data: dict,
+    update_tag: int,
+) -> None:
+
+    load(
+        neo4j_session,
+        SpaceliftAccountSchema(),
+        [account_data],
+        lastupdated=update_tag,
+    )
+
+    logger.info(f"Loaded Spacelift account: {account_data['id']}")
+
+
+@timeit
+def cleanup_account(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+) -> None:
+
+    logger.debug("Running SpaceliftAccount cleanup job")
+    GraphJob.from_node_schema(SpaceliftAccountSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync_account(
+    neo4j_session: neo4j.Session,
+    api_endpoint: str,
+    common_job_parameters: dict[str, Any],
+) -> str:
+
+    account_id = get_account(api_endpoint)
+
+    # Transform account data (all fields are just spacelift_account_id)
+    account_data = {
+        "id": account_id,
+        "spacelift_account_id": account_id,
+        "name": account_id,
+    }
+
+    load_account(neo4j_session, account_data, common_job_parameters["UPDATE_TAG"])
+    cleanup_account(neo4j_session, common_job_parameters)
+    logger.info(f"Synced Spacelift account: {account_id}")
+    return account_id

cartography/intel/spacelift/ec2_ownership.py

@@ -0,0 +1,280 @@
+import json
+import logging
+import re
+from typing import Any
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.spacelift.cloudtrailevent import CloudTrailSpaceliftEventSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+# Regex pattern to match EC2 instance IDs
+INSTANCE_ID_PATTERN = re.compile(r"\b(i-[0-9a-f]{8,17})\b")
+
+
+@aws_handle_regions
+@timeit
+def get_ec2_ownership(
+    aws_session: boto3.Session, bucket_name: str, object_prefix: str
+) -> list[dict[str, Any]]:
+    """
+    Fetch EC2 ownership data from all JSON files under an S3 prefix containing Athena query results.
+
+    Args:
+        aws_session: AWS session for making S3 requests
+        bucket_name: S3 bucket name
+        object_prefix: S3 prefix to search for JSON files. Trailing slash is optional and will be normalized.
+
+    Returns:
+        Aggregated list of CloudTrail records from all JSON files under the prefix
+    """
+    # Normalize prefix - ensure it ends with '/' to treat it as a directory
+    # This prevents matching prefixes like "data" matching "data-backup" objects
+    normalized_prefix = object_prefix.rstrip("/") + "/" if object_prefix else ""
+
+    logger.info(
+        f"Fetching EC2 ownership data from s3://{bucket_name}/{normalized_prefix}"
+    )
+
+    # Create S3 client from the boto3 session
+    s3_client = aws_session.client("s3")
+
+    paginator = s3_client.get_paginator("list_objects_v2")
+    page_iterator = paginator.paginate(Bucket=bucket_name, Prefix=normalized_prefix)
+
+    all_records = []
+    total_files_processed = 0
+
+    for page in page_iterator:
+        if "Contents" not in page:
+            logger.warning(
+                f"No objects found under prefix s3://{bucket_name}/{normalized_prefix}"
+            )
+            continue
+
+        # Filter for JSON files and exclude the prefix itself if it appears as an object
+        json_files = [
+            obj
+            for obj in page["Contents"]
+            if obj["Key"].endswith(".json") and obj["Key"] != normalized_prefix
+        ]
+
+        if not json_files:
+            continue
+
+        logger.info(f"Found {len(json_files)} JSON files to process in this page")
+
+        for s3_object in json_files:
+            object_key = s3_object["Key"]
+            logger.info(f"Processing {object_key}")
+
+            try:
+                response = s3_client.get_object(Bucket=bucket_name, Key=object_key)
+                object_body = response["Body"].read()
+                json_content = json.loads(object_body.decode("utf-8"))
+
+                # Handle both single objects and arrays
+                if isinstance(json_content, list):
+                    all_records.extend(json_content)
+                    records_added = len(json_content)
+                else:
+                    all_records.append(json_content)
+                    records_added = 1
+
+                total_files_processed += 1
+                logger.info(
+                    f"Successfully processed {object_key}, added {records_added} records"
+                )
+
+            except Exception as e:
+                logger.error(f"Failed to process {object_key}: {e}")
+                continue
+
+    logger.info(
+        f"Successfully processed {total_files_processed} files and fetched {len(all_records)} total CloudTrail records from S3"
+    )
+
+    return all_records
+
+
+def extract_spacelift_run_id(useridentity_str: str) -> str | None:
+    """
+    Extract Spacelift run ID from the useridentity field.
+    """
+    # Only process if 'spacelift' is in the useridentity
+    if "spacelift" not in useridentity_str.lower():
+        return None
+
+    # Extract the ARN using regex
+    # Format: arn=arn:aws:sts::ACCOUNT:assumed-role/ROLE_NAME/SESSION_NAME
+    arn_match = re.search(
+        r"arn=arn:aws:sts::[^:]+:assumed-role/[^/]+/([^,}]+)", useridentity_str
+    )
+
+    if not arn_match:
+        return None
+
+    session_name = arn_match.group(1).strip()
+
+    # Run ID is the first part before the @ symbol
+    run_id = session_name.split("@")[0]
+
+    return run_id if run_id else None
+
+
+def extract_instance_ids(record: dict[str, Any]) -> list[str]:
+    """
+    Extract EC2 instance IDs from a CloudTrail record.
+    """
+    instance_ids = set()
+
+    # Check resources field
+    resources = record.get("resources")
+    if resources:
+        resources_str = str(resources)
+        instance_ids.update(INSTANCE_ID_PATTERN.findall(resources_str))
+
+    # Check requestparameters field
+    request_params = record.get("requestparameters")
+    if request_params:
+        instance_ids.update(INSTANCE_ID_PATTERN.findall(str(request_params)))
+
+    # Check responseelements field
+    response_elements = record.get("responseelements")
+    if response_elements:
+        instance_ids.update(INSTANCE_ID_PATTERN.findall(str(response_elements)))
+
+    return list(instance_ids)
+
+
+@timeit
+def transform_ec2_ownership(
+    cloudtrail_data: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+    """
+    Transform CloudTrail data to create CloudTrailSpaceliftEvent nodes.
+
+    This function filters CloudTrail records to find those that have BOTH:
+    1. A Spacelift run ID (from the useridentity field)
+    2. One or more EC2 instance IDs (from resources, requestparameters, or responseelements)
+
+    Each CloudTrail record becomes one CloudTrailSpaceliftEvent node that can connect to multiple instances.
+    """
+    logger.info(
+        f"Transforming {len(cloudtrail_data)} CloudTrail records to create CloudTrailSpaceliftEvent nodes"
+    )
+
+    events = []
+
+    for record in cloudtrail_data:
+        # Extract run ID from useridentity
+        useridentity = record.get("useridentity", "")
+        run_id = extract_spacelift_run_id(useridentity)
+
+        if not run_id:
+            # Skip records without Spacelift run IDs
+            continue
+
+        # Extract instance IDs
+        instance_ids = extract_instance_ids(record)
+
+        if not instance_ids:
+            # Skip records without instance IDs
+            continue
+
+        event_id = record["eventid"]
+
+        event = {
+            "id": event_id,
+            "run_id": run_id,
+            "instance_ids": instance_ids,
+            "event_time": record.get("eventtime"),
+            "event_name": record.get("eventname"),
+            "aws_account": record.get("account"),
+            "aws_region": record.get("awsregion"),
+        }
+        events.append(event)
+
+    logger.info(
+        f"Created {len(events)} CloudTrailSpaceliftEvent records affecting EC2 instances"
+    )
+
+    return events
+
+
+@timeit
+def load_cloudtrail_events(
+    neo4j_session: neo4j.Session,
+    events: list[dict[str, Any]],
+    update_tag: int,
+    account_id: str,
+) -> None:
+    """
+    Load CloudTrailSpaceliftEvent nodes with relationships to SpaceliftRun and EC2Instance nodes.
+    """
+    logger.info(
+        f"Loading {len(events)} CloudTrailSpaceliftEvent nodes with relationships into Neo4j"
+    )
+
+    load(
+        neo4j_session,
+        CloudTrailSpaceliftEventSchema(),
+        events,
+        lastupdated=update_tag,
+        spacelift_account_id=account_id,
+    )
+
+    logger.info(f"Successfully loaded {len(events)} CloudTrailSpaceliftEvent nodes")
+
+
+@timeit
+def cleanup_cloudtrail_events(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    """
+    Remove stale CloudTrailSpaceliftEvent nodes and their relationships from Neo4j.
+    """
+    logger.debug("Running CloudTrailSpaceliftEvent cleanup job")
+
+    GraphJob.from_node_schema(
+        CloudTrailSpaceliftEventSchema(), common_job_parameters
+    ).run(neo4j_session)
+
+
+@timeit
+def sync_ec2_ownership(
+    neo4j_session: neo4j.Session,
+    aws_session: boto3.Session,
+    bucket_name: str,
+    object_prefix: str,
+    update_tag: int,
+    account_id: str,
+) -> None:
+    """
+    Sync EC2 ownership data from CloudTrail into Neo4j as CloudTrailSpaceliftEvent nodes.
+    """
+    logger.info("Starting EC2 ownership sync")
+
+    cloudtrail_data = get_ec2_ownership(aws_session, bucket_name, object_prefix)
+
+    events = transform_ec2_ownership(cloudtrail_data)
+
+    if events:
+        load_cloudtrail_events(neo4j_session, events, update_tag, account_id)
+    else:
+        logger.warning("No CloudTrail events found - no nodes created")
+
+    common_job_parameters = {
+        "UPDATE_TAG": update_tag,
+        "spacelift_account_id": account_id,
+    }
+    cleanup_cloudtrail_events(neo4j_session, common_job_parameters)
+
+    logger.info("EC2 ownership sync completed successfully")