cartography 0.93.0rc1__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/aws/ssm.py

@@ -1,4 +1,6 @@
+import json
 import logging
+import re
 from typing import Any
 from typing import Dict
 from typing import List
@@ -7,9 +9,11 @@ import boto3
 import neo4j
 
 from cartography.client.core.tx import load
+from cartography.client.core.tx import read_list_of_values_tx
 from cartography.graph.job import GraphJob
 from cartography.models.aws.ssm.instance_information import SSMInstanceInformationSchema
 from cartography.models.aws.ssm.instance_patch import SSMInstancePatchSchema
+from cartography.models.aws.ssm.parameters import SSMParameterSchema
 from cartography.util import aws_handle_regions
 from cartography.util import dict_date_to_epoch
 from cartography.util import timeit
@@ -18,58 +22,71 @@ logger = logging.getLogger(__name__)
 
 
 @timeit
-def get_instance_ids(neo4j_session: neo4j.Session, region: str, current_aws_account_id: str) -> List[str]:
+def get_instance_ids(
+    neo4j_session: neo4j.Session,
+    region: str,
+    current_aws_account_id: str,
+) -> List[str]:
     get_instances_query = """
     MATCH (:AWSAccount{id: $AWS_ACCOUNT_ID})-[:RESOURCE]->(i:EC2Instance)
     WHERE i.region = $Region
     RETURN i.id
     """
-    results = neo4j_session.run(get_instances_query, AWS_ACCOUNT_ID=current_aws_account_id, Region=region)
-    instance_ids = []
-    for r in results:
-        instance_ids.append(r['i.id'])
-    return instance_ids
+    return neo4j_session.execute_read(
+        read_list_of_values_tx,
+        get_instances_query,
+        AWS_ACCOUNT_ID=current_aws_account_id,
+        Region=region,
+    )
 
 
 @timeit
 @aws_handle_regions
 def get_instance_information(
-        boto3_session: boto3.session.Session,
-        region: str,
-        instance_ids: List[str],
+    boto3_session: boto3.session.Session,
+    region: str,
+    instance_ids: List[str],
 ) -> List[Dict[str, Any]]:
-    client = boto3_session.client('ssm', region_name=region)
+    client = boto3_session.client("ssm", region_name=region)
     instance_information: List[Dict[str, Any]] = []
-    paginator = client.get_paginator('describe_instance_information')
+    paginator = client.get_paginator("describe_instance_information")
     for i in range(0, len(instance_ids), 50):
-        instance_ids_chunk = instance_ids[i:i + 50]
+        instance_ids_chunk = instance_ids[i : i + 50]
         for info_chunk in paginator.paginate(
             Filters=[{"Key": "InstanceIds", "Values": instance_ids_chunk}],
             MaxResults=50,
         ):
-            instance_information.extend(info_chunk.get('InstanceInformationList', []))
+            instance_information.extend(info_chunk.get("InstanceInformationList", []))
     return instance_information
 
 
-def transform_instance_information(data_list: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+def transform_instance_information(
+    data_list: List[Dict[str, Any]],
+) -> List[Dict[str, Any]]:
     for ii in data_list:
         ii["LastPingDateTime"] = dict_date_to_epoch(ii, "LastPingDateTime")
         ii["RegistrationDate"] = dict_date_to_epoch(ii, "RegistrationDate")
-        ii["LastAssociationExecutionDate"] = dict_date_to_epoch(ii, "LastAssociationExecutionDate")
-        ii["LastSuccessfulAssociationExecutionDate"] = dict_date_to_epoch(ii, "LastSuccessfulAssociationExecutionDate")
+        ii["LastAssociationExecutionDate"] = dict_date_to_epoch(
+            ii,
+            "LastAssociationExecutionDate",
+        )
+        ii["LastSuccessfulAssociationExecutionDate"] = dict_date_to_epoch(
+            ii,
+            "LastSuccessfulAssociationExecutionDate",
+        )
     return data_list
 
 
 @timeit
 @aws_handle_regions
 def get_instance_patches(
-        boto3_session: boto3.session.Session,
-        region: str,
-        instance_ids: List[str],
+    boto3_session: boto3.session.Session,
+    region: str,
+    instance_ids: List[str],
 ) -> List[Dict[str, Any]]:
-    client = boto3_session.client('ssm', region_name=region)
+    client = boto3_session.client("ssm", region_name=region)
     instance_patches: List[Dict[str, Any]] = []
-    paginator = client.get_paginator('describe_instance_patches')
+    paginator = client.get_paginator("describe_instance_patches")
     for instance_id in instance_ids:
         patches = []
         for page in paginator.paginate(InstanceId=instance_id):
@@ -91,6 +108,42 @@ def transform_instance_patches(data_list: List[Dict[str, Any]]) -> List[Dict[str
     return data_list
 
 
+@timeit
+@aws_handle_regions
+def get_ssm_parameters(
+    boto3_session: boto3.session.Session,
+    region: str,
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client("ssm", region_name=region)
+    paginator = client.get_paginator("describe_parameters")
+    ssm_parameters_data: List[Dict[str, Any]] = []
+    for page in paginator.paginate(PaginationConfig={"PageSize": 50}):
+        ssm_parameters_data.extend(page.get("Parameters", []))
+    return ssm_parameters_data
+
+
+def transform_ssm_parameters(
+    raw_parameters_data: List[Dict[str, Any]],
+) -> List[Dict[str, Any]]:
+    transformed_list: List[Dict[str, Any]] = []
+    for param in raw_parameters_data:
+        param["LastModifiedDate"] = dict_date_to_epoch(param, "LastModifiedDate")
+        param["PoliciesJson"] = json.dumps(param.get("Policies", []))
+        # KMSKey uses shorter UUID as their primary id
+        # SSM Parameters, when encrypted, reference KMS keys using their full ARNs in the KeyId field
+        # Adding a param to match on the id property of the target node
+        if param.get("Type") == "SecureString" and param.get("KeyId") is not None:
+            match = re.match(r".*key/(.*)$", param["KeyId"])
+            if match:
+                param["KMSKeyIdShort"] = match.group(1)
+            else:
+                param["KMSKeyIdShort"] = None
+        else:
+            param["KMSKeyIdShort"] = None
+        transformed_list.append(param)
+    return transformed_list
+
+
 @timeit
 def load_instance_information(
     neo4j_session: neo4j.Session,
@@ -128,29 +181,85 @@ def load_instance_patches(
 
 
 @timeit
-def cleanup_ssm(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
+def load_ssm_parameters(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        SSMParameterSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup_ssm(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
     logger.info("Running SSM cleanup")
-    GraphJob.from_node_schema(SSMInstanceInformationSchema(), common_job_parameters).run(neo4j_session)
-    GraphJob.from_node_schema(SSMInstancePatchSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(
+        SSMInstanceInformationSchema(),
+        common_job_parameters,
+    ).run(neo4j_session)
+    GraphJob.from_node_schema(SSMInstancePatchSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
+    GraphJob.from_node_schema(SSMParameterSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
 
 
 @timeit
 def sync(
-        neo4j_session: neo4j.Session,
-        boto3_session: boto3.session.Session,
-        regions: List[str],
-        current_aws_account_id: str,
-        update_tag: int,
-        common_job_parameters: Dict[str, Any],
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
 ) -> None:
     for region in regions:
-        logger.info("Syncing SSM for region '%s' in account '%s'.", region, current_aws_account_id)
+        logger.info(
+            "Syncing SSM for region '%s' in account '%s'.",
+            region,
+            current_aws_account_id,
+        )
         instance_ids = get_instance_ids(neo4j_session, region, current_aws_account_id)
         data = get_instance_information(boto3_session, region, instance_ids)
         data = transform_instance_information(data)
-        load_instance_information(neo4j_session, data, region, current_aws_account_id, update_tag)
+        load_instance_information(
+            neo4j_session,
+            data,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
 
         data = get_instance_patches(boto3_session, region, instance_ids)
         data = transform_instance_patches(data)
-        load_instance_patches(neo4j_session, data, region, current_aws_account_id, update_tag)
+        load_instance_patches(
+            neo4j_session,
+            data,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+        data = get_ssm_parameters(boto3_session, region)
+        data = transform_ssm_parameters(data)
+        load_ssm_parameters(
+            neo4j_session,
+            data,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
     cleanup_ssm(neo4j_session, common_job_parameters)

cartography/intel/aws/util/arns.py

@@ -2,16 +2,16 @@ from typing import Optional
 
 
 def build_arn(
-        resource: str,
-        account: str,
-        typename: str,
-        name: str,
-        region: Optional[str] = None,
-        partition: Optional[str] = None,
+    resource: str,
+    account: str,
+    typename: str,
+    name: str,
+    region: Optional[str] = None,
+    partition: Optional[str] = None,
 ) -> str:
     if not partition:
         # TODO: support partitions from others. Please file an issue on this if needed, would love to hear from you
-        partition = 'aws'
+        partition = "aws"
     if not region:
         # Some resources are present in all regions, e.g. IAM policies
         region = ""

cartography/intel/aws/util/common.py

@@ -1,21 +1,48 @@
+import logging
 from typing import List
 
 from cartography.intel.aws.resources import RESOURCE_FUNCTIONS
 
+logger = logging.getLogger(__name__)
+
 
 def parse_and_validate_aws_requested_syncs(aws_requested_syncs: str) -> List[str]:
     validated_resources: List[str] = []
-    for resource in aws_requested_syncs.split(','):
+    for resource in aws_requested_syncs.split(","):
         resource = resource.strip()
 
         if resource in RESOURCE_FUNCTIONS:
             validated_resources.append(resource)
         else:
-            valid_syncs: str = ', '.join(RESOURCE_FUNCTIONS.keys())
+            valid_syncs: str = ", ".join(RESOURCE_FUNCTIONS.keys())
             raise ValueError(
                 f'Error parsing `aws-requested-syncs`. You specified "{aws_requested_syncs}". '
-                f'Please check that your string is formatted properly. '
+                f"Please check that your string is formatted properly. "
                 f'Example valid input looks like "s3,iam,rds" or "s3, ec2:instance, dynamodb". '
-                f'Our full list of valid values is: {valid_syncs}.',
+                f"Our full list of valid values is: {valid_syncs}.",
             )
     return validated_resources
+
+
+def parse_and_validate_aws_regions(aws_regions: str) -> list[str]:
+    """
+    Parse and validate a comma-separated string of AWS regions.
+    :param aws_regions: Comma-separated string of AWS regions
+    :return: A validated list of AWS regions
+    """
+    validated_regions: List[str] = []
+    for region in aws_regions.split(","):
+        region = region.strip()
+        if region:
+            validated_regions.append(region)
+        else:
+            logger.warning(
+                f'Unable to parse string "{region}". Please check the value you passed to `aws-regions`. '
+                f'You specified "{aws_regions}". Continuing on with sync.',
+            )
+
+    if not validated_regions:
+        raise ValueError(
+            f'`aws-regions` was set but no regions were specified. You provided this string: "{aws_regions}"',
+        )
+    return validated_regions

cartography/intel/azure/__init__.py

@@ -1,55 +1,250 @@
 import logging
 from typing import Dict
 from typing import List
-from typing import Optional
 
 import neo4j
 
+from cartography.config import Config
+from cartography.util import timeit
+
+from . import aks
+from . import app_service
 from . import compute
+from . import container_instances
 from . import cosmosdb
+from . import data_factory
+from . import data_factory_dataset
+from . import data_factory_linked_service
+from . import data_factory_pipeline
+from . import data_lake
+from . import event_grid
+from . import functions
+from . import load_balancers
+from . import logic_apps
+from . import monitor
+from . import network
+from . import permission_relationships
+from . import rbac
+from . import resource_groups
+from . import security_center
 from . import sql
 from . import storage
 from . import subscription
 from . import tenant
 from .util.credentials import Authenticator
 from .util.credentials import Credentials
-from cartography.config import Config
-from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
 
 def _sync_one_subscription(
-    neo4j_session: neo4j.Session, credentials: Credentials, subscription_id: str, update_tag: int,
+    neo4j_session: neo4j.Session,
+    credentials: Credentials,
+    subscription_id: str,
+    update_tag: int,
     common_job_parameters: Dict,
 ) -> None:
-    compute.sync(neo4j_session, credentials.arm_credentials, subscription_id, update_tag, common_job_parameters)
-    cosmosdb.sync(neo4j_session, credentials.arm_credentials, subscription_id, update_tag, common_job_parameters)
-    sql.sync(neo4j_session, credentials.arm_credentials, subscription_id, update_tag, common_job_parameters)
-    storage.sync(neo4j_session, credentials.arm_credentials, subscription_id, update_tag, common_job_parameters)
+    container_instances.sync(
+        neo4j_session,
+        credentials,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    compute.sync(
+        neo4j_session,
+        credentials.credential,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    cosmosdb.sync(
+        neo4j_session,
+        credentials.credential,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    app_service.sync(
+        neo4j_session,
+        credentials,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    functions.sync(
+        neo4j_session,
+        credentials,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    event_grid.sync(
+        neo4j_session,
+        credentials,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    logic_apps.sync(
+        neo4j_session,
+        credentials,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    rbac.sync(
+        neo4j_session,
+        credentials,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    sql.sync(
+        neo4j_session,
+        credentials.credential,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    storage.sync(
+        neo4j_session,
+        credentials.credential,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    resource_groups.sync(
+        neo4j_session,
+        credentials,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    aks.sync(
+        neo4j_session,
+        credentials,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    factories_raw = data_factory.sync_data_factories(
+        neo4j_session,
+        credentials,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    linked_services_by_factory = (
+        data_factory_linked_service.sync_data_factory_linked_services(
+            neo4j_session,
+            credentials,
+            factories_raw,
+            subscription_id,
+            update_tag,
+            common_job_parameters,
+        )
+    )
+    datasets_by_factory = data_factory_dataset.sync_data_factory_datasets(
+        neo4j_session,
+        credentials,
+        factories_raw,
+        linked_services_by_factory,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    data_factory_pipeline.sync_data_factory_pipelines(
+        neo4j_session,
+        credentials,
+        factories_raw,
+        datasets_by_factory,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    data_lake.sync(
+        neo4j_session,
+        credentials,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    network.sync(
+        neo4j_session,
+        credentials,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    load_balancers.sync(
+        neo4j_session,
+        credentials,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    monitor.sync(
+        neo4j_session,
+        credentials,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    security_center.sync(
+        neo4j_session,
+        credentials,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    permission_relationships.sync(
+        neo4j_session,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
 
 
 def _sync_tenant(
-    neo4j_session: neo4j.Session, tenant_id: str, current_user: Optional[str], update_tag: int,
+    neo4j_session: neo4j.Session,
+    credentials: Credentials,
+    update_tag: int,
     common_job_parameters: Dict,
 ) -> None:
-    logger.info("Syncing Azure Tenant: %s", tenant_id)
-    tenant.sync(neo4j_session, tenant_id, current_user, update_tag, common_job_parameters)  # type: ignore
+    logger.info("Syncing Azure Tenant: %s", credentials.tenant_id)
+    tenant.sync(neo4j_session, credentials.tenant_id, None, update_tag, common_job_parameters)  # type: ignore
 
 
 def _sync_multiple_subscriptions(
-    neo4j_session: neo4j.Session, credentials: Credentials, tenant_id: str, subscriptions: List[Dict],
-    update_tag: int, common_job_parameters: Dict,
+    neo4j_session: neo4j.Session,
+    credentials: Credentials,
+    tenant_id: str,
+    subscriptions: List[Dict],
+    update_tag: int,
+    common_job_parameters: Dict,
 ) -> None:
     logger.info("Syncing Azure subscriptions")
 
-    subscription.sync(neo4j_session, tenant_id, subscriptions, update_tag, common_job_parameters)
+    subscription.sync(
+        neo4j_session,
+        tenant_id,
+        subscriptions,
+        update_tag,
+        common_job_parameters,
+    )
 
     for sub in subscriptions:
-        logger.info("Syncing Azure Subscription with ID '%s'", sub['subscriptionId'])
-        common_job_parameters['AZURE_SUBSCRIPTION_ID'] = sub['subscriptionId']
+        logger.info("Syncing Azure Subscription with ID '%s'", sub["subscriptionId"])
+        common_job_parameters["AZURE_SUBSCRIPTION_ID"] = sub["subscriptionId"]
 
-        _sync_one_subscription(neo4j_session, credentials, sub['subscriptionId'], update_tag, common_job_parameters)
+        _sync_one_subscription(
+            neo4j_session,
+            credentials,
+            sub["subscriptionId"],
+            update_tag,
+            common_job_parameters,
+        )
 
     del common_job_parameters["AZURE_SUBSCRIPTION_ID"]
 
@@ -59,44 +254,62 @@ def start_azure_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
     common_job_parameters = {
         "UPDATE_TAG": config.update_tag,
         "permission_relationships_file": config.permission_relationships_file,
+        "azure_permission_relationships_file": config.azure_permission_relationships_file,
     }
 
-    try:
-        if config.azure_sp_auth:
-            credentials = Authenticator().authenticate_sp(
-                config.azure_tenant_id, config.azure_client_id, config.azure_client_secret,
+    if config.azure_sp_auth:
+        if not (
+            config.azure_tenant_id
+            and config.azure_client_id
+            and config.azure_client_secret
+        ):
+            raise ValueError(
+                "Azure Service Principal authentication requested, but tenant ID, client ID, "
+                "and client secret were not all provided.",
             )
-        else:
-            credentials = Authenticator().authenticate_cli()
-
-    except Exception as e:
-        logger.error(
-            (
-                "Unable to authenticate with Azure Service Principal, an error occurred: %s."
-                "Make sure your Azure Service Principal details are provided correctly."
-            ),
-            e,
+
+        credentials = Authenticator().authenticate_sp(
+            config.azure_tenant_id,
+            config.azure_client_id,
+            config.azure_client_secret,
+        )
+    else:
+        credentials = Authenticator().authenticate_cli()
+
+    if not credentials:
+        raise RuntimeError(
+            "Azure authentication failed. Ensure Azure CLI login or Service Principal credentials are configured.",
         )
-        return
+
+    common_job_parameters["TENANT_ID"] = credentials.tenant_id
 
     _sync_tenant(
-        neo4j_session, credentials.get_tenant_id(), credentials.get_current_user(), config.update_tag,
+        neo4j_session,
+        credentials,
+        config.update_tag,
         common_job_parameters,
     )
+    if credentials.tenant_id:
+        if config.azure_sync_all_subscriptions:
+            subscriptions = subscription.get_all_azure_subscriptions(credentials)
 
-    if config.azure_sync_all_subscriptions:
-        subscriptions = subscription.get_all_azure_subscriptions(credentials)
+        else:
+            sub_id_to_sync = config.azure_subscription_id or credentials.subscription_id
+            subscriptions = subscription.get_current_azure_subscription(
+                credentials,
+                sub_id_to_sync,
+            )
 
-    else:
-        subscriptions = subscription.get_current_azure_subscription(credentials, credentials.subscription_id)
+        if not subscriptions:
+            raise RuntimeError(
+                "No Azure subscriptions found. Ensure the credentials have access to at least one subscription.",
+            )
 
-    if not subscriptions:
-        logger.warning(
-            "No valid Azure credentials are found. No Azure subscriptions can be synced. Exiting Azure sync stage.",
+        _sync_multiple_subscriptions(
+            neo4j_session,
+            credentials,
+            credentials.tenant_id,
+            subscriptions,
+            config.update_tag,
+            common_job_parameters,
         )
-        return
-
-    _sync_multiple_subscriptions(
-        neo4j_session, credentials, credentials.get_tenant_id(), subscriptions, config.update_tag,
-        common_job_parameters,
-    )