cartography 0.107.0rc2__tar.gz → 0.107.0rc3__tar.gz

{cartography-0.107.0rc2 → cartography-0.107.0rc3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cartography
-Version: 0.107.0rc2
+Version: 0.107.0rc3
 Summary: Explore assets and their relationships across your technical infrastructure.
 Maintainer: Cartography Contributors
 License: apache2

{cartography-0.107.0rc2 → cartography-0.107.0rc3}/cartography/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.107.0rc2'
-__version_tuple__ = version_tuple = (0, 107, 0, 'rc2')
+__version__ = version = '0.107.0rc3'
+__version_tuple__ = version_tuple = (0, 107, 0, 'rc3')

cartography-0.107.0rc3/cartography/intel/aws/cloudwatch.py

@@ -0,0 +1,170 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.cloudwatch.log_metric_filter import (
+    CloudWatchLogMetricFilterSchema,
+)
+from cartography.models.aws.cloudwatch.loggroup import CloudWatchLogGroupSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_cloudwatch_log_groups(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "logs", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("describe_log_groups")
+    logGroups = []
+    for page in paginator.paginate():
+        logGroups.extend(page["logGroups"])
+    return logGroups
+
+
+@timeit
+@aws_handle_regions
+def get_cloudwatch_log_metric_filters(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    logs_client = boto3_session.client(
+        "logs", region_name=region, config=get_botocore_config()
+    )
+    paginator = logs_client.get_paginator("describe_metric_filters")
+    metric_filters = []
+
+    for page in paginator.paginate():
+        metric_filters.extend(page.get("metricFilters", []))
+
+    return metric_filters
+
+
+def transform_metric_filters(
+    metric_filters: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    """
+    Transform CloudWatch log metric filter data for ingestion into Neo4j.
+    Ensures that the 'id' field is a unique combination of logGroupName and filterName.
+    """
+    transformed_filters = []
+    for filter in metric_filters:
+        transformed_filter = {
+            "id": f"{filter['logGroupName']}:{filter['filterName']}",
+            "arn": f"{filter['logGroupName']}:{filter['filterName']}",
+            "filterName": filter["filterName"],
+            "filterPattern": filter.get("filterPattern"),
+            "logGroupName": filter["logGroupName"],
+            "metricName": filter["metricTransformations"][0]["metricName"],
+            "metricNamespace": filter["metricTransformations"][0]["metricNamespace"],
+            "metricValue": filter["metricTransformations"][0]["metricValue"],
+            "Region": region,
+        }
+        transformed_filters.append(transformed_filter)
+    return transformed_filters
+
+
+@timeit
+def load_cloudwatch_log_groups(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading CloudWatch {len(data)} log groups for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        CloudWatchLogGroupSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def load_cloudwatch_log_metric_filters(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading CloudWatch {len(data)} log metric filters for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        CloudWatchLogMetricFilterSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.debug("Running CloudWatch cleanup job.")
+    cleanup_job = GraphJob.from_node_schema(
+        CloudWatchLogGroupSchema(), common_job_parameters
+    )
+    cleanup_job.run(neo4j_session)
+    GraphJob.from_node_schema(
+        CloudWatchLogMetricFilterSchema(), common_job_parameters
+    ).run(neo4j_session)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing CloudWatch for region '{region}' in account '{current_aws_account_id}'.",
+        )
+        logGroups = get_cloudwatch_log_groups(boto3_session, region)
+        group_data: List[Dict[str, Any]] = []
+        for logGroup in logGroups:
+            group_data.append(logGroup)
+
+        load_cloudwatch_log_groups(
+            neo4j_session,
+            group_data,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+        metric_filters = get_cloudwatch_log_metric_filters(boto3_session, region)
+        transformed_filters = transform_metric_filters(metric_filters, region)
+        load_cloudwatch_log_metric_filters(
+            neo4j_session,
+            transformed_filters,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+    cleanup(neo4j_session, common_job_parameters)

{cartography-0.107.0rc2 → cartography-0.107.0rc3}/cartography/intel/aws/ec2/subnets.py

@@ -53,7 +53,7 @@ def load_subnets(
     snet.state = subnet.State, snet.assignipv6addressoncreation = subnet.AssignIpv6AddressOnCreation,
     snet.map_public_ip_on_launch = subnet.MapPublicIpOnLaunch, snet.subnet_arn = subnet.SubnetArn,
     snet.availability_zone = subnet.AvailabilityZone, snet.availability_zone_id = subnet.AvailabilityZoneId,
-    snet.subnetid = subnet.SubnetId
+    snet.subnet_id = subnet.SubnetId
     """
 
     ingest_subnet_vpc_relations = """

{cartography-0.107.0rc2 → cartography-0.107.0rc3}/cartography/intel/aws/ecs.py

@@ -169,6 +169,22 @@ def _get_containers_from_tasks(tasks: list[dict[str, Any]]) -> list[dict[str, An
     return containers
 
 
+def transform_ecs_tasks(tasks: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """
+    Extract network interface ID from task attachments.
+    """
+    for task in tasks:
+        for attachment in task.get("attachments", []):
+            if attachment.get("type") == "ElasticNetworkInterface":
+                details = attachment.get("details", [])
+                for detail in details:
+                    if detail.get("name") == "networkInterfaceId":
+                        task["networkInterfaceId"] = detail.get("value")
+                        break
+                break
+    return tasks
+
+
 @timeit
 def load_ecs_clusters(
     neo4j_session: neo4j.Session,
@@ -407,6 +423,7 @@ def _sync_ecs_task_and_container_defns(
         boto3_session,
         region,
     )
+    tasks = transform_ecs_tasks(tasks)
     containers = _get_containers_from_tasks(tasks)
     load_ecs_tasks(
         neo4j_session,

{cartography-0.107.0rc2 → cartography-0.107.0rc3}/cartography/intel/sentinelone/__init__.py

@@ -3,6 +3,7 @@ import logging
 import neo4j
 
 import cartography.intel.sentinelone.agent
+import cartography.intel.sentinelone.application
 from cartography.config import Config
 from cartography.intel.sentinelone.account import sync_accounts
 from cartography.stats import get_stats_client
@@ -39,7 +40,7 @@ def start_sentinelone_ingestion(neo4j_session: neo4j.Session, config: Config) ->
         config.sentinelone_account_ids,
     )
 
-    # Sync agents for each account
+    # Sync agents and applications for each account
     for account_id in synced_account_ids:
         # Add account-specific parameter
         common_job_parameters["S1_ACCOUNT_ID"] = account_id
@@ -49,7 +50,12 @@ def start_sentinelone_ingestion(neo4j_session: neo4j.Session, config: Config) ->
             common_job_parameters,
         )
 
-        # Clean up account-specific parameter
+        cartography.intel.sentinelone.application.sync(
+            neo4j_session,
+            common_job_parameters,
+        )
+
+        # Clean up account-specific parameters
         del common_job_parameters["S1_ACCOUNT_ID"]
 
     # Record that the sync is complete

cartography-0.107.0rc3/cartography/intel/sentinelone/application.py

@@ -0,0 +1,248 @@
+import logging
+from typing import Any
+
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.sentinelone.api import get_paginated_results
+from cartography.intel.sentinelone.utils import get_application_id
+from cartography.intel.sentinelone.utils import get_application_version_id
+from cartography.models.sentinelone.application import S1ApplicationSchema
+from cartography.models.sentinelone.application_version import (
+    S1ApplicationVersionSchema,
+)
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_application_data(
+    account_id: str, api_url: str, api_token: str
+) -> list[dict[str, Any]]:
+    """
+    Get application data from SentinelOne API
+    :param account_id: SentinelOne account ID
+    :param api_url: SentinelOne API URL
+    :param api_token: SentinelOne API token
+    :return: A list of application data dictionaries
+    """
+    logger.info(f"Retrieving SentinelOne application data for account {account_id}")
+    applications = get_paginated_results(
+        api_url=api_url,
+        endpoint="/web/api/v2.1/application-management/inventory",
+        api_token=api_token,
+        params={
+            "accountIds": account_id,
+            "limit": 1000,
+        },
+    )
+
+    logger.info(f"Retrieved {len(applications)} applications from SentinelOne")
+    return applications
+
+
+@timeit
+def get_application_installs(
+    app_inventory: list[dict[str, Any]], account_id: str, api_url: str, api_token: str
+) -> list[dict[str, Any]]:
+    """
+    Get application installs from SentinelOne API
+    :param app_inventory: List of applications to get installs for
+    :param account_id: SentinelOne account ID
+    :param api_url: SentinelOne API URL
+    :param api_token: SentinelOne API token
+    :return: A list of application installs data dictionaries
+    """
+    logger.info(
+        f"Retrieving SentinelOne application installs for "
+        f"{len(app_inventory)} applications in account "
+        f"{account_id}",
+    )
+
+    application_installs = []
+    for i, app in enumerate(app_inventory):
+        logger.info(
+            f"Retrieving SentinelOne installs for {app.get('applicationName')} "
+            f"{app.get('applicationVendor')} ({i + 1}/{len(app_inventory)})",
+        )
+        name = app["applicationName"]
+        vendor = app["applicationVendor"]
+        app_installs = get_paginated_results(
+            api_url=api_url,
+            endpoint="/web/api/v2.1/application-management/inventory/endpoints",
+            api_token=api_token,
+            params={
+                "accountIds": account_id,
+                "limit": 1000,
+                "applicationName": name,
+                "applicationVendor": vendor,
+            },
+        )
+
+        # Replace applicationVendor and applicationName with original values
+        # for consistency with the application data
+        for app_install in app_installs:
+            app_install["applicationVendor"] = vendor
+            app_install["applicationName"] = name
+        application_installs.extend(app_installs)
+
+    return application_installs
+
+
+def transform_applications(
+    applications_list: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+    """
+    Transform SentinelOne application data for loading into Neo4j
+    :param applications_list: Raw application data from the API
+    :return: Transformed application data
+    """
+    transformed_data = []
+    for app in applications_list:
+        vendor = app["applicationVendor"].strip()
+        name = app["applicationName"].strip()
+        app_id = get_application_id(name, vendor)
+        transformed_app = {
+            "id": app_id,
+            "name": name,
+            "vendor": vendor,
+        }
+        transformed_data.append(transformed_app)
+
+    return transformed_data
+
+
+def transform_application_versions(
+    application_installs_list: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+    """
+    Transform SentinelOne application installs for loading into Neo4j
+    :param application_installs_list: Raw application installs data from the API
+    :return: Transformed application installs data
+    """
+    transformed_data = []
+    for installed_version in application_installs_list:
+        app_name = installed_version["applicationName"]
+        app_vendor = installed_version["applicationVendor"]
+        version = installed_version["version"]
+
+        transformed_version = {
+            "id": get_application_version_id(
+                app_name,
+                app_vendor,
+                version,
+            ),
+            "application_id": get_application_id(
+                app_name,
+                app_vendor,
+            ),
+            "application_name": app_name,
+            "application_vendor": app_vendor,
+            "agent_uuid": installed_version.get("endpointUuid"),
+            "installation_path": installed_version.get("applicationInstallationPath"),
+            "installed_dt": installed_version.get("applicationInstallationDate"),
+            "version": version,
+        }
+        transformed_data.append(transformed_version)
+
+    return transformed_data
+
+
+@timeit
+def load_application_data(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    account_id: str,
+    update_tag: int,
+) -> None:
+    """
+    Load SentinelOne application data into Neo4j
+    :param neo4j_session: Neo4j session
+    :param data: The transformed application data
+    :param account_id: The SentinelOne account ID
+    :param update_tag: Update tag to set on the nodes
+    :return: None
+    """
+    logger.info(f"Loading {len(data)} SentinelOne applications into Neo4j")
+    load(
+        neo4j_session,
+        S1ApplicationSchema(),
+        data,
+        lastupdated=update_tag,
+        S1_ACCOUNT_ID=account_id,
+    )
+
+
+@timeit
+def load_application_versions(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    account_id: str,
+    update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(data)} SentinelOne application versions into Neo4j")
+    load(
+        neo4j_session,
+        S1ApplicationVersionSchema(),
+        data,
+        lastupdated=update_tag,
+        S1_ACCOUNT_ID=account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    logger.debug("Running SentinelOne S1Application cleanup")
+    GraphJob.from_node_schema(S1ApplicationSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    logger.debug("Running SentinelOne S1ApplicationVersion cleanup")
+    GraphJob.from_node_schema(S1ApplicationVersionSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    """
+    Sync SentinelOne applications
+    :param neo4j_session: Neo4j session
+    :param common_job_parameters: Common job parameters containing API_URL, API_TOKEN, etc.
+    :return: None
+    """
+    logger.info("Syncing SentinelOne application data")
+    account_id = str(common_job_parameters["S1_ACCOUNT_ID"])
+    api_url = str(common_job_parameters["API_URL"])
+    api_token = str(common_job_parameters["API_TOKEN"])
+
+    applications = get_application_data(account_id, api_url, api_token)
+    application_versions = get_application_installs(
+        applications, account_id, api_url, api_token
+    )
+    transformed_applications = transform_applications(applications)
+    transformed_application_versions = transform_application_versions(
+        application_versions
+    )
+
+    load_application_data(
+        neo4j_session,
+        transformed_applications,
+        account_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+
+    load_application_versions(
+        neo4j_session,
+        transformed_application_versions,
+        account_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+
+    cleanup(neo4j_session, common_job_parameters)

cartography-0.107.0rc3/cartography/intel/sentinelone/utils.py

@@ -0,0 +1,28 @@
+import re
+
+
+def get_application_id(name: str, vendor: str) -> str:
+    """
+    Generate a normalized unique identifier for an application.
+    :param name: Application name
+    :param vendor: Application vendor/publisher
+    :return: Normalized unique identifier in format 'vendor:name'
+    """
+    name_normalized = name.strip().lower().replace(" ", "_")
+    vendor_normalized = vendor.strip().lower().replace(" ", "_")
+    name_normalized = re.sub(r"[^\w]", "", name_normalized)
+    vendor_normalized = re.sub(r"[^\w]", "", vendor_normalized)
+    return f"{vendor_normalized}:{name_normalized}"
+
+
+def get_application_version_id(name: str, vendor: str, version: str) -> str:
+    """
+    Generate a unique identifier for an application version
+    :param name: Application name
+    :param vendor: Application vendor
+    :param version: Application version
+    :return: Unique identifier for the application version in format 'vendor:name:version'
+    """
+    app_id = get_application_id(name, vendor)
+    version_normalized = version.strip().lower().replace(" ", "_")
+    return f"{app_id}:{version_normalized}"

cartography-0.107.0rc3/cartography/models/aws/cloudwatch/log_metric_filter.py

@@ -0,0 +1,79 @@
+from dataclasses import dataclass
+
+from cartography.models.core.common import PropertyRef
+from cartography.models.core.nodes import CartographyNodeProperties
+from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.relationships import CartographyRelProperties
+from cartography.models.core.relationships import CartographyRelSchema
+from cartography.models.core.relationships import LinkDirection
+from cartography.models.core.relationships import make_target_node_matcher
+from cartography.models.core.relationships import OtherRelationships
+from cartography.models.core.relationships import TargetNodeMatcher
+
+
+@dataclass(frozen=True)
+class CloudWatchLogMetricFilterNodeProperties(CartographyNodeProperties):
+    id: PropertyRef = PropertyRef("id")
+    arn: PropertyRef = PropertyRef("filterName", extra_index=True)
+    region: PropertyRef = PropertyRef("Region", set_in_kwargs=True)
+    filter_name: PropertyRef = PropertyRef("filterName")
+    filter_pattern: PropertyRef = PropertyRef("filterPattern")
+    log_group_name: PropertyRef = PropertyRef("logGroupName")
+    metric_name: PropertyRef = PropertyRef("metricName")
+    metric_namespace: PropertyRef = PropertyRef("metricNamespace")
+    metric_value: PropertyRef = PropertyRef("metricValue")
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class CloudWatchLogMetricFilterToAwsAccountRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class CloudWatchLogMetricFilterToAWSAccountRel(CartographyRelSchema):
+    target_node_label: str = "AWSAccount"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {"id": PropertyRef("AWS_ID", set_in_kwargs=True)},
+    )
+    direction: LinkDirection = LinkDirection.INWARD
+    rel_label: str = "RESOURCE"
+    properties: CloudWatchLogMetricFilterToAwsAccountRelProperties = (
+        CloudWatchLogMetricFilterToAwsAccountRelProperties()
+    )
+
+
+@dataclass(frozen=True)
+class CloudWatchLogMetricFilterToCloudWatchLogGroupRelProperties(
+    CartographyRelProperties
+):
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class CloudWatchLogMetricFilterToCloudWatchLogGroupRel(CartographyRelSchema):
+    target_node_label: str = "CloudWatchLogGroup"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {"log_group_name": PropertyRef("logGroupName")},
+    )
+    direction: LinkDirection = LinkDirection.OUTWARD
+    rel_label: str = "METRIC_FILTER_OF"
+    properties: CloudWatchLogMetricFilterToCloudWatchLogGroupRelProperties = (
+        CloudWatchLogMetricFilterToCloudWatchLogGroupRelProperties()
+    )
+
+
+@dataclass(frozen=True)
+class CloudWatchLogMetricFilterSchema(CartographyNodeSchema):
+    label: str = "CloudWatchLogMetricFilter"
+    properties: CloudWatchLogMetricFilterNodeProperties = (
+        CloudWatchLogMetricFilterNodeProperties()
+    )
+    sub_resource_relationship: CloudWatchLogMetricFilterToAWSAccountRel = (
+        CloudWatchLogMetricFilterToAWSAccountRel()
+    )
+    other_relationships: OtherRelationships = OtherRelationships(
+        [
+            CloudWatchLogMetricFilterToCloudWatchLogGroupRel(),
+        ]
+    )

{cartography-0.107.0rc2 → cartography-0.107.0rc3}/cartography/models/aws/ec2/networkinterfaces.py

@@ -44,7 +44,9 @@ class EC2NetworkInterfaceNodeProperties(CartographyNodeProperties):
     requester_id: PropertyRef = PropertyRef("RequesterId", extra_index=True)
     requester_managed: PropertyRef = PropertyRef("RequesterManaged")
     source_dest_check: PropertyRef = PropertyRef("SourceDestCheck")
+    # TODO: remove subnetid once we have migrated to subnet_id
     subnetid: PropertyRef = PropertyRef("SubnetId", extra_index=True)
+    subnet_id: PropertyRef = PropertyRef("SubnetId", extra_index=True)
 
 
 @dataclass(frozen=True)

{cartography-0.107.0rc2 → cartography-0.107.0rc3}/cartography/models/aws/ec2/subnet_instance.py

@@ -15,7 +15,9 @@ from cartography.models.core.relationships import TargetNodeMatcher
 class EC2SubnetInstanceNodeProperties(CartographyNodeProperties):
     # arn: PropertyRef = PropertyRef('Arn', extra_index=True) TODO use arn; issue #1024
     id: PropertyRef = PropertyRef("SubnetId")
+    # TODO: remove subnetid once we have migrated to subnet_id
     subnetid: PropertyRef = PropertyRef("SubnetId", extra_index=True)
+    subnet_id: PropertyRef = PropertyRef("SubnetId", extra_index=True)
     region: PropertyRef = PropertyRef("Region", set_in_kwargs=True)
     lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
 

{cartography-0.107.0rc2 → cartography-0.107.0rc3}/cartography/models/aws/ec2/subnet_networkinterface.py

@@ -16,6 +16,8 @@ from cartography.models.core.relationships import TargetNodeMatcher
 @dataclass(frozen=True)
 class EC2SubnetNetworkInterfaceNodeProperties(CartographyNodeProperties):
     id: PropertyRef = PropertyRef("SubnetId")
+    # TODO: remove subnetid once we have migrated to subnet_id
+    subnetid: PropertyRef = PropertyRef("SubnetId", extra_index=True)
     subnet_id: PropertyRef = PropertyRef("SubnetId", extra_index=True)
     region: PropertyRef = PropertyRef("Region", set_in_kwargs=True)
     lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)