cartography 0.101.1rc2__tar.gz → 0.102.0rc1__tar.gz

{cartography-0.101.1rc2 → cartography-0.102.0rc1}/.github/workflows/ossf-scorecard.yml

@@ -57,6 +57,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@fc7e4a0fa01c3cca5fd6a1fddec5c0740c977aa2 # v3.28.14
+        uses: github/codeql-action/upload-sarif@45775bd8235c68ba998cffa5171334d58593da47 # v3.28.15
         with:
           sarif_file: results.sarif

{cartography-0.101.1rc2 → cartography-0.102.0rc1}/Dockerfile

@@ -1,6 +1,6 @@
 # This is a thin distribution of the cartography software.
 # It is published at ghcr.io.
-FROM python:3.10-slim@sha256:06f6d69d229bb55fab83dded514e54eede977e33e92d855ba3f97ce0e3234abc
+FROM python:3.10-slim@sha256:65c843653048a3ba22c8d5083a022f44aef774974f0f7f70cbf8cee4e931ac96
 
 # Default to ''. Overridden with a specific version specifier e.g. '==0.98.0' by build args or from GitHub actions.
 ARG VERSION_SPECIFIER

{cartography-0.101.1rc2 → cartography-0.102.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cartography
-Version: 0.101.1rc2
+Version: 0.102.0rc1
 Summary: Explore assets and their relationships across your technical infrastructure.
 Maintainer: Cartography Contributors
 License: apache2
@@ -47,6 +47,7 @@ Requires-Dist: msrestazure>=0.6.4
 Requires-Dist: azure-mgmt-storage>=16.0.0
 Requires-Dist: azure-mgmt-sql<=1.0.0
 Requires-Dist: azure-identity>=1.5.0
+Requires-Dist: msgraph-sdk
 Requires-Dist: kubernetes>=22.6.0
 Requires-Dist: pdpyras>=4.3.0
 Requires-Dist: crowdstrike-falconpy>=0.5.1
@@ -61,6 +62,7 @@ Requires-Dist: pytest>=6.2.4; extra == "dev"
 Requires-Dist: pytest-mock; extra == "dev"
 Requires-Dist: pytest-cov==6.1.1; extra == "dev"
 Requires-Dist: pytest-rerunfailures; extra == "dev"
+Requires-Dist: pytest-asyncio; extra == "dev"
 Requires-Dist: types-PyYAML; extra == "dev"
 Requires-Dist: types-requests<2.32.0.20250329; extra == "dev"
 Dynamic: license-file
@@ -97,6 +99,7 @@ You can learn more about the story behind Cartography in our [presentation at BS
 - [GitHub](https://cartography-cncf.github.io/cartography/modules/github/index.html) - repos, branches, users, teams
 - [DigitalOcean](https://cartography-cncf.github.io/cartography/modules/digitalocean/index.html)
 - [Microsoft Azure](https://cartography-cncf.github.io/cartography/modules/azure/index.html) -  CosmosDB, SQL, Storage, Virtual Machine
+- [Microsoft Entra ID](https://cartography-cncf.github.io/cartography/modules/entra/index.html) -  Users
 - [Kubernetes](https://cartography-cncf.github.io/cartography/modules/kubernetes/index.html) - Cluster, Namespace, Service, Pod, Container
 - [PagerDuty](https://cartography-cncf.github.io/cartography/modules/pagerduty/index.html) - Users, teams, services, schedules, escalation policies, integrations, vendors
 - [Crowdstrike Falcon](https://cartography-cncf.github.io/cartography/modules/crowdstrike/index.html) - Hosts, Spotlight vulnerabilities, CVEs

{cartography-0.101.1rc2 → cartography-0.102.0rc1}/README.md

@@ -30,6 +30,7 @@ You can learn more about the story behind Cartography in our [presentation at BS
 - [GitHub](https://cartography-cncf.github.io/cartography/modules/github/index.html) - repos, branches, users, teams
 - [DigitalOcean](https://cartography-cncf.github.io/cartography/modules/digitalocean/index.html)
 - [Microsoft Azure](https://cartography-cncf.github.io/cartography/modules/azure/index.html) -  CosmosDB, SQL, Storage, Virtual Machine
+- [Microsoft Entra ID](https://cartography-cncf.github.io/cartography/modules/entra/index.html) -  Users
 - [Kubernetes](https://cartography-cncf.github.io/cartography/modules/kubernetes/index.html) - Cluster, Namespace, Service, Pod, Container
 - [PagerDuty](https://cartography-cncf.github.io/cartography/modules/pagerduty/index.html) - Users, teams, services, schedules, escalation policies, integrations, vendors
 - [Crowdstrike Falcon](https://cartography-cncf.github.io/cartography/modules/crowdstrike/index.html) - Hosts, Spotlight vulnerabilities, CVEs

{cartography-0.101.1rc2 → cartography-0.102.0rc1}/cartography/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.101.1rc2'
-__version_tuple__ = version_tuple = (0, 101, 1)
+__version__ = version = '0.102.0rc1'
+__version_tuple__ = version_tuple = (0, 102, 0)

{cartography-0.101.1rc2 → cartography-0.102.0rc1}/cartography/cli.py

@@ -211,6 +211,30 @@ class CLI:
                 'The name of environment variable containing Azure Client Secret for Service Principal Authentication.'
             ),
         )
+        parser.add_argument(
+            '--entra-tenant-id',
+            type=str,
+            default=None,
+            help=(
+                'Entra Tenant Id for Service Principal Authentication.'
+            ),
+        )
+        parser.add_argument(
+            '--entra-client-id',
+            type=str,
+            default=None,
+            help=(
+                'Entra Client Id for Service Principal Authentication.'
+            ),
+        )
+        parser.add_argument(
+            '--entra-client-secret-env-var',
+            type=str,
+            default=None,
+            help=(
+                'The name of environment variable containing Entra Client Secret for Service Principal Authentication.'
+            ),
+        )
         parser.add_argument(
             '--aws-requested-syncs',
             type=str,
@@ -615,6 +639,16 @@ class CLI:
         else:
             config.azure_client_secret = None
 
+        # Entra config
+        if config.entra_tenant_id and config.entra_client_id and config.entra_client_secret_env_var:
+            logger.debug(
+                "Reading Client Secret for Entra Authentication from environment variable %s",
+                config.entra_client_secret_env_var,
+            )
+            config.entra_client_secret = os.environ.get(config.entra_client_secret_env_var)
+        else:
+            config.entra_client_secret = None
+
         # Okta config
         if config.okta_org_id and config.okta_api_key_env_var:
             logger.debug(f"Reading API key for Okta from environment variable {config.okta_api_key_env_var}")
@@ -798,5 +832,9 @@ def main(argv=None):
     logging.getLogger('botocore').setLevel(logging.WARNING)
     logging.getLogger('googleapiclient').setLevel(logging.WARNING)
     logging.getLogger('neo4j').setLevel(logging.WARNING)
+    logging.getLogger('azure.identity').setLevel(logging.WARNING)
+    logging.getLogger('httpx').setLevel(logging.WARNING)
+    logging.getLogger('azure.core.pipeline.policies.http_logging_policy').setLevel(logging.WARNING)
+
     argv = argv if argv is not None else sys.argv[1:]
     sys.exit(CLI(prog='cartography').main(argv))

{cartography-0.101.1rc2 → cartography-0.102.0rc1}/cartography/config.py

@@ -41,6 +41,12 @@ class Config:
     :param azure_client_id: Client Id for connecting in a Service Principal Authentication approach. Optional.
     :type azure_client_secret: str
     :param azure_client_secret: Client Secret for connecting in a Service Principal Authentication approach. Optional.
+    :type entra_tenant_id: str
+    :param entra_tenant_id: Tenant Id for connecting in a Service Principal Authentication approach. Optional.
+    :type entra_client_id: str
+    :param entra_client_id: Client Id for connecting in a Service Principal Authentication approach. Optional.
+    :type entra_client_secret: str
+    :param entra_client_secret: Client Secret for connecting in a Service Principal Authentication approach. Optional.
     :type aws_requested_syncs: str
     :param aws_requested_syncs: Comma-separated list of AWS resources to sync. Optional.
     :type analysis_job_directory: str
@@ -133,6 +139,9 @@ class Config:
         azure_tenant_id=None,
         azure_client_id=None,
         azure_client_secret=None,
+        entra_tenant_id=None,
+        entra_client_id=None,
+        entra_client_secret=None,
         aws_requested_syncs=None,
         analysis_job_directory=None,
         oci_sync_all_profiles=None,
@@ -191,6 +200,9 @@ class Config:
         self.azure_tenant_id = azure_tenant_id
         self.azure_client_id = azure_client_id
         self.azure_client_secret = azure_client_secret
+        self.entra_tenant_id = entra_tenant_id
+        self.entra_client_id = entra_client_id
+        self.entra_client_secret = entra_client_secret
         self.aws_requested_syncs = aws_requested_syncs
         self.analysis_job_directory = analysis_job_directory
         self.oci_sync_all_profiles = oci_sync_all_profiles

{cartography-0.101.1rc2 → cartography-0.102.0rc1}/cartography/data/indexes.cypher

@@ -191,9 +191,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:KMSGrant) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:LaunchConfiguration) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:LaunchConfiguration) ON (n.name);
 CREATE INDEX IF NOT EXISTS FOR (n:LaunchConfiguration) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:LoadBalancer) ON (n.dnsname);
-CREATE INDEX IF NOT EXISTS FOR (n:LoadBalancer) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:LoadBalancer) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:LoadBalancerV2) ON (n.dnsname);
 CREATE INDEX IF NOT EXISTS FOR (n:LoadBalancerV2) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:LoadBalancerV2) ON (n.lastupdated);

{cartography-0.101.1rc2 → cartography-0.102.0rc1}/cartography/intel/aws/ec2/launch_templates.py

@@ -37,37 +37,18 @@ def get_launch_template_versions(
     boto3_session: boto3.session.Session,
     region: str,
     launch_templates: list[dict[str, Any]],
-) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
-    found_versions: list[dict[str, Any]] = []
-    found_templates: list[dict[str, Any]] = []
+) -> list[dict[str, Any]]:
+    template_versions: list[dict[str, Any]] = []
     for template in launch_templates:
         launch_template_id = template['LaunchTemplateId']
-        try:
-            versions = get_launch_template_versions_by_template(
-                boto3_session,
-                launch_template_id,
-                region,
-            )
-            # If the call succeeded, the template still exists.
-            # Add it and its versions (list might be empty if no versions exist).
-            found_templates.append(template)
-            found_versions.extend(versions)
-        except botocore.exceptions.ClientError as e:
-            if e.response['Error']['Code'] == 'InvalidLaunchTemplateId.NotFound':
-                logger.warning(
-                    "Launch template %s no longer exists in region %s, skipping.",
-                    launch_template_id, region,
-                )
-                # Skip this template, don't add it or its versions
-                continue
-            else:
-                # Re-raise any other client error
-                raise
-
-    return found_versions, found_templates
+        versions = get_launch_template_versions_by_template(boto3_session, launch_template_id, region)
+        template_versions.extend(versions)
+
+    return template_versions
 
 
 @timeit
+@aws_handle_regions
 def get_launch_template_versions_by_template(
         boto3_session: boto3.session.Session,
         launch_template_id: str,
@@ -76,15 +57,27 @@ def get_launch_template_versions_by_template(
     client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
     v_paginator = client.get_paginator('describe_launch_template_versions')
     template_versions = []
-    for versions_page in v_paginator.paginate(LaunchTemplateId=launch_template_id):
-        template_versions.extend(versions_page['LaunchTemplateVersions'])
+    try:
+        for versions in v_paginator.paginate(LaunchTemplateId=launch_template_id):
+            template_versions.extend(versions['LaunchTemplateVersions'])
+    except botocore.exceptions.ClientError as e:
+        error_code = e.response['Error']['Code']
+        if error_code == 'InvalidLaunchTemplateId.NotFound':
+            logger.warning("Launch template %s no longer exists in region %s", launch_template_id, region)
+        else:
+            raise
     return template_versions
 
 
-def transform_launch_templates(templates: list[dict[str, Any]]) -> list[dict[str, Any]]:
+def transform_launch_templates(templates: list[dict[str, Any]], versions: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    valid_template_ids = {v['LaunchTemplateId'] for v in versions}
     result: list[dict[str, Any]] = []
     for template in templates:
+        if template['LaunchTemplateId'] not in valid_template_ids:
+            continue
+
         current = template.copy()
+        # Convert CreateTime to timestamp string
         current['CreateTime'] = str(int(current['CreateTime'].timestamp()))
         result.append(current)
     return result
@@ -176,11 +169,13 @@ def sync_ec2_launch_templates(
     for region in regions:
         logger.info(f"Syncing launch templates for region '{region}' in account '{current_aws_account_id}'.")
         templates = get_launch_templates(boto3_session, region)
-        versions, found_templates = get_launch_template_versions(boto3_session, region, templates)
+        versions = get_launch_template_versions(boto3_session, region, templates)
 
-        # Transform and load only the templates that were found to exist
-        transformed_templates = transform_launch_templates(found_templates)
+        # Transform and load the templates that have versions
+        transformed_templates = transform_launch_templates(templates, versions)
         load_launch_templates(neo4j_session, transformed_templates, region, current_aws_account_id, update_tag)
+
+        # Transform and load the versions
         transformed_versions = transform_launch_template_versions(versions)
         load_launch_template_versions(neo4j_session, transformed_versions, region, current_aws_account_id, update_tag)
 

cartography-0.102.0rc1/cartography/intel/aws/ec2/load_balancers.py

@@ -0,0 +1,168 @@
+import logging
+
+import boto3
+import neo4j
+
+from .util import get_botocore_config
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.ec2.load_balancer_listeners import ELBListenerSchema
+from cartography.models.aws.ec2.load_balancers import LoadBalancerSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+def _get_listener_id(load_balancer_id: str, port: int, protocol: str) -> str:
+    """
+    Generate a unique ID for a load balancer listener.
+
+    Args:
+        load_balancer_id: The ID of the load balancer
+        port: The listener port
+        protocol: The listener protocol
+
+    Returns:
+        A unique ID string for the listener
+    """
+    return f"{load_balancer_id}{port}{protocol}"
+
+
+def transform_load_balancer_listener_data(load_balancer_id: str, listener_data: list[dict]) -> list[dict]:
+    """
+    Transform load balancer listener data into a format suitable for cartography ingestion.
+
+    Args:
+        load_balancer_id: The ID of the load balancer
+        listener_data: List of listener data from AWS API
+
+    Returns:
+        List of transformed listener data
+    """
+    transformed = []
+    for listener in listener_data:
+        listener_info = listener['Listener']
+        transformed_listener = {
+            'id': _get_listener_id(load_balancer_id, listener_info['LoadBalancerPort'], listener_info['Protocol']),
+            'port': listener_info.get('LoadBalancerPort'),
+            'protocol': listener_info.get('Protocol'),
+            'instance_port': listener_info.get('InstancePort'),
+            'instance_protocol': listener_info.get('InstanceProtocol'),
+            'policy_names': listener.get('PolicyNames', []),
+            'LoadBalancerId': load_balancer_id,
+        }
+        transformed.append(transformed_listener)
+    return transformed
+
+
+def transform_load_balancer_data(load_balancers: list[dict]) -> tuple[list[dict], list[dict]]:
+    """
+    Transform load balancer data into a format suitable for cartography ingestion.
+
+    Args:
+        load_balancers: List of load balancer data from AWS API
+
+    Returns:
+        Tuple of (transformed load balancer data, transformed listener data)
+    """
+    transformed = []
+    listener_data = []
+
+    for lb in load_balancers:
+        load_balancer_id = lb['DNSName']
+        transformed_lb = {
+            'id': load_balancer_id,
+            'name': lb['LoadBalancerName'],
+            'dnsname': lb['DNSName'],
+            'canonicalhostedzonename': lb.get('CanonicalHostedZoneName'),
+            'canonicalhostedzonenameid': lb.get('CanonicalHostedZoneNameID'),
+            'scheme': lb.get('Scheme'),
+            'createdtime': str(lb['CreatedTime']),
+            'GROUP_NAME': lb.get('SourceSecurityGroup', {}).get('GroupName'),
+            'GROUP_IDS': [str(group) for group in lb.get('SecurityGroups', [])],
+            'INSTANCE_IDS': [instance['InstanceId'] for instance in lb.get('Instances', [])],
+            'LISTENER_IDS': [
+                _get_listener_id(
+                    load_balancer_id,
+                    listener['Listener']['LoadBalancerPort'],
+                    listener['Listener']['Protocol'],
+                ) for listener in lb.get('ListenerDescriptions', [])
+            ],
+        }
+        transformed.append(transformed_lb)
+
+        # Classic ELB listeners are not returned anywhere else in AWS, so we must parse them out
+        # of the describe_load_balancers response.
+        if lb.get('ListenerDescriptions'):
+            listener_data.extend(
+                transform_load_balancer_listener_data(
+                    load_balancer_id,
+                    lb.get('ListenerDescriptions', []),
+                ),
+            )
+
+    return transformed, listener_data
+
+
+@timeit
+@aws_handle_regions
+def get_loadbalancer_data(boto3_session: boto3.session.Session, region: str) -> list[dict]:
+    client = boto3_session.client('elb', region_name=region, config=get_botocore_config())
+    paginator = client.get_paginator('describe_load_balancers')
+    elbs: list[dict] = []
+    for page in paginator.paginate():
+        elbs.extend(page['LoadBalancerDescriptions'])
+    return elbs
+
+
+@timeit
+def load_load_balancers(
+    neo4j_session: neo4j.Session, data: list[dict], region: str, current_aws_account_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        LoadBalancerSchema(),
+        data,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+def load_load_balancer_listeners(
+    neo4j_session: neo4j.Session, data: list[dict], region: str, current_aws_account_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        ELBListenerSchema(),
+        data,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+def cleanup_load_balancers(neo4j_session: neo4j.Session, common_job_parameters: dict) -> None:
+    GraphJob.from_node_schema(ELBListenerSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(LoadBalancerSchema(), common_job_parameters).run(neo4j_session)
+
+
+@timeit
+def sync_load_balancers(
+    neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: list[str], current_aws_account_id: str,
+    update_tag: int, common_job_parameters: dict,
+) -> None:
+    for region in regions:
+        logger.info("Syncing EC2 load balancers for region '%s' in account '%s'.", region, current_aws_account_id)
+        data = get_loadbalancer_data(boto3_session, region)
+        transformed_data, listener_data = transform_load_balancer_data(data)
+
+        load_load_balancers(neo4j_session, transformed_data, region, current_aws_account_id, update_tag)
+        load_load_balancer_listeners(neo4j_session, listener_data, region, current_aws_account_id, update_tag)
+
+    cleanup_load_balancers(neo4j_session, common_job_parameters)

cartography-0.102.0rc1/cartography/intel/entra/__init__.py

@@ -0,0 +1,43 @@
+import asyncio
+import logging
+
+import neo4j
+
+from cartography.config import Config
+from cartography.intel.entra.users import sync_entra_users
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def start_entra_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of Entra data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+
+    if not config.entra_tenant_id or not config.entra_client_id or not config.entra_client_secret:
+        logger.info(
+            'Entra import is not configured - skipping this module. '
+            'See docs to configure.',
+        )
+        return
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+        "TENANT_ID": config.entra_tenant_id,
+    }
+
+    asyncio.run(
+        sync_entra_users(
+            neo4j_session,
+            config.entra_tenant_id,
+            config.entra_client_id,
+            config.entra_client_secret,
+            config.update_tag,
+            common_job_parameters,
+        ),
+    )