cartography 0.96.0rc1__tar.gz → 0.96.0rc3__tar.gz

{cartography-0.96.0rc1 → cartography-0.96.0rc3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cartography
-Version: 0.96.0rc1
+Version: 0.96.0rc3
 Summary: Explore assets and their relationships across your technical infrastructure.
 Home-page: https://www.github.com/cartography-cncf/cartography
 Maintainer: Cartography Contributors

{cartography-0.96.0rc1 → cartography-0.96.0rc3}/cartography/cli.py

@@ -9,6 +9,7 @@ import cartography.config
 import cartography.sync
 import cartography.util
 from cartography.intel.aws.util.common import parse_and_validate_aws_requested_syncs
+from cartography.intel.semgrep.dependencies import parse_and_validate_semgrep_ecosystems
 
 
 logger = logging.getLogger(__name__)
@@ -524,6 +525,17 @@ class CLI:
                 'Required if you are using the Semgrep intel module. Ignored otherwise.'
             ),
         )
+        parser.add_argument(
+            '--semgrep-dependency-ecosystems',
+            type=str,
+            default=None,
+            help=(
+                'Comma-separated list of language ecosystems for which dependencies will be retrieved from Semgrep. '
+                'For example, a value of "gomod,npm" will retrieve Go and NPM dependencies. '
+                'See the full list of supported ecosystems in source code at cartography.intel.semgrep.dependencies. '
+                'Required if you are using the Semgrep dependencies intel module. Ignored otherwise.'
+            ),
+        )
         parser.add_argument(
             '--snipeit-base-uri',
             type=str,
@@ -734,6 +746,9 @@ class CLI:
             config.semgrep_app_token = os.environ.get(config.semgrep_app_token_env_var)
         else:
             config.semgrep_app_token = None
+        if config.semgrep_dependency_ecosystems:
+            # No need to store the returned value; we're using this for input validation.
+            parse_and_validate_semgrep_ecosystems(config.semgrep_dependency_ecosystems)
 
         # CVE feed config
         if config.cve_api_key_env_var:

{cartography-0.96.0rc1 → cartography-0.96.0rc3}/cartography/client/core/tx.py

@@ -122,7 +122,7 @@ def read_list_of_tuples_tx(tx: neo4j.Transaction, query: str, **kwargs) -> List[
     return [tuple(val) for val in values]
 
 
-def read_single_dict_tx(tx: neo4j.Transaction, query: str, **kwargs) -> Dict[str, Any]:
+def read_single_dict_tx(tx: neo4j.Transaction, query: str, **kwargs) -> Any:
     """
     Runs the given Neo4j query in the given transaction object and returns the single dict result. This is intended to
     be run only with queries that return a single dict.

{cartography-0.96.0rc1 → cartography-0.96.0rc3}/cartography/config.py

@@ -107,6 +107,8 @@ class Config:
     :param duo_api_hostname: The Duo api hostname, e.g. "api-abc123.duosecurity.com". Optional.
     :param semgrep_app_token: The Semgrep api token. Optional.
     :type semgrep_app_token: str
+    :param semgrep_dependency_ecosystems: Comma-separated list of Semgrep dependency ecosystems to fetch. Optional.
+    :type semgrep_dependency_ecosystems: str
     :type snipeit_base_uri: string
     :param snipeit_base_uri: SnipeIT data provider base URI. Optional.
     :type snipeit_token: string
@@ -155,7 +157,7 @@ class Config:
         pagerduty_request_timeout=None,
         nist_cve_url=None,
         cve_enabled=False,
-        cve_api_key=None,
+        cve_api_key: str | None = None,
         crowdstrike_client_id=None,
         crowdstrike_client_secret=None,
         crowdstrike_api_url=None,
@@ -170,6 +172,7 @@ class Config:
         duo_api_secret=None,
         duo_api_hostname=None,
         semgrep_app_token=None,
+        semgrep_dependency_ecosystems=None,
         snipeit_base_uri=None,
         snipeit_token=None,
         snipeit_tenant_id=None,
@@ -212,7 +215,7 @@ class Config:
         self.pagerduty_request_timeout = pagerduty_request_timeout
         self.nist_cve_url = nist_cve_url
         self.cve_enabled = cve_enabled
-        self.cve_api_key = cve_api_key
+        self.cve_api_key: str | None = cve_api_key
         self.crowdstrike_client_id = crowdstrike_client_id
         self.crowdstrike_client_secret = crowdstrike_client_secret
         self.crowdstrike_api_url = crowdstrike_api_url
@@ -227,6 +230,7 @@ class Config:
         self.duo_api_secret = duo_api_secret
         self.duo_api_hostname = duo_api_hostname
         self.semgrep_app_token = semgrep_app_token
+        self.semgrep_dependency_ecosystems = semgrep_dependency_ecosystems
         self.snipeit_base_uri = snipeit_base_uri
         self.snipeit_token = snipeit_token
         self.snipeit_tenant_id = snipeit_tenant_id

{cartography-0.96.0rc1 → cartography-0.96.0rc3}/cartography/data/indexes.cypher

@@ -305,8 +305,7 @@ CREATE INDEX IF NOT EXISTS FOR (n:SpotlightVulnerability) ON (n.host_info_local_
 CREATE INDEX IF NOT EXISTS FOR (n:SpotlightVulnerability) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:SQSQueue) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:SQSQueue) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:User) ON (n.arn);
-CREATE INDEX IF NOT EXISTS FOR (n:User) ON (n.lastupdated);
+CREATE INDEX IF NOT EXISTS FOR (n:UserAccount) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:AzureTenant) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:AzureTenant) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:AzurePrincipal) ON (n.email);

cartography-0.96.0rc3/cartography/data/jobs/cleanup/aws_import_identity_center_cleanup.json

@@ -0,0 +1,16 @@
+{
+  "statements": [
+
+    {
+      "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(:AWSSSOUser)<-[r:CAN_ASSUME_IDENTITY]-(:OktaUser) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r) RETURN COUNT(*) as TotalDeleted",
+      "iterative": true,
+      "iterationsize": 100
+    },
+    {
+      "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(:AWSRole)-[r:ALLOWED_BY]->(:AWSSSOUser) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r) RETURN COUNT(*) as TotalDeleted",
+      "iterative": true,
+      "iterationsize": 100
+    }
+  ],
+  "name": "cleanup AWS Identity Center Instances and Related Data"
+}

cartography-0.96.0rc1/cartography/data/jobs/cleanup/github_users_cleanup.json → cartography-0.96.0rc3/cartography/data/jobs/cleanup/github_org_and_users_cleanup.json

@@ -18,6 +18,11 @@
     "query": "MATCH (:GitHubUser)-[r:MEMBER_OF]->(:GitHubOrganization) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
     "iterative": true,
     "iterationsize": 100
+  },
+  {
+    "query": "MATCH (:GitHubUser)-[r:UNAFFILIATED]->(:GitHubOrganization) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
+    "iterative": true,
+    "iterationsize": 100
   }],
   "name": "cleanup GitHub users data"
 }

{cartography-0.96.0rc1 → cartography-0.96.0rc3}/cartography/intel/aws/apigateway.py

@@ -43,7 +43,7 @@ def get_rest_api_details(
     for api in rest_apis:
         stages = get_rest_api_stages(api, client)
         # clientcertificate id is given by the api stage
-        certificate = get_rest_api_client_certificate(stages, client)  # type: ignore
+        certificate = get_rest_api_client_certificate(stages, client)
         resources = get_rest_api_resources(api, client)
         policy = get_rest_api_policy(api, client)
         apis.append((api['id'], stages, certificate, resources, policy))
@@ -51,7 +51,7 @@ def get_rest_api_details(
 
 
 @timeit
-def get_rest_api_stages(api: Dict, client: botocore.client.BaseClient) -> List[Any]:
+def get_rest_api_stages(api: Dict, client: botocore.client.BaseClient) -> Any:
     """
     Gets the REST API Stage Resources.
     """
@@ -99,7 +99,7 @@ def get_rest_api_resources(api: Dict, client: botocore.client.BaseClient) -> Lis
 
 
 @timeit
-def get_rest_api_policy(api: Dict, client: botocore.client.BaseClient) -> List[Any]:
+def get_rest_api_policy(api: Dict, client: botocore.client.BaseClient) -> Any:
     """
     Gets the REST API policy. Returns policy string or None if no policy is present.
     """

cartography-0.96.0rc3/cartography/intel/aws/identitycenter.py

@@ -0,0 +1,307 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.identitycenter.awsidentitycenter import AWSIdentityCenterInstanceSchema
+from cartography.models.aws.identitycenter.awspermissionset import AWSPermissionSetSchema
+from cartography.models.aws.identitycenter.awsssouser import AWSSSOUserSchema
+from cartography.util import aws_handle_regions
+from cartography.util import run_cleanup_job
+from cartography.util import timeit
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_identity_center_instances(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
+    """
+    Get all AWS IAM Identity Center instances in the current region
+    """
+    client = boto3_session.client('sso-admin', region_name=region)
+    instances = []
+
+    paginator = client.get_paginator('list_instances')
+    for page in paginator.paginate():
+        instances.extend(page.get('Instances', []))
+
+    return instances
+
+
+@timeit
+def load_identity_center_instances(
+    neo4j_session: neo4j.Session,
+    instance_data: List[Dict],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    """
+    Load Identity Center instances into the graph
+    """
+    logger.info(f"Loading {len(instance_data)} Identity Center instances for region {region}")
+    load(
+        neo4j_session,
+        AWSIdentityCenterInstanceSchema(),
+        instance_data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def get_permission_sets(boto3_session: boto3.session.Session, instance_arn: str, region: str) -> List[Dict]:
+    """
+    Get all permission sets for a given Identity Center instance
+    """
+    client = boto3_session.client('sso-admin', region_name=region)
+    permission_sets = []
+
+    paginator = client.get_paginator('list_permission_sets')
+    for page in paginator.paginate(InstanceArn=instance_arn):
+        # Get detailed info for each permission set
+        for arn in page.get('PermissionSets', []):
+            details = client.describe_permission_set(
+                InstanceArn=instance_arn,
+                PermissionSetArn=arn,
+            )
+            permission_set = details.get('PermissionSet', {})
+            if permission_set:
+                permission_set['RoleHint'] = (
+                    f":role/aws-reserved/sso.amazonaws.com/AWSReservedSSO_{permission_set.get('Name')}"
+                )
+                permission_sets.append(permission_set)
+
+    return permission_sets
+
+
+@timeit
+def get_permission_set_roles(
+    boto3_session: boto3.session.Session,
+    instance_arn: str,
+    permission_set_arn: str,
+    region: str,
+) -> List[Dict]:
+    """
+    Get all accounts associated with a given permission set
+    """
+    client = boto3_session.client('sso-admin', region_name=region)
+    accounts = []
+
+    paginator = client.get_paginator('list_accounts_for_provisioned_permission_set')
+    for page in paginator.paginate(InstanceArn=instance_arn, PermissionSetArn=permission_set_arn):
+        accounts.extend(page.get('AccountIds', []))
+
+    return accounts
+
+
+@timeit
+def load_permission_sets(
+    neo4j_session: neo4j.Session,
+    permission_sets: List[Dict],
+    instance_arn: str,
+    region: str,
+    aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    """
+    Load Identity Center permission sets into the graph
+    """
+    logger.info(f"Loading {len(permission_sets)} permission sets for instance {instance_arn} in region {region}")
+
+    load(
+        neo4j_session,
+        AWSPermissionSetSchema(),
+        permission_sets,
+        lastupdated=aws_update_tag,
+        InstanceArn=instance_arn,
+        Region=region,
+        AWS_ID=aws_account_id,
+    )
+
+
+@timeit
+def get_sso_users(
+    boto3_session: boto3.session.Session,
+    identity_store_id: str,
+    region: str,
+) -> List[Dict]:
+    """
+    Get all SSO users for a given Identity Store
+    """
+    client = boto3_session.client('identitystore', region_name=region)
+    users = []
+
+    paginator = client.get_paginator('list_users')
+    for page in paginator.paginate(IdentityStoreId=identity_store_id):
+        user_page = page.get('Users', [])
+        for user in user_page:
+            if user.get('ExternalIds', None):
+                user['ExternalId'] = user.get('ExternalIds')[0].get('Id')
+            users.append(user)
+
+    return users
+
+
+@timeit
+def load_sso_users(
+    neo4j_session: neo4j.Session,
+    users: List[Dict],
+    identity_store_id: str,
+    region: str,
+    aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    """
+    Load SSO users into the graph
+    """
+    logger.info(f"Loading {len(users)} SSO users for identity store {identity_store_id} in region {region}")
+
+    load(
+        neo4j_session,
+        AWSSSOUserSchema(),
+        users,
+        lastupdated=aws_update_tag,
+        IdentityStoreId=identity_store_id,
+        AWS_ID=aws_account_id,
+        Region=region,
+    )
+
+
+@timeit
+def get_role_assignments(
+    boto3_session: boto3.session.Session,
+    users: List[Dict],
+    instance_arn: str,
+    region: str,
+) -> List[Dict]:
+    """
+    Get role assignments for SSO users
+    """
+
+    logger.info(f"Getting role assignments for {len(users)} users")
+    client = boto3_session.client('sso-admin', region_name=region)
+    role_assignments = []
+
+    for user in users:
+        user_id = user['UserId']
+        paginator = client.get_paginator('list_account_assignments_for_principal')
+        for page in paginator.paginate(InstanceArn=instance_arn, PrincipalId=user_id, PrincipalType='USER'):
+            for assignment in page.get('AccountAssignments', []):
+                role_assignments.append({
+                    'UserId': user_id,
+                    'PermissionSetArn': assignment.get('PermissionSetArn'),
+                    'AccountId': assignment.get('AccountId'),
+                })
+
+    return role_assignments
+
+
+@timeit
+def load_role_assignments(
+    neo4j_session: neo4j.Session,
+    role_assignments: List[Dict],
+    aws_update_tag: int,
+) -> None:
+    """
+    Load role assignments into the graph
+    """
+    logger.info(f"Loading {len(role_assignments)} role assignments")
+    if role_assignments:
+        neo4j_session.run(
+            """
+            UNWIND $role_assignments AS ra
+            MATCH (acc:AWSAccount{id:ra.AccountId}) -[:RESOURCE]->
+            (role:AWSRole)<-[:ASSIGNED_TO_ROLE]-
+            (permset:AWSPermissionSet {id: ra.PermissionSetArn})
+            MATCH (sso:AWSSSOUser {id: ra.UserId})
+            MERGE (role)-[r:ALLOWED_BY]->(sso)
+            SET r.lastupdated = $aws_update_tag,
+            r.permission_set_arn = ra.PermissionSetArn
+            """,
+            role_assignments=role_assignments,
+            aws_update_tag=aws_update_tag,
+        )
+
+
+def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
+    GraphJob.from_node_schema(AWSIdentityCenterInstanceSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(AWSPermissionSetSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(AWSSSOUserSchema(), common_job_parameters).run(neo4j_session)
+    run_cleanup_job(
+        'aws_import_identity_center_cleanup.json',
+        neo4j_session,
+        common_job_parameters,
+    )
+
+
+@timeit
+def sync_identity_center_instances(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict,
+) -> None:
+    """
+    Sync Identity Center instances, their permission sets, and SSO users
+    """
+    logger.info(f"Syncing Identity Center instances for regions {regions}")
+    for region in regions:
+        logger.info(f"Syncing Identity Center instances for region {region}")
+        instances = get_identity_center_instances(boto3_session, region)
+        load_identity_center_instances(
+            neo4j_session,
+            instances,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+        # For each instance, get and load its permission sets and SSO users
+        for instance in instances:
+            instance_arn = instance['InstanceArn']
+            identity_store_id = instance['IdentityStoreId']
+
+            permission_sets = get_permission_sets(boto3_session, instance_arn, region)
+
+            load_permission_sets(
+                neo4j_session,
+                permission_sets,
+                instance_arn,
+                region,
+                current_aws_account_id,
+                update_tag,
+            )
+
+            users = get_sso_users(boto3_session, identity_store_id, region)
+            load_sso_users(
+                neo4j_session,
+                users,
+                identity_store_id,
+                region,
+                current_aws_account_id,
+                update_tag,
+            )
+
+            # Get and load role assignments
+            role_assignments = get_role_assignments(
+                boto3_session,
+                users,
+                instance_arn,
+                region,
+            )
+            load_role_assignments(
+                neo4j_session,
+                role_assignments,
+                update_tag,
+            )
+
+    cleanup(neo4j_session, common_job_parameters)

{cartography-0.96.0rc1 → cartography-0.96.0rc3}/cartography/intel/aws/resources.py

@@ -10,6 +10,7 @@ from . import elasticache
 from . import elasticsearch
 from . import emr
 from . import iam
+from . import identitycenter
 from . import inspector
 from . import kms
 from . import lambda_function
@@ -88,4 +89,5 @@ RESOURCE_FUNCTIONS: Dict = {
     'ssm': ssm.sync,
     'inspector': inspector.sync,
     'config': config.sync,
+    'identitycenter': identitycenter.sync_identity_center_instances,
 }

{cartography-0.96.0rc1 → cartography-0.96.0rc3}/cartography/intel/cve/__init__.py

@@ -25,7 +25,7 @@ def start_cve_ingestion(
     """
     if not config.cve_enabled:
         return
-    cve_api_key = config.cve_api_key if config.cve_api_key else None
+    cve_api_key: str | None = config.cve_api_key if config.cve_api_key else None
 
     # sync CVE year archives, if not yet synced
     existing_years = feed.get_cve_sync_metadata(neo4j_session)

{cartography-0.96.0rc1 → cartography-0.96.0rc3}/cartography/intel/cve/feed.py

@@ -68,7 +68,7 @@ def _map_cve_dict(cve_dict: Dict[Any, Any], data: Dict[Any, Any]) -> None:
     cve_dict["startIndex"] = data["startIndex"]
 
 
-def _call_cves_api(url: str, api_key: str, params: Dict[str, Any]) -> Dict[Any, Any]:
+def _call_cves_api(url: str, api_key: str | None, params: Dict[str, Any]) -> Dict[Any, Any]:
     totalResults = 0
     sleep_time = DEFAULT_SLEEP_TIME
     retries = 0
@@ -114,7 +114,7 @@ def get_cves_in_batches(
     start_date: datetime,
     end_date: datetime,
     date_param_names: Dict[str, str],
-    api_key: str,
+    api_key: str | None,
 ) -> Dict[Any, Any]:
     cves: Dict[Any, Any] = dict()
     current_start_date: datetime = start_date
@@ -153,7 +153,7 @@ def get_cves_in_batches(
 
 
 def get_modified_cves(
-    nist_cve_url: str, last_modified_date: str, api_key: str,
+    nist_cve_url: str, last_modified_date: str, api_key: str | None,
 ) -> Dict[Any, Any]:
     cves = dict()
     end_date = datetime.now(tz=timezone.utc)
@@ -171,7 +171,7 @@ def get_modified_cves(
 
 
 def get_published_cves_per_year(
-    nist_cve_url: str, year: str, api_key: str,
+    nist_cve_url: str, year: str, api_key: str | None,
 ) -> Dict[Any, Any]:
     cves = {}
     start_of_year = datetime.strptime(f"{year}-01-01", "%Y-%m-%d")