cartography 0.103.0__py3-none-any.whl → 0.104.0__py3-none-any.whl

cartography/intel/anthropic/workspaces.py

@@ -0,0 +1,95 @@
+import logging
+from typing import Any
+from typing import Tuple
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.anthropic.util import paginated_get
+from cartography.models.anthropic.workspace import AnthropicWorkspaceSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: dict[str, Any],
+) -> list[dict]:
+    org_id, workspaces = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+    )
+    common_job_parameters["ORG_ID"] = org_id
+    for workspace in workspaces:
+        workspace["users"] = []
+        workspace["admins"] = []
+        for user in get_workspace_users(
+            api_session,
+            common_job_parameters["BASE_URL"],
+            workspace["id"],
+        ):
+            workspace["users"].append(user["user_id"])
+            if user["workspace_role"] == "workspace_admin":
+                workspace["admins"].append(user["user_id"])
+    load_workspaces(
+        neo4j_session, workspaces, org_id, common_job_parameters["UPDATE_TAG"]
+    )
+    cleanup(neo4j_session, common_job_parameters)
+    return workspaces
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+) -> Tuple[str, list[dict[str, Any]]]:
+    return paginated_get(
+        api_session, f"{base_url}/organizations/workspaces", timeout=_TIMEOUT
+    )
+
+
+@timeit
+def get_workspace_users(
+    api_session: requests.Session,
+    base_url: str,
+    workspace_id: str,
+) -> list[dict[str, Any]]:
+    _, result = paginated_get(
+        api_session,
+        f"{base_url}/organizations/workspaces/{workspace_id}/members",
+        timeout=_TIMEOUT,
+    )
+    return result
+
+
+@timeit
+def load_workspaces(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    ORG_ID: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Anthropic workspaces into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        AnthropicWorkspaceSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=ORG_ID,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(AnthropicWorkspaceSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/aws/cloudtrail.py

@@ -4,7 +4,6 @@ from typing import Dict
 from typing import List
 
 import boto3
-import botocore.exceptions
 import neo4j
 
 from cartography.client.core.tx import load
@@ -25,10 +24,8 @@ def get_cloudtrail_trails(
     client = boto3_session.client(
         "cloudtrail", region_name=region, config=get_botocore_config()
     )
-    paginator = client.get_paginator("list_trails")
-    trails = []
-    for page in paginator.paginate():
-        trails.extend(page["Trails"])
+
+    trails = client.describe_trails()["trailList"]
 
     # CloudTrail multi-region trails are shown in list_trails,
     # but the get_trail call only works in the home region
@@ -36,28 +33,6 @@ def get_cloudtrail_trails(
     return trails_filtered
 
 
-@timeit
-def get_cloudtrail_trail(
-    boto3_session: boto3.Session,
-    region: str,
-    trail_name: str,
-) -> Dict[str, Any]:
-    client = boto3_session.client(
-        "cloudtrail", region_name=region, config=get_botocore_config()
-    )
-    trail_details: Dict[str, Any] = {}
-    try:
-        response = client.get_trail(Name=trail_name)
-        trail_details = response["Trail"]
-    except botocore.exceptions.ClientError as e:
-        code = e.response["Error"]["Code"]
-        msg = e.response["Error"]["Message"]
-        logger.warning(
-            f"Could not run CloudTrail get_trail due to boto3 error {code}: {msg}. Skipping.",
-        )
-    return trail_details
-
-
 @timeit
 def load_cloudtrail_trails(
     neo4j_session: neo4j.Session,
@@ -105,20 +80,10 @@ def sync(
             f"Syncing CloudTrail for region '{region}' in account '{current_aws_account_id}'.",
         )
         trails = get_cloudtrail_trails(boto3_session, region)
-        trail_data: List[Dict[str, Any]] = []
-        for trail in trails:
-            trail_name = trail["Name"]
-            trail_details = get_cloudtrail_trail(
-                boto3_session,
-                region,
-                trail_name,
-            )
-            if trail_details:
-                trail_data.append(trail_details)
 
         load_cloudtrail_trails(
             neo4j_session,
-            trail_data,
+            trails,
             region,
             current_aws_account_id,
             update_tag,

cartography/intel/aws/cloudwatch.py

@@ -22,7 +22,7 @@ def get_cloudwatch_log_groups(
     boto3_session: boto3.Session, region: str
 ) -> List[Dict[str, Any]]:
     client = boto3_session.client(
-        "cloudwatch", region_name=region, config=get_botocore_config()
+        "logs", region_name=region, config=get_botocore_config()
     )
     paginator = client.get_paginator("describe_log_groups")
     logGroups = []

cartography/intel/aws/ec2/load_balancer_v2s.py

@@ -99,7 +99,10 @@ def load_load_balancer_v2s(
     SET r.lastupdated = $update_tag
     """
     for lb in data:
-        load_balancer_id = lb["DNSName"]
+        load_balancer_id = lb.get("DNSName")
+        if not load_balancer_id:
+            logger.warning("Skipping load balancer entry with missing DNSName: %r", lb)
+            continue
 
         neo4j_session.run(
             ingest_load_balancer_v2,

cartography/intel/aws/resources.py

@@ -10,7 +10,6 @@ from . import config
 from . import dynamodb
 from . import ecr
 from . import ecs
-from . import efs
 from . import eks
 from . import elasticache
 from . import elasticsearch
@@ -105,5 +104,4 @@ RESOURCE_FUNCTIONS: Dict[str, Callable[..., None]] = {
     "identitycenter": identitycenter.sync_identity_center_instances,
     "cloudtrail": cloudtrail.sync,
     "cloudwatch": cloudwatch.sync,
-    "efs": efs.sync,
 }

cartography/intel/aws/secretsmanager.py

@@ -5,12 +5,20 @@ from typing import List
 import boto3
 import neo4j
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.secretsmanager.secret_version import (
+    SecretsManagerSecretVersionSchema,
+)
+from cartography.stats import get_stats_client
 from cartography.util import aws_handle_regions
 from cartography.util import dict_date_to_epoch
+from cartography.util import merge_module_sync_metadata
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
+stat_handler = get_stats_client(__name__)
 
 
 @timeit
@@ -76,6 +84,107 @@ def cleanup_secrets(neo4j_session: neo4j.Session, common_job_parameters: Dict) -
     )
 
 
+@timeit
+@aws_handle_regions
+def get_secret_versions(
+    boto3_session: boto3.session.Session, region: str, secret_arn: str
+) -> List[Dict]:
+    """
+    Get all versions of a secret from AWS Secrets Manager.
+
+    Note: list_secret_version_ids is not paginatable through boto3's paginator,
+    so we implement manual pagination.
+    """
+    client = boto3_session.client("secretsmanager", region_name=region)
+    next_token = None
+    versions = []
+
+    while True:
+        params = {"SecretId": secret_arn, "IncludeDeprecated": True}
+        if next_token:
+            params["NextToken"] = next_token
+
+        response = client.list_secret_version_ids(**params)
+
+        for version in response.get("Versions", []):
+            version["SecretId"] = secret_arn
+            version["ARN"] = f"{secret_arn}:version:{version['VersionId']}"
+
+        versions.extend(response.get("Versions", []))
+
+        next_token = response.get("NextToken")
+        if not next_token:
+            break
+
+    return versions
+
+
+def transform_secret_versions(
+    versions: List[Dict],
+    region: str,
+    aws_account_id: str,
+) -> List[Dict]:
+    """
+    Transform AWS Secrets Manager Secret Versions to match the data model.
+    """
+    transformed_data = []
+    for version in versions:
+        transformed = {
+            "ARN": version["ARN"],
+            "SecretId": version["SecretId"],
+            "VersionId": version["VersionId"],
+            "VersionStages": version.get("VersionStages"),
+            "CreatedDate": dict_date_to_epoch(version, "CreatedDate"),
+        }
+
+        if "KmsKeyId" in version and version["KmsKeyId"]:
+            transformed["KmsKeyId"] = version["KmsKeyId"]
+
+        if "Tags" in version and version["Tags"]:
+            transformed["Tags"] = version["Tags"]
+
+        transformed_data.append(transformed)
+
+    return transformed_data
+
+
+@timeit
+def load_secret_versions(
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    region: str,
+    aws_account_id: str,
+    update_tag: int,
+) -> None:
+    """
+    Load secret versions into Neo4j using the data model.
+    """
+    logger.info(f"Loading {len(data)} Secret Versions for region {region} into graph.")
+
+    load(
+        neo4j_session,
+        SecretsManagerSecretVersionSchema(),
+        data,
+        lastupdated=update_tag,
+        Region=region,
+        AWS_ID=aws_account_id,
+    )
+
+
+@timeit
+def cleanup_secret_versions(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict
+) -> None:
+    """
+    Run Secret Versions cleanup job.
+    """
+    logger.debug("Running Secret Versions cleanup job.")
+    cleanup_job = GraphJob.from_node_schema(
+        SecretsManagerSecretVersionSchema(), common_job_parameters
+    )
+    cleanup_job.run(neo4j_session)
+
+
 @timeit
 def sync(
     neo4j_session: neo4j.Session,
@@ -85,12 +194,50 @@ def sync(
     update_tag: int,
     common_job_parameters: Dict,
 ) -> None:
+    """
+    Sync AWS Secrets Manager resources.
+    """
     for region in regions:
         logger.info(
-            "Syncing Secrets Manager for region '%s' in account '%s'.",
-            region,
-            current_aws_account_id,
+            f"Syncing Secrets Manager for region '{region}' in account '{current_aws_account_id}'."
         )
         secrets = get_secret_list(boto3_session, region)
+
         load_secrets(neo4j_session, secrets, region, current_aws_account_id, update_tag)
+
+        all_versions = []
+        for secret in secrets:
+            logger.info(
+                f"Getting versions for secret {secret.get('Name', 'unnamed')} ({secret['ARN']})"
+            )
+            versions = get_secret_versions(boto3_session, region, secret["ARN"])
+            logger.info(
+                f"Found {len(versions)} versions for secret {secret.get('Name', 'unnamed')}"
+            )
+            all_versions.extend(versions)
+
+        transformed_data = transform_secret_versions(
+            all_versions,
+            region,
+            current_aws_account_id,
+        )
+
+        load_secret_versions(
+            neo4j_session,
+            transformed_data,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
     cleanup_secrets(neo4j_session, common_job_parameters)
+    cleanup_secret_versions(neo4j_session, common_job_parameters)
+
+    merge_module_sync_metadata(
+        neo4j_session,
+        group_type="AWSAccount",
+        group_id=current_aws_account_id,
+        synced_type="SecretsManagerSecretVersion",
+        update_tag=update_tag,
+        stat_handler=stat_handler,
+    )

cartography/intel/aws/ssm.py

@@ -1,4 +1,6 @@
+import json
 import logging
+import re
 from typing import Any
 from typing import Dict
 from typing import List
@@ -10,6 +12,7 @@ from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
 from cartography.models.aws.ssm.instance_information import SSMInstanceInformationSchema
 from cartography.models.aws.ssm.instance_patch import SSMInstancePatchSchema
+from cartography.models.aws.ssm.parameters import SSMParameterSchema
 from cartography.util import aws_handle_regions
 from cartography.util import dict_date_to_epoch
 from cartography.util import timeit
@@ -107,6 +110,42 @@ def transform_instance_patches(data_list: List[Dict[str, Any]]) -> List[Dict[str
     return data_list
 
 
+@timeit
+@aws_handle_regions
+def get_ssm_parameters(
+    boto3_session: boto3.session.Session,
+    region: str,
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client("ssm", region_name=region)
+    paginator = client.get_paginator("describe_parameters")
+    ssm_parameters_data: List[Dict[str, Any]] = []
+    for page in paginator.paginate(PaginationConfig={"PageSize": 50}):
+        ssm_parameters_data.extend(page.get("Parameters", []))
+    return ssm_parameters_data
+
+
+def transform_ssm_parameters(
+    raw_parameters_data: List[Dict[str, Any]],
+) -> List[Dict[str, Any]]:
+    transformed_list: List[Dict[str, Any]] = []
+    for param in raw_parameters_data:
+        param["LastModifiedDate"] = dict_date_to_epoch(param, "LastModifiedDate")
+        param["PoliciesJson"] = json.dumps(param.get("Policies", []))
+        # KMSKey uses shorter UUID as their primary id
+        # SSM Parameters, when encrypted, reference KMS keys using their full ARNs in the KeyId field
+        # Adding a param to match on the id property of the target node
+        if param.get("Type") == "SecureString" and param.get("KeyId") is not None:
+            match = re.match(r".*key/(.*)$", param["KeyId"])
+            if match:
+                param["KMSKeyIdShort"] = match.group(1)
+            else:
+                param["KMSKeyIdShort"] = None
+        else:
+            param["KMSKeyIdShort"] = None
+        transformed_list.append(param)
+    return transformed_list
+
+
 @timeit
 def load_instance_information(
     neo4j_session: neo4j.Session,
@@ -143,6 +182,24 @@ def load_instance_patches(
     )
 
 
+@timeit
+def load_ssm_parameters(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        SSMParameterSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
 @timeit
 def cleanup_ssm(
     neo4j_session: neo4j.Session,
@@ -156,6 +213,9 @@ def cleanup_ssm(
     GraphJob.from_node_schema(SSMInstancePatchSchema(), common_job_parameters).run(
         neo4j_session,
     )
+    GraphJob.from_node_schema(SSMParameterSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
 
 
 @timeit
@@ -193,4 +253,15 @@ def sync(
             current_aws_account_id,
             update_tag,
         )
+
+        data = get_ssm_parameters(boto3_session, region)
+        data = transform_ssm_parameters(data)
+        load_ssm_parameters(
+            neo4j_session,
+            data,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
     cleanup_ssm(neo4j_session, common_job_parameters)

cartography/intel/entra/ou.py

@@ -22,12 +22,28 @@ async def get_entra_ous(client: GraphServiceClient) -> list[AdministrativeUnit]:
     Get all OUs from Microsoft Graph API with pagination support
     """
     all_units: list[AdministrativeUnit] = []
-    request = client.directory.administrative_units.request()
 
-    while request:
-        response = await request.get()
-        all_units.extend(response.value)
-        request = response.odata_next_link if response.odata_next_link else None
+    # Initialize first page request
+    current_request = client.directory.administrative_units
+
+    while current_request:
+        try:
+            response = await current_request.get()
+            if response and response.value:
+                all_units.extend(response.value)
+
+                # Handle next page using OData link
+                if response.odata_next_link:
+                    current_request = client.directory.administrative_units.with_url(
+                        response.odata_next_link
+                    )
+                else:
+                    current_request = None
+            else:
+                current_request = None
+        except Exception as e:
+            logger.error(f"Failed to retrieve administrative units: {str(e)}")
+            current_request = None
 
     return all_units
 

cartography/intel/openai/adminapikeys.py

@@ -23,7 +23,7 @@ def sync(
     api_session: requests.Session,
     common_job_parameters: Dict[str, Any],
     ORG_ID: str,
-) -> List[Dict]:
+) -> None:
     adminapikeys = get(
         api_session,
         common_job_parameters["BASE_URL"],
@@ -36,7 +36,6 @@ def sync(
         common_job_parameters["UPDATE_TAG"],
     )
     cleanup(neo4j_session, common_job_parameters)
-    return adminapikeys
 
 
 @timeit

cartography/intel/openai/apikeys.py

@@ -77,7 +77,7 @@ def load_apikeys(
     project_id: str,
     update_tag: int,
 ) -> None:
-    logger.info("Loading %d OpenAI Project APIKey into Neo4j.", len(data))
+    logger.info("Loading %d OpenAI APIKey into Neo4j.", len(data))
     load(
         neo4j_session,
         OpenAIApiKeySchema(),

cartography/intel/openai/projects.py

@@ -30,12 +30,15 @@ def sync(
     )
     for project in projects:
         project["users"] = []
+        project["admins"] = []
         for user in get_project_users(
             api_session,
             common_job_parameters["BASE_URL"],
             project["id"],
         ):
             project["users"].append(user["id"])
+            if user["role"] == "owner":
+                project["admins"].append(user["id"])
     load_projects(neo4j_session, projects, ORG_ID, common_job_parameters["UPDATE_TAG"])
     cleanup(neo4j_session, common_job_parameters)
     return projects
@@ -75,7 +78,7 @@ def load_projects(
     ORG_ID: str,
     update_tag: int,
 ) -> None:
-    logger.info("Loading %d OpenAIProjectSchema into Neo4j.", len(data))
+    logger.info("Loading %d OpenAI Projects into Neo4j.", len(data))
     load(
         neo4j_session,
         OpenAIProjectSchema(),

cartography/intel/openai/serviceaccounts.py

@@ -63,7 +63,7 @@ def load_serviceaccounts(
     project_id: str,
     update_tag: int,
 ) -> None:
-    logger.info("Loading %d OpenAI ProjectServiceAccount into Neo4j.", len(data))
+    logger.info("Loading %d OpenAI ServiceAccount into Neo4j.", len(data))
     load(
         neo4j_session,
         OpenAIServiceAccountSchema(),

cartography/intel/openai/users.py

@@ -70,9 +70,6 @@ def load_users(
 def cleanup(
     neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
 ) -> None:
-    GraphJob.from_node_schema(OpenAIOrganizationSchema(), common_job_parameters).run(
-        neo4j_session
-    )
     GraphJob.from_node_schema(OpenAIUserSchema(), common_job_parameters).run(
         neo4j_session
     )

cartography/intel/openai/util.py

@@ -10,7 +10,23 @@ def paginated_get(
     timeout: tuple[int, int],
     after: str | None = None,
 ) -> Generator[dict[str, Any], None, None]:
-    # DOC
+    """Helper function to get paginated data from the OpenAI API.
+
+    This function handles the pagination of the API requests and returns
+    the results as a generator. It will continue to make requests until
+    all pages of data have been retrieved. The results are returned as a
+    list of dictionaries, where each dictionary represents a single
+    entity.
+
+    Args:
+        api_session (requests.Session): The requests session to use for making API calls.
+        url (str): The URL to make the API call to.
+        timeout (tuple[int, int]): The timeout for the API call.
+        after (str | None): The ID of the last item retrieved in the previous request.
+            If None, the first page of results will be retrieved.
+    Returns:
+        Generator[dict[str, Any], None, None]: A generator yielding dictionaries representing the results.
+    """
     params = {"after": after} if after else {}
     req = api_session.get(
         url,