cartography 0.103.0__py3-none-any.whl → 0.104.0rc1__py3-none-any.whl

cartography/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.103.0'
-__version_tuple__ = version_tuple = (0, 103, 0)
+__version__ = version = '0.104.0rc1'
+__version_tuple__ = version_tuple = (0, 104, 0, 'rc1')

cartography/cli.py

@@ -561,7 +561,7 @@ class CLI:
             type=str,
             default=None,
             help=(
-                "Your SnipeIT base URI"
+                "Your SnipeIT base URI. "
                 "Required if you are using the SnipeIT intel module. Ignored otherwise."
             ),
         )
@@ -588,7 +588,7 @@ class CLI:
             type=str,
             default=None,
             help=(
-                "The name of an environment variable containing a Tailscale API token."
+                "The name of an environment variable containing a Tailscale API token. "
                 "Required if you are using the Tailscale intel module. Ignored otherwise."
             ),
         )
@@ -615,7 +615,7 @@ class CLI:
             type=str,
             default=None,
             help=(
-                "The name of an environment variable containing a OpenAI API Key."
+                "The name of an environment variable containing a OpenAI API Key. "
                 "Required if you are using the OpenAI intel module. Ignored otherwise."
             ),
         )
@@ -628,6 +628,15 @@ class CLI:
                 "Required if you are using the OpenAI intel module. Ignored otherwise."
             ),
         )
+        parser.add_argument(
+            "--anthropic-apikey-env-var",
+            type=str,
+            default=None,
+            help=(
+                "The name of an environment variable containing an Anthropic API Key. "
+                "Required if you are using the Anthropic intel module. Ignored otherwise."
+            ),
+        )
 
         return parser
 
@@ -937,6 +946,15 @@ class CLI:
         else:
             config.openai_apikey = None
 
+        # Anthropic config
+        if config.anthropic_apikey_env_var:
+            logger.debug(
+                f"Reading Anthropic API key from environment variable {config.anthropic_apikey_env_var}",
+            )
+            config.anthropic_apikey = os.environ.get(config.anthropic_apikey_env_var)
+        else:
+            config.anthropic_apikey = None
+
         # Run cartography
         try:
             return cartography.sync.run_with_config(self.sync, config)

cartography/config.py

@@ -135,6 +135,8 @@ class Config:
     :param openai_apikey: OpenAI API key. Optional.
     :type openai_org_id: string
     :param openai_org_id: OpenAI organization id. Optional.
+    :type anthropic_apikey: string
+    :param anthropic_apikey: Anthropic API key. Optional.
     """
 
     def __init__(
@@ -206,6 +208,7 @@ class Config:
         cloudflare_token=None,
         openai_apikey=None,
         openai_org_id=None,
+        anthropic_apikey=None,
     ):
         self.neo4j_uri = neo4j_uri
         self.neo4j_user = neo4j_user
@@ -274,3 +277,4 @@ class Config:
         self.cloudflare_token = cloudflare_token
         self.openai_apikey = openai_apikey
         self.openai_org_id = openai_org_id
+        self.anthropic_apikey = anthropic_apikey

cartography/intel/anthropic/__init__.py

@@ -0,0 +1,62 @@
+import logging
+
+import neo4j
+import requests
+
+import cartography.intel.anthropic.apikeys
+import cartography.intel.anthropic.users
+import cartography.intel.anthropic.workspaces
+from cartography.config import Config
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def start_anthropic_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of Anthropic data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+
+    if not config.anthropic_apikey:
+        logger.info(
+            "Anthropic import is not configured - skipping this module. "
+            "See docs to configure.",
+        )
+        return
+
+    # Create requests sessions
+    api_session = requests.session()
+    api_session.headers.update(
+        {
+            "X-Api-Key": config.anthropic_apikey,
+            "anthropic-version": "2023-06-01",
+        }
+    )
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+        "BASE_URL": "https://api.anthropic.com/v1",
+    }
+
+    # Organization node is created during the users sync
+    cartography.intel.anthropic.users.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+    )
+
+    cartography.intel.anthropic.workspaces.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+    )
+
+    cartography.intel.anthropic.apikeys.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+    )

cartography/intel/anthropic/apikeys.py

@@ -0,0 +1,72 @@
+import logging
+from typing import Any
+from typing import Tuple
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.anthropic.util import paginated_get
+from cartography.models.anthropic.apikey import AnthropicApiKeySchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    org_id, apikeys = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+    )
+    common_job_parameters["ORG_ID"] = org_id
+    load_apikeys(
+        neo4j_session,
+        apikeys,
+        org_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+) -> Tuple[str, list[dict[str, Any]]]:
+    return paginated_get(
+        api_session, f"{base_url}/organizations/api_keys", timeout=_TIMEOUT
+    )
+
+
+@timeit
+def load_apikeys(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    ORG_ID: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Anthropic ApiKey into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        AnthropicApiKeySchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=ORG_ID,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(AnthropicApiKeySchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/anthropic/users.py

@@ -0,0 +1,75 @@
+import logging
+from typing import Any
+from typing import Tuple
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.anthropic.util import paginated_get
+from cartography.models.anthropic.organization import AnthropicOrganizationSchema
+from cartography.models.anthropic.user import AnthropicUserSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: dict[str, Any],
+) -> str:
+    org_id, users = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+    )
+    common_job_parameters["ORG_ID"] = org_id
+    load_users(neo4j_session, users, org_id, common_job_parameters["UPDATE_TAG"])
+    cleanup(neo4j_session, common_job_parameters)
+    return org_id
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+) -> Tuple[str, list[dict[str, Any]]]:
+    return paginated_get(
+        api_session, f"{base_url}/organizations/users", timeout=_TIMEOUT
+    )
+
+
+@timeit
+def load_users(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    ORG_ID: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        AnthropicOrganizationSchema(),
+        [{"id": ORG_ID}],
+        lastupdated=update_tag,
+    )
+    logger.info("Loading %d Anthropic User into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        AnthropicUserSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=ORG_ID,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(AnthropicUserSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/anthropic/util.py

@@ -0,0 +1,51 @@
+from typing import Any
+
+import requests
+
+
+def paginated_get(
+    api_session: requests.Session,
+    url: str,
+    timeout: tuple[int, int],
+    after: str | None = None,
+) -> tuple[str, list[dict[str, Any]]]:
+    """Helper function to get paginated data from the Anthropic API.
+
+    This function handles the pagination of the API requests and returns
+    the results in a list. It also retrieves the organization ID from the
+    response headers. The function will continue to make requests until
+    all pages of data have been retrieved. The results are returned as a
+    list of dictionaries, where each dictionary represents a single
+    entity.
+
+    Args:
+        api_session (requests.Session): The requests session to use for making API calls.
+        url (str): The URL to make the API call to.
+        timeout (tuple[int, int]): The timeout for the API call.
+        after (str | None): The ID of the last item retrieved in the previous request.
+            If None, the first page of results will be retrieved.
+    Returns:
+        tuple[str, list[dict[str, Any]]]: A tuple containing the organization ID and a list of
+            dictionaries representing the results.
+    """
+    results: list[dict[str, Any]] = []
+    params = {"after_id": after} if after else {}
+    req = api_session.get(
+        url,
+        params=params,
+        timeout=timeout,
+    )
+    req.raise_for_status()
+    # Get organization_id from the headers
+    organization_id = req.headers.get("anthropic-organization-id", "")
+    result = req.json()
+    results.extend(result.get("data", []))
+    if result.get("has_more"):
+        _, next_results = paginated_get(
+            api_session,
+            url,
+            timeout=timeout,
+            after=result.get("last_id"),
+        )
+        results.extend(next_results)
+    return organization_id, results

cartography/intel/anthropic/workspaces.py

@@ -0,0 +1,95 @@
+import logging
+from typing import Any
+from typing import Tuple
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.anthropic.util import paginated_get
+from cartography.models.anthropic.workspace import AnthropicWorkspaceSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: dict[str, Any],
+) -> list[dict]:
+    org_id, workspaces = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+    )
+    common_job_parameters["ORG_ID"] = org_id
+    for workspace in workspaces:
+        workspace["users"] = []
+        workspace["admins"] = []
+        for user in get_workspace_users(
+            api_session,
+            common_job_parameters["BASE_URL"],
+            workspace["id"],
+        ):
+            workspace["users"].append(user["user_id"])
+            if user["workspace_role"] == "workspace_admin":
+                workspace["admins"].append(user["user_id"])
+    load_workspaces(
+        neo4j_session, workspaces, org_id, common_job_parameters["UPDATE_TAG"]
+    )
+    cleanup(neo4j_session, common_job_parameters)
+    return workspaces
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+) -> Tuple[str, list[dict[str, Any]]]:
+    return paginated_get(
+        api_session, f"{base_url}/organizations/workspaces", timeout=_TIMEOUT
+    )
+
+
+@timeit
+def get_workspace_users(
+    api_session: requests.Session,
+    base_url: str,
+    workspace_id: str,
+) -> list[dict[str, Any]]:
+    _, result = paginated_get(
+        api_session,
+        f"{base_url}/organizations/workspaces/{workspace_id}/members",
+        timeout=_TIMEOUT,
+    )
+    return result
+
+
+@timeit
+def load_workspaces(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    ORG_ID: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Anthropic workspaces into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        AnthropicWorkspaceSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=ORG_ID,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(AnthropicWorkspaceSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/aws/ec2/load_balancer_v2s.py

@@ -99,7 +99,10 @@ def load_load_balancer_v2s(
     SET r.lastupdated = $update_tag
     """
     for lb in data:
-        load_balancer_id = lb["DNSName"]
+        load_balancer_id = lb.get("DNSName")
+        if not load_balancer_id:
+            logger.warning("Skipping load balancer entry with missing DNSName: %r", lb)
+            continue
 
         neo4j_session.run(
             ingest_load_balancer_v2,

cartography/intel/aws/secretsmanager.py

@@ -5,12 +5,20 @@ from typing import List
 import boto3
 import neo4j
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.secretsmanager.secret_version import (
+    SecretsManagerSecretVersionSchema,
+)
+from cartography.stats import get_stats_client
 from cartography.util import aws_handle_regions
 from cartography.util import dict_date_to_epoch
+from cartography.util import merge_module_sync_metadata
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
+stat_handler = get_stats_client(__name__)
 
 
 @timeit
@@ -76,6 +84,93 @@ def cleanup_secrets(neo4j_session: neo4j.Session, common_job_parameters: Dict) -
     )
 
 
+@timeit
+@aws_handle_regions
+def get_secret_versions(
+    boto3_session: boto3.session.Session, region: str, secret_arn: str
+) -> List[Dict]:
+    """
+    Get all versions of a secret from AWS Secrets Manager.
+    """
+    client = boto3_session.client("secretsmanager", region_name=region)
+    paginator = client.get_paginator("list_secret_version_ids")
+    versions = []
+
+    for page in paginator.paginate(SecretId=secret_arn, IncludeDeprecated=True):
+        for version in page["Versions"]:
+            version["SecretId"] = secret_arn
+            version["ARN"] = f"{secret_arn}:version:{version['VersionId']}"
+        versions.extend(page["Versions"])
+
+    return versions
+
+
+def transform_secret_versions(
+    versions: List[Dict],
+    region: str,
+    aws_account_id: str,
+) -> List[Dict]:
+    """
+    Transform AWS Secrets Manager Secret Versions to match the data model.
+    """
+    transformed_data = []
+    for version in versions:
+        transformed = {
+            "ARN": version["ARN"],
+            "SecretId": version["SecretId"],
+            "VersionId": version["VersionId"],
+            "VersionStages": version["VersionStages"],
+            "CreatedDate": dict_date_to_epoch(version, "CreatedDate"),
+        }
+
+        if "KmsKeyId" in version and version["KmsKeyId"]:
+            transformed["KmsKeyId"] = version["KmsKeyId"]
+
+        if "Tags" in version and version["Tags"]:
+            transformed["Tags"] = version["Tags"]
+
+        transformed_data.append(transformed)
+
+    return transformed_data
+
+
+@timeit
+def load_secret_versions(
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    region: str,
+    aws_account_id: str,
+    update_tag: int,
+) -> None:
+    """
+    Load secret versions into Neo4j using the data model.
+    """
+    logger.info(f"Loading {len(data)} Secret Versions for region {region} into graph.")
+
+    load(
+        neo4j_session,
+        SecretsManagerSecretVersionSchema(),
+        data,
+        lastupdated=update_tag,
+        Region=region,
+        AWS_ID=aws_account_id,
+    )
+
+
+@timeit
+def cleanup_secret_versions(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict
+) -> None:
+    """
+    Run Secret Versions cleanup job.
+    """
+    logger.debug("Running Secret Versions cleanup job.")
+    cleanup_job = GraphJob.from_node_schema(
+        SecretsManagerSecretVersionSchema(), common_job_parameters
+    )
+    cleanup_job.run(neo4j_session)
+
+
 @timeit
 def sync(
     neo4j_session: neo4j.Session,
@@ -85,12 +180,50 @@ def sync(
     update_tag: int,
     common_job_parameters: Dict,
 ) -> None:
+    """
+    Sync AWS Secrets Manager resources.
+    """
     for region in regions:
         logger.info(
-            "Syncing Secrets Manager for region '%s' in account '%s'.",
-            region,
-            current_aws_account_id,
+            f"Syncing Secrets Manager for region '{region}' in account '{current_aws_account_id}'."
         )
         secrets = get_secret_list(boto3_session, region)
+
         load_secrets(neo4j_session, secrets, region, current_aws_account_id, update_tag)
+
+        all_versions = []
+        for secret in secrets:
+            logger.info(
+                f"Getting versions for secret {secret.get('Name', 'unnamed')} ({secret['ARN']})"
+            )
+            versions = get_secret_versions(boto3_session, region, secret["ARN"])
+            logger.info(
+                f"Found {len(versions)} versions for secret {secret.get('Name', 'unnamed')}"
+            )
+            all_versions.extend(versions)
+
+        transformed_data = transform_secret_versions(
+            all_versions,
+            region,
+            current_aws_account_id,
+        )
+
+        load_secret_versions(
+            neo4j_session,
+            transformed_data,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
     cleanup_secrets(neo4j_session, common_job_parameters)
+    cleanup_secret_versions(neo4j_session, common_job_parameters)
+
+    merge_module_sync_metadata(
+        neo4j_session,
+        group_type="AWSAccount",
+        group_id=current_aws_account_id,
+        synced_type="SecretsManagerSecretVersion",
+        update_tag=update_tag,
+        stat_handler=stat_handler,
+    )