cartography 0.103.0rc1__py3-none-any.whl → 0.104.0rc2__py3-none-any.whl

cartography/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.103.0rc1'
-__version_tuple__ = version_tuple = (0, 103, 0, 'rc1')
+__version__ = version = '0.104.0rc2'
+__version_tuple__ = version_tuple = (0, 104, 0, 'rc2')

cartography/cli.py

@@ -561,7 +561,7 @@ class CLI:
             type=str,
             default=None,
             help=(
-                "Your SnipeIT base URI"
+                "Your SnipeIT base URI. "
                 "Required if you are using the SnipeIT intel module. Ignored otherwise."
             ),
         )
@@ -577,6 +577,66 @@ class CLI:
             default=None,
             help="An ID for the SnipeIT tenant.",
         )
+        parser.add_argument(
+            "--cloudflare-token-env-var",
+            type=str,
+            default=None,
+            help="The name of an environment variable containing ApiKey with which to authenticate to Cloudflare.",
+        )
+        parser.add_argument(
+            "--tailscale-token-env-var",
+            type=str,
+            default=None,
+            help=(
+                "The name of an environment variable containing a Tailscale API token. "
+                "Required if you are using the Tailscale intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--tailscale-org",
+            type=str,
+            default=None,
+            help=(
+                "The name of the Tailscale organization to sync. "
+                "Required if you are using the Tailscale intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--tailscale-base-url",
+            type=str,
+            default="https://api.tailscale.com/api/v2",
+            help=(
+                "The base URL for the Tailscale API. "
+                "Required if you are using the Tailscale intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--openai-apikey-env-var",
+            type=str,
+            default=None,
+            help=(
+                "The name of an environment variable containing a OpenAI API Key. "
+                "Required if you are using the OpenAI intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--openai-org-id",
+            type=str,
+            default=None,
+            help=(
+                "The ID of the OpenAI organization to sync. "
+                "Required if you are using the OpenAI intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--anthropic-apikey-env-var",
+            type=str,
+            default=None,
+            help=(
+                "The name of an environment variable containing an Anthropic API Key. "
+                "Required if you are using the Anthropic intel module. Ignored otherwise."
+            ),
+        )
 
         return parser
 
@@ -859,6 +919,42 @@ class CLI:
             logger.warning("A SnipeIT base URI was not provided.")
             config.snipeit_base_uri = None
 
+        # Tailscale config
+        if config.tailscale_token_env_var:
+            logger.debug(
+                f"Reading Tailscale API token from environment variable {config.tailscale_token_env_var}",
+            )
+            config.tailscale_token = os.environ.get(config.tailscale_token_env_var)
+        else:
+            config.tailscale_token = None
+
+        # Cloudflare config
+        if config.cloudflare_token_env_var:
+            logger.debug(
+                f"Reading Cloudflare ApiKey from environment variable {config.cloudflare_token_env_var}",
+            )
+            config.cloudflare_token = os.environ.get(config.cloudflare_token_env_var)
+        else:
+            config.cloudflare_token = None
+
+        # OpenAI config
+        if config.openai_apikey_env_var:
+            logger.debug(
+                f"Reading OpenAI API key from environment variable {config.openai_apikey_env_var}",
+            )
+            config.openai_apikey = os.environ.get(config.openai_apikey_env_var)
+        else:
+            config.openai_apikey = None
+
+        # Anthropic config
+        if config.anthropic_apikey_env_var:
+            logger.debug(
+                f"Reading Anthropic API key from environment variable {config.anthropic_apikey_env_var}",
+            )
+            config.anthropic_apikey = os.environ.get(config.anthropic_apikey_env_var)
+        else:
+            config.anthropic_apikey = None
+
         # Run cartography
         try:
             return cartography.sync.run_with_config(self.sync, config)

cartography/config.py

@@ -123,6 +123,20 @@ class Config:
     :param snipeit_token: Token used to authenticate to the SnipeIT data provider. Optional.
     :type snipeit_tenant_id: string
     :param snipeit_tenant_id: Token used to authenticate to the SnipeIT data provider. Optional.
+    :type tailscale_token: str
+    :param tailscale_token: Tailscale API token. Optional.
+    :type tailscale_org: str
+    :param tailscale_org: Tailscale organization name. Optional.
+    :type tailscale_base_url: str
+    :param tailscale_base_url: Tailscale API base URL. Optional.
+    :type cloudflare_token: string
+    :param cloudflare_token: Cloudflare API key. Optional.
+    :type openai_apikey: string
+    :param openai_apikey: OpenAI API key. Optional.
+    :type openai_org_id: string
+    :param openai_org_id: OpenAI organization id. Optional.
+    :type anthropic_apikey: string
+    :param anthropic_apikey: Anthropic API key. Optional.
     """
 
     def __init__(
@@ -188,6 +202,13 @@ class Config:
         snipeit_base_uri=None,
         snipeit_token=None,
         snipeit_tenant_id=None,
+        tailscale_token=None,
+        tailscale_org=None,
+        tailscale_base_url=None,
+        cloudflare_token=None,
+        openai_apikey=None,
+        openai_org_id=None,
+        anthropic_apikey=None,
     ):
         self.neo4j_uri = neo4j_uri
         self.neo4j_user = neo4j_user
@@ -250,3 +271,10 @@ class Config:
         self.snipeit_base_uri = snipeit_base_uri
         self.snipeit_token = snipeit_token
         self.snipeit_tenant_id = snipeit_tenant_id
+        self.tailscale_token = tailscale_token
+        self.tailscale_org = tailscale_org
+        self.tailscale_base_url = tailscale_base_url
+        self.cloudflare_token = cloudflare_token
+        self.openai_apikey = openai_apikey
+        self.openai_org_id = openai_org_id
+        self.anthropic_apikey = anthropic_apikey

cartography/intel/anthropic/__init__.py

@@ -0,0 +1,62 @@
+import logging
+
+import neo4j
+import requests
+
+import cartography.intel.anthropic.apikeys
+import cartography.intel.anthropic.users
+import cartography.intel.anthropic.workspaces
+from cartography.config import Config
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def start_anthropic_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of Anthropic data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+
+    if not config.anthropic_apikey:
+        logger.info(
+            "Anthropic import is not configured - skipping this module. "
+            "See docs to configure.",
+        )
+        return
+
+    # Create requests sessions
+    api_session = requests.session()
+    api_session.headers.update(
+        {
+            "X-Api-Key": config.anthropic_apikey,
+            "anthropic-version": "2023-06-01",
+        }
+    )
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+        "BASE_URL": "https://api.anthropic.com/v1",
+    }
+
+    # Organization node is created during the users sync
+    cartography.intel.anthropic.users.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+    )
+
+    cartography.intel.anthropic.workspaces.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+    )
+
+    cartography.intel.anthropic.apikeys.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+    )

cartography/intel/anthropic/apikeys.py

@@ -0,0 +1,72 @@
+import logging
+from typing import Any
+from typing import Tuple
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.anthropic.util import paginated_get
+from cartography.models.anthropic.apikey import AnthropicApiKeySchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    org_id, apikeys = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+    )
+    common_job_parameters["ORG_ID"] = org_id
+    load_apikeys(
+        neo4j_session,
+        apikeys,
+        org_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+) -> Tuple[str, list[dict[str, Any]]]:
+    return paginated_get(
+        api_session, f"{base_url}/organizations/api_keys", timeout=_TIMEOUT
+    )
+
+
+@timeit
+def load_apikeys(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    ORG_ID: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Anthropic ApiKey into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        AnthropicApiKeySchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=ORG_ID,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(AnthropicApiKeySchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/anthropic/users.py

@@ -0,0 +1,75 @@
+import logging
+from typing import Any
+from typing import Tuple
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.anthropic.util import paginated_get
+from cartography.models.anthropic.organization import AnthropicOrganizationSchema
+from cartography.models.anthropic.user import AnthropicUserSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: dict[str, Any],
+) -> str:
+    org_id, users = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+    )
+    common_job_parameters["ORG_ID"] = org_id
+    load_users(neo4j_session, users, org_id, common_job_parameters["UPDATE_TAG"])
+    cleanup(neo4j_session, common_job_parameters)
+    return org_id
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+) -> Tuple[str, list[dict[str, Any]]]:
+    return paginated_get(
+        api_session, f"{base_url}/organizations/users", timeout=_TIMEOUT
+    )
+
+
+@timeit
+def load_users(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    ORG_ID: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        AnthropicOrganizationSchema(),
+        [{"id": ORG_ID}],
+        lastupdated=update_tag,
+    )
+    logger.info("Loading %d Anthropic User into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        AnthropicUserSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=ORG_ID,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(AnthropicUserSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/anthropic/util.py

@@ -0,0 +1,51 @@
+from typing import Any
+
+import requests
+
+
+def paginated_get(
+    api_session: requests.Session,
+    url: str,
+    timeout: tuple[int, int],
+    after: str | None = None,
+) -> tuple[str, list[dict[str, Any]]]:
+    """Helper function to get paginated data from the Anthropic API.
+
+    This function handles the pagination of the API requests and returns
+    the results in a list. It also retrieves the organization ID from the
+    response headers. The function will continue to make requests until
+    all pages of data have been retrieved. The results are returned as a
+    list of dictionaries, where each dictionary represents a single
+    entity.
+
+    Args:
+        api_session (requests.Session): The requests session to use for making API calls.
+        url (str): The URL to make the API call to.
+        timeout (tuple[int, int]): The timeout for the API call.
+        after (str | None): The ID of the last item retrieved in the previous request.
+            If None, the first page of results will be retrieved.
+    Returns:
+        tuple[str, list[dict[str, Any]]]: A tuple containing the organization ID and a list of
+            dictionaries representing the results.
+    """
+    results: list[dict[str, Any]] = []
+    params = {"after_id": after} if after else {}
+    req = api_session.get(
+        url,
+        params=params,
+        timeout=timeout,
+    )
+    req.raise_for_status()
+    # Get organization_id from the headers
+    organization_id = req.headers.get("anthropic-organization-id", "")
+    result = req.json()
+    results.extend(result.get("data", []))
+    if result.get("has_more"):
+        _, next_results = paginated_get(
+            api_session,
+            url,
+            timeout=timeout,
+            after=result.get("last_id"),
+        )
+        results.extend(next_results)
+    return organization_id, results

cartography/intel/anthropic/workspaces.py

@@ -0,0 +1,95 @@
+import logging
+from typing import Any
+from typing import Tuple
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.anthropic.util import paginated_get
+from cartography.models.anthropic.workspace import AnthropicWorkspaceSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: dict[str, Any],
+) -> list[dict]:
+    org_id, workspaces = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+    )
+    common_job_parameters["ORG_ID"] = org_id
+    for workspace in workspaces:
+        workspace["users"] = []
+        workspace["admins"] = []
+        for user in get_workspace_users(
+            api_session,
+            common_job_parameters["BASE_URL"],
+            workspace["id"],
+        ):
+            workspace["users"].append(user["user_id"])
+            if user["workspace_role"] == "workspace_admin":
+                workspace["admins"].append(user["user_id"])
+    load_workspaces(
+        neo4j_session, workspaces, org_id, common_job_parameters["UPDATE_TAG"]
+    )
+    cleanup(neo4j_session, common_job_parameters)
+    return workspaces
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+) -> Tuple[str, list[dict[str, Any]]]:
+    return paginated_get(
+        api_session, f"{base_url}/organizations/workspaces", timeout=_TIMEOUT
+    )
+
+
+@timeit
+def get_workspace_users(
+    api_session: requests.Session,
+    base_url: str,
+    workspace_id: str,
+) -> list[dict[str, Any]]:
+    _, result = paginated_get(
+        api_session,
+        f"{base_url}/organizations/workspaces/{workspace_id}/members",
+        timeout=_TIMEOUT,
+    )
+    return result
+
+
+@timeit
+def load_workspaces(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    ORG_ID: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Anthropic workspaces into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        AnthropicWorkspaceSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=ORG_ID,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(AnthropicWorkspaceSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/aws/cloudwatch.py

@@ -0,0 +1,93 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.cloudwatch.loggroup import CloudWatchLogGroupSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_cloudwatch_log_groups(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "logs", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("describe_log_groups")
+    logGroups = []
+    for page in paginator.paginate():
+        logGroups.extend(page["logGroups"])
+    return logGroups
+
+
+@timeit
+def load_cloudwatch_log_groups(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading CloudWatch {len(data)} log groups for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        CloudWatchLogGroupSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.debug("Running CloudWatch cleanup job.")
+    cleanup_job = GraphJob.from_node_schema(
+        CloudWatchLogGroupSchema(), common_job_parameters
+    )
+    cleanup_job.run(neo4j_session)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing CloudWatch for region '{region}' in account '{current_aws_account_id}'.",
+        )
+        logGroups = get_cloudwatch_log_groups(boto3_session, region)
+        group_data: List[Dict[str, Any]] = []
+        for logGroup in logGroups:
+            group_data.append(logGroup)
+
+        load_cloudwatch_log_groups(
+            neo4j_session,
+            group_data,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/ec2/load_balancer_v2s.py

@@ -99,7 +99,10 @@ def load_load_balancer_v2s(
     SET r.lastupdated = $update_tag
     """
     for lb in data:
-        load_balancer_id = lb["DNSName"]
+        load_balancer_id = lb.get("DNSName")
+        if not load_balancer_id:
+            logger.warning("Skipping load balancer entry with missing DNSName: %r", lb)
+            continue
 
         neo4j_session.run(
             ingest_load_balancer_v2,