cartography 0.103.0rc1__py3-none-any.whl → 0.104.0__py3-none-any.whl

cartography/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.103.0rc1'
-__version_tuple__ = version_tuple = (0, 103, 0, 'rc1')
+__version__ = version = '0.104.0'
+__version_tuple__ = version_tuple = (0, 104, 0)

cartography/cli.py

@@ -561,7 +561,7 @@ class CLI:
             type=str,
             default=None,
             help=(
-                "Your SnipeIT base URI"
+                "Your SnipeIT base URI. "
                 "Required if you are using the SnipeIT intel module. Ignored otherwise."
             ),
         )
@@ -577,6 +577,66 @@ class CLI:
             default=None,
             help="An ID for the SnipeIT tenant.",
         )
+        parser.add_argument(
+            "--cloudflare-token-env-var",
+            type=str,
+            default=None,
+            help="The name of an environment variable containing ApiKey with which to authenticate to Cloudflare.",
+        )
+        parser.add_argument(
+            "--tailscale-token-env-var",
+            type=str,
+            default=None,
+            help=(
+                "The name of an environment variable containing a Tailscale API token. "
+                "Required if you are using the Tailscale intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--tailscale-org",
+            type=str,
+            default=None,
+            help=(
+                "The name of the Tailscale organization to sync. "
+                "Required if you are using the Tailscale intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--tailscale-base-url",
+            type=str,
+            default="https://api.tailscale.com/api/v2",
+            help=(
+                "The base URL for the Tailscale API. "
+                "Required if you are using the Tailscale intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--openai-apikey-env-var",
+            type=str,
+            default=None,
+            help=(
+                "The name of an environment variable containing a OpenAI API Key. "
+                "Required if you are using the OpenAI intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--openai-org-id",
+            type=str,
+            default=None,
+            help=(
+                "The ID of the OpenAI organization to sync. "
+                "Required if you are using the OpenAI intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--anthropic-apikey-env-var",
+            type=str,
+            default=None,
+            help=(
+                "The name of an environment variable containing an Anthropic API Key. "
+                "Required if you are using the Anthropic intel module. Ignored otherwise."
+            ),
+        )
 
         return parser
 
@@ -859,6 +919,42 @@ class CLI:
             logger.warning("A SnipeIT base URI was not provided.")
             config.snipeit_base_uri = None
 
+        # Tailscale config
+        if config.tailscale_token_env_var:
+            logger.debug(
+                f"Reading Tailscale API token from environment variable {config.tailscale_token_env_var}",
+            )
+            config.tailscale_token = os.environ.get(config.tailscale_token_env_var)
+        else:
+            config.tailscale_token = None
+
+        # Cloudflare config
+        if config.cloudflare_token_env_var:
+            logger.debug(
+                f"Reading Cloudflare ApiKey from environment variable {config.cloudflare_token_env_var}",
+            )
+            config.cloudflare_token = os.environ.get(config.cloudflare_token_env_var)
+        else:
+            config.cloudflare_token = None
+
+        # OpenAI config
+        if config.openai_apikey_env_var:
+            logger.debug(
+                f"Reading OpenAI API key from environment variable {config.openai_apikey_env_var}",
+            )
+            config.openai_apikey = os.environ.get(config.openai_apikey_env_var)
+        else:
+            config.openai_apikey = None
+
+        # Anthropic config
+        if config.anthropic_apikey_env_var:
+            logger.debug(
+                f"Reading Anthropic API key from environment variable {config.anthropic_apikey_env_var}",
+            )
+            config.anthropic_apikey = os.environ.get(config.anthropic_apikey_env_var)
+        else:
+            config.anthropic_apikey = None
+
         # Run cartography
         try:
             return cartography.sync.run_with_config(self.sync, config)

cartography/config.py

@@ -123,6 +123,20 @@ class Config:
     :param snipeit_token: Token used to authenticate to the SnipeIT data provider. Optional.
     :type snipeit_tenant_id: string
     :param snipeit_tenant_id: Token used to authenticate to the SnipeIT data provider. Optional.
+    :type tailscale_token: str
+    :param tailscale_token: Tailscale API token. Optional.
+    :type tailscale_org: str
+    :param tailscale_org: Tailscale organization name. Optional.
+    :type tailscale_base_url: str
+    :param tailscale_base_url: Tailscale API base URL. Optional.
+    :type cloudflare_token: string
+    :param cloudflare_token: Cloudflare API key. Optional.
+    :type openai_apikey: string
+    :param openai_apikey: OpenAI API key. Optional.
+    :type openai_org_id: string
+    :param openai_org_id: OpenAI organization id. Optional.
+    :type anthropic_apikey: string
+    :param anthropic_apikey: Anthropic API key. Optional.
     """
 
     def __init__(
@@ -188,6 +202,13 @@ class Config:
         snipeit_base_uri=None,
         snipeit_token=None,
         snipeit_tenant_id=None,
+        tailscale_token=None,
+        tailscale_org=None,
+        tailscale_base_url=None,
+        cloudflare_token=None,
+        openai_apikey=None,
+        openai_org_id=None,
+        anthropic_apikey=None,
     ):
         self.neo4j_uri = neo4j_uri
         self.neo4j_user = neo4j_user
@@ -250,3 +271,10 @@ class Config:
         self.snipeit_base_uri = snipeit_base_uri
         self.snipeit_token = snipeit_token
         self.snipeit_tenant_id = snipeit_tenant_id
+        self.tailscale_token = tailscale_token
+        self.tailscale_org = tailscale_org
+        self.tailscale_base_url = tailscale_base_url
+        self.cloudflare_token = cloudflare_token
+        self.openai_apikey = openai_apikey
+        self.openai_org_id = openai_org_id
+        self.anthropic_apikey = anthropic_apikey

cartography/graph/cleanupbuilder.py

@@ -15,33 +15,40 @@ from cartography.models.core.relationships import TargetNodeMatcher
 def build_cleanup_queries(node_schema: CartographyNodeSchema) -> List[str]:
     """
     Generates queries to clean up stale nodes and relationships from the given CartographyNodeSchema.
+    Properly handles cases where a node schema has a scoped cleanup or not.
     Note that auto-cleanups for a node with no relationships is not currently supported.
-
-    Algorithm:
-    1. If node_schema has no relationships at all, return empty.
-
-    Otherwise,
-
-    1. If node_schema doesn't have a sub_resource relationship, generate queries only to clean up its other
-    relationships. No nodes will be cleaned up.
-
-    Otherwise,
-
-    1. First delete all stale nodes attached to the node_schema's sub resource
-    2. Delete all stale node to sub resource relationships
-        - We don't expect this to be very common (never for AWS resources, at least), but in case it is possible for an
-          asset to change sub resources, we want to handle it properly.
-    3. For all relationships defined on the node schema, delete all stale ones.
     :param node_schema: The given CartographyNodeSchema
     :return: A list of Neo4j queries to clean up nodes and relationships.
     """
+    # If the node has no relationships, do not delete the node. Leave this behind for the user to manage.
+    # Oftentimes these are SyncMetadata nodes.
     if (
         not node_schema.sub_resource_relationship
         and not node_schema.other_relationships
     ):
         return []
 
-    if not node_schema.sub_resource_relationship:
+    # Case 1 [Standard]: the node has a sub resource and scoped cleanup is true => clean up stale nodes
+    # of this type, scoped to the sub resource. Continue on to clean up the other_relationships too.
+    if node_schema.sub_resource_relationship and node_schema.scoped_cleanup:
+        queries = _build_cleanup_node_and_rel_queries(
+            node_schema,
+            node_schema.sub_resource_relationship,
+        )
+
+    # Case 2: The node has a sub resource but scoped cleanup is false => this does not make sense
+    # because if have a sub resource, we are implying that we are doing scoped cleanup.
+    elif node_schema.sub_resource_relationship and not node_schema.scoped_cleanup:
+        raise ValueError(
+            f"This is not expected: {node_schema.label} has a sub_resource_relationship but scoped_cleanup=False."
+            "Please check the class definition for this node schema. It doesn't make sense for a node to have a "
+            "sub resource relationship and an unscoped cleanup. Doing this will cause all stale nodes of this type "
+            "to be deleted regardless of the sub resource they are attached to."
+        )
+
+    # Case 3: The node has no sub resource but scoped cleanup is true => do not delete any nodes, but clean up stale relationships.
+    # Return early.
+    elif not node_schema.sub_resource_relationship and node_schema.scoped_cleanup:
         queries = []
         other_rels = (
             node_schema.other_relationships.rels
@@ -53,17 +60,20 @@ def build_cleanup_queries(node_schema: CartographyNodeSchema) -> List[str]:
             queries.append(query)
         return queries
 
-    result = _build_cleanup_node_and_rel_queries(
-        node_schema,
-        node_schema.sub_resource_relationship,
-    )
+    # Case 4: The node has no sub resource and scoped cleanup is false => clean up the stale nodes. Continue on to clean up the other_relationships too.
+    else:
+        queries = [_build_cleanup_node_query_unscoped(node_schema)]
+
     if node_schema.other_relationships:
         for rel in node_schema.other_relationships.rels:
-            # [0] is the delete node query, [1] is the delete relationship query. We only want the latter.
-            _, rel_query = _build_cleanup_node_and_rel_queries(node_schema, rel)
-            result.append(rel_query)
+            if node_schema.scoped_cleanup:
+                # [0] is the delete node query, [1] is the delete relationship query. We only want the latter.
+                _, rel_query = _build_cleanup_node_and_rel_queries(node_schema, rel)
+                queries.append(rel_query)
+            else:
+                queries.append(_build_cleanup_rel_queries_unscoped(node_schema, rel))
 
-    return result
+    return queries
 
 
 def _build_cleanup_rel_query_no_sub_resource(
@@ -94,6 +104,46 @@ def _build_cleanup_rel_query_no_sub_resource(
     )
 
 
+def _build_match_statement_for_cleanup(node_schema: CartographyNodeSchema) -> str:
+    """
+    Helper function to build a MATCH statement for a given node schema for cleanup.
+    """
+    if not node_schema.sub_resource_relationship and not node_schema.scoped_cleanup:
+        template = Template("MATCH (n:$node_label)")
+        return template.safe_substitute(
+            node_label=node_schema.label,
+        )
+
+    # if it has a sub resource relationship defined, we need to match on the sub resource to make sure we only delete
+    # nodes that are attached to the sub resource.
+    template = Template(
+        "MATCH (n:$node_label)$sub_resource_link(:$sub_resource_label{$match_sub_res_clause})"
+    )
+    sub_resource_link = ""
+    sub_resource_label = ""
+    match_sub_res_clause = ""
+
+    if node_schema.sub_resource_relationship:
+        # Draw sub resource rel with correct direction
+        if node_schema.sub_resource_relationship.direction == LinkDirection.INWARD:
+            sub_resource_link_template = Template("<-[s:$SubResourceRelLabel]-")
+        else:
+            sub_resource_link_template = Template("-[s:$SubResourceRelLabel]->")
+        sub_resource_link = sub_resource_link_template.safe_substitute(
+            SubResourceRelLabel=node_schema.sub_resource_relationship.rel_label,
+        )
+        sub_resource_label = node_schema.sub_resource_relationship.target_node_label
+        match_sub_res_clause = _build_match_clause(
+            node_schema.sub_resource_relationship.target_node_matcher,
+        )
+    return template.safe_substitute(
+        node_label=node_schema.label,
+        sub_resource_link=sub_resource_link,
+        sub_resource_label=sub_resource_label,
+        match_sub_res_clause=match_sub_res_clause,
+    )
+
+
 def _build_cleanup_node_and_rel_queries(
     node_schema: CartographyNodeSchema,
     selected_relationship: CartographyRelSchema,
@@ -120,15 +170,6 @@ def _build_cleanup_node_and_rel_queries(
             "verify the node class definition for the relationships that it has.",
         )
 
-    # Draw sub resource rel with correct direction
-    if node_schema.sub_resource_relationship.direction == LinkDirection.INWARD:
-        sub_resource_link_template = Template("<-[s:$SubResourceRelLabel]-")
-    else:
-        sub_resource_link_template = Template("-[s:$SubResourceRelLabel]->")
-    sub_resource_link = sub_resource_link_template.safe_substitute(
-        SubResourceRelLabel=node_schema.sub_resource_relationship.rel_label,
-    )
-
     # The cleanup node query must always be before the cleanup rel query
     delete_action_clauses = [
         """
@@ -161,19 +202,14 @@ def _build_cleanup_node_and_rel_queries(
     # Ensure the node is attached to the sub resource and delete the node
     query_template = Template(
         """
-        MATCH (n:$node_label)$sub_resource_link(:$sub_resource_label{$match_sub_res_clause})
+        $match_statement
         $selected_rel_clause
         $delete_action_clause
         """,
     )
     return [
         query_template.safe_substitute(
-            node_label=node_schema.label,
-            sub_resource_link=sub_resource_link,
-            sub_resource_label=node_schema.sub_resource_relationship.target_node_label,
-            match_sub_res_clause=_build_match_clause(
-                node_schema.sub_resource_relationship.target_node_matcher,
-            ),
+            match_statement=_build_match_statement_for_cleanup(node_schema),
             selected_rel_clause=(
                 ""
                 if selected_relationship == node_schema.sub_resource_relationship
@@ -185,6 +221,80 @@ def _build_cleanup_node_and_rel_queries(
     ]
 
 
+def _build_cleanup_node_query_unscoped(
+    node_schema: CartographyNodeSchema,
+) -> str:
+    """
+    Generates a cleanup query for a node_schema to allow unscoped cleanup.
+    """
+    if node_schema.scoped_cleanup:
+        raise ValueError(
+            f"_build_cleanup_node_query_for_unscoped_cleanup() failed: '{node_schema.label}' does not have "
+            "scoped_cleanup=False, so we cannot generate a query to clean it up. Please verify that the class "
+            "definition is what you expect.",
+        )
+
+    # The cleanup node query must always be before the cleanup rel query
+    delete_action_clause = """
+        WHERE n.lastupdated <> $UPDATE_TAG
+        WITH n LIMIT $LIMIT_SIZE
+        DETACH DELETE n;
+    """
+
+    # Ensure the node is attached to the sub resource and delete the node
+    query_template = Template(
+        """
+        $match_statement
+        $delete_action_clause
+        """,
+    )
+    return query_template.safe_substitute(
+        match_statement=_build_match_statement_for_cleanup(node_schema),
+        delete_action_clause=delete_action_clause,
+    )
+
+
+def _build_cleanup_rel_queries_unscoped(
+    node_schema: CartographyNodeSchema,
+    selected_relationship: CartographyRelSchema,
+) -> str:
+    """
+    Generates relationship cleanup query for a node_schema with scoped_cleanup=False.
+    """
+    if node_schema.scoped_cleanup:
+        raise ValueError(
+            f"_build_cleanup_node_and_rel_queries_unscoped() failed: '{node_schema.label}' does not have "
+            "scoped_cleanup=False, so we cannot generate a query to clean it up. Please verify that the class "
+            "definition is what you expect.",
+        )
+    if not rel_present_on_node_schema(node_schema, selected_relationship):
+        raise ValueError(
+            f"_build_cleanup_node_query(): Attempted to build cleanup query for node '{node_schema.label}' and "
+            f"relationship {selected_relationship.rel_label} but that relationship is not present on the node. Please "
+            "verify the node class definition for the relationships that it has.",
+        )
+
+    # The cleanup node query must always be before the cleanup rel query
+    delete_action_clause = """WHERE r.lastupdated <> $UPDATE_TAG
+        WITH r LIMIT $LIMIT_SIZE
+        DELETE r;
+        """
+
+    # Ensure the node is attached to the sub resource and delete the node
+    query_template = Template(
+        """
+        $match_statement
+        $selected_rel_clause
+        $delete_action_clause
+        """,
+    )
+    return query_template.safe_substitute(
+        match_statement=_build_match_statement_for_cleanup(node_schema),
+        selected_rel_clause=_build_selected_rel_clause(selected_relationship),
+        delete_action_clause=delete_action_clause,
+    )
+
+
 def _build_selected_rel_clause(selected_relationship: CartographyRelSchema) -> str:
     """
     Draw selected relationship with correct direction. Returns a string that looks like either

cartography/intel/anthropic/__init__.py

@@ -0,0 +1,62 @@
+import logging
+
+import neo4j
+import requests
+
+import cartography.intel.anthropic.apikeys
+import cartography.intel.anthropic.users
+import cartography.intel.anthropic.workspaces
+from cartography.config import Config
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def start_anthropic_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of Anthropic data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+
+    if not config.anthropic_apikey:
+        logger.info(
+            "Anthropic import is not configured - skipping this module. "
+            "See docs to configure.",
+        )
+        return
+
+    # Create requests sessions
+    api_session = requests.session()
+    api_session.headers.update(
+        {
+            "X-Api-Key": config.anthropic_apikey,
+            "anthropic-version": "2023-06-01",
+        }
+    )
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+        "BASE_URL": "https://api.anthropic.com/v1",
+    }
+
+    # Organization node is created during the users sync
+    cartography.intel.anthropic.users.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+    )
+
+    cartography.intel.anthropic.workspaces.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+    )
+
+    cartography.intel.anthropic.apikeys.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+    )

cartography/intel/anthropic/apikeys.py

@@ -0,0 +1,72 @@
+import logging
+from typing import Any
+from typing import Tuple
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.anthropic.util import paginated_get
+from cartography.models.anthropic.apikey import AnthropicApiKeySchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    org_id, apikeys = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+    )
+    common_job_parameters["ORG_ID"] = org_id
+    load_apikeys(
+        neo4j_session,
+        apikeys,
+        org_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+) -> Tuple[str, list[dict[str, Any]]]:
+    return paginated_get(
+        api_session, f"{base_url}/organizations/api_keys", timeout=_TIMEOUT
+    )
+
+
+@timeit
+def load_apikeys(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    ORG_ID: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Anthropic ApiKey into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        AnthropicApiKeySchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=ORG_ID,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(AnthropicApiKeySchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/anthropic/users.py

@@ -0,0 +1,75 @@
+import logging
+from typing import Any
+from typing import Tuple
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.anthropic.util import paginated_get
+from cartography.models.anthropic.organization import AnthropicOrganizationSchema
+from cartography.models.anthropic.user import AnthropicUserSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: dict[str, Any],
+) -> str:
+    org_id, users = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+    )
+    common_job_parameters["ORG_ID"] = org_id
+    load_users(neo4j_session, users, org_id, common_job_parameters["UPDATE_TAG"])
+    cleanup(neo4j_session, common_job_parameters)
+    return org_id
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+) -> Tuple[str, list[dict[str, Any]]]:
+    return paginated_get(
+        api_session, f"{base_url}/organizations/users", timeout=_TIMEOUT
+    )
+
+
+@timeit
+def load_users(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    ORG_ID: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        AnthropicOrganizationSchema(),
+        [{"id": ORG_ID}],
+        lastupdated=update_tag,
+    )
+    logger.info("Loading %d Anthropic User into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        AnthropicUserSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=ORG_ID,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(AnthropicUserSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/anthropic/util.py

@@ -0,0 +1,51 @@
+from typing import Any
+
+import requests
+
+
+def paginated_get(
+    api_session: requests.Session,
+    url: str,
+    timeout: tuple[int, int],
+    after: str | None = None,
+) -> tuple[str, list[dict[str, Any]]]:
+    """Helper function to get paginated data from the Anthropic API.
+
+    This function handles the pagination of the API requests and returns
+    the results in a list. It also retrieves the organization ID from the
+    response headers. The function will continue to make requests until
+    all pages of data have been retrieved. The results are returned as a
+    list of dictionaries, where each dictionary represents a single
+    entity.
+
+    Args:
+        api_session (requests.Session): The requests session to use for making API calls.
+        url (str): The URL to make the API call to.
+        timeout (tuple[int, int]): The timeout for the API call.
+        after (str | None): The ID of the last item retrieved in the previous request.
+            If None, the first page of results will be retrieved.
+    Returns:
+        tuple[str, list[dict[str, Any]]]: A tuple containing the organization ID and a list of
+            dictionaries representing the results.
+    """
+    results: list[dict[str, Any]] = []
+    params = {"after_id": after} if after else {}
+    req = api_session.get(
+        url,
+        params=params,
+        timeout=timeout,
+    )
+    req.raise_for_status()
+    # Get organization_id from the headers
+    organization_id = req.headers.get("anthropic-organization-id", "")
+    result = req.json()
+    results.extend(result.get("data", []))
+    if result.get("has_more"):
+        _, next_results = paginated_get(
+            api_session,
+            url,
+            timeout=timeout,
+            after=result.get("last_id"),
+        )
+        results.extend(next_results)
+    return organization_id, results