cartography 0.93.0rc1__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/spacelift/util.py

@@ -0,0 +1,122 @@
+"""
+Utility functions for Spacelift GraphQL API interactions.
+"""
+
+import logging
+from typing import Any
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+# Timeout for API calls: (connection timeout, read timeout) in seconds
+_TIMEOUT = (60, 60)
+
+# GraphQL mutation to exchange API key ID and secret for a JWT token
+_TOKEN_EXCHANGE_MUTATION = """
+mutation GetSpaceliftToken($id: ID!, $secret: String!) {
+  apiKeyUser(id: $id, secret: $secret) {
+    jwt
+  }
+}
+"""
+
+
+def call_spacelift_api(
+    session: requests.Session,
+    api_endpoint: str,
+    query: str,
+    variables: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """
+    Make a GraphQL query to the Spacelift API.
+    """
+    logger.debug(f"Making GraphQL request to {api_endpoint}")
+
+    # Prepare the GraphQL request payload
+    payload: dict[str, Any] = {"query": query}
+    if variables:
+        payload["variables"] = variables
+
+    # Make the POST request to the GraphQL endpoint
+    response = session.post(
+        api_endpoint,
+        json=payload,
+        timeout=_TIMEOUT,
+    )
+
+    # Raise an exception for HTTP errors (4xx, 5xx)
+    response.raise_for_status()
+
+    # Parse the JSON response
+    result = response.json()
+
+    # Check for GraphQL errors in the response
+    if "errors" in result:
+        error_messages = [
+            error.get("message", "Unknown error") for error in result["errors"]
+        ]
+        error_string = "; ".join(error_messages)
+        raise ValueError(f"GraphQL query failed: {error_string}")
+
+    return result
+
+
+def get_spacelift_token(api_endpoint: str, key_id: str, key_secret: str) -> str:
+    """
+    Exchange Spacelift API key ID and secret for a JWT bearer token.
+
+    This uses the Spacelift GraphQL API's apiKeyUser mutation to obtain a JWT token
+    that can be used for authenticated requests. This is the recommended authentication
+    method as it avoids storing long-lived tokens.
+
+    :param api_endpoint: Spacelift GraphQL API endpoint (e.g., https://your-account.app.spacelift.io/graphql)
+    :param key_id: Spacelift API key ID (26-character ULID)
+    :param key_secret: Spacelift API key secret
+    :return: JWT bearer token string
+    :raises: requests.exceptions.RequestException if the API request fails
+    :raises: ValueError if the response doesn't contain a valid token
+    """
+    logger.info("Exchanging Spacelift API key for JWT token")
+
+    payload = {
+        "query": _TOKEN_EXCHANGE_MUTATION,
+        "variables": {
+            "id": key_id,
+            "secret": key_secret,
+        },
+    }
+
+    try:
+        response = requests.post(
+            api_endpoint,
+            json=payload,
+            headers={"Content-Type": "application/json"},
+            timeout=_TIMEOUT,
+        )
+        response.raise_for_status()
+
+        data: dict[str, Any] = response.json()
+
+        # Check for GraphQL errors
+        if "errors" in data:
+            error_messages = [
+                error.get("message", "Unknown error") for error in data["errors"]
+            ]
+            error_string = "; ".join(error_messages)
+            raise ValueError(f"Token exchange failed: {error_string}")
+
+        # Extract JWT token from response
+        jwt = data.get("data", {}).get("apiKeyUser", {}).get("jwt")
+        if not jwt:
+            raise ValueError("Token exchange response did not contain a JWT token")
+
+        logger.info("Successfully obtained JWT token from Spacelift API")
+        return jwt
+
+    except requests.exceptions.RequestException as e:
+        logger.error(f"Failed to exchange Spacelift API key for token: {e}")
+        raise
+    except (KeyError, ValueError) as e:
+        logger.error(f"Invalid response from Spacelift token exchange: {e}")
+        raise

cartography/intel/spacelift/workerpools.py

@@ -0,0 +1,131 @@
+import logging
+from typing import Any
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.spacelift.util import call_spacelift_api
+from cartography.models.spacelift.workerpool import SpaceliftWorkerPoolSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+# GraphQL query to fetch all worker pools (direct array, no pagination)
+GET_WORKER_POOLS_QUERY = """
+query {
+    workerPools {
+        id
+        name
+        description
+        space
+    }
+}
+"""
+
+
+@timeit
+def get_worker_pools(
+    session: requests.Session, api_endpoint: str
+) -> list[dict[str, Any]]:
+
+    logger.info("Fetching Spacelift worker pools")
+
+    response = call_spacelift_api(session, api_endpoint, GET_WORKER_POOLS_QUERY)
+    worker_pools_data = response.get("data", {}).get("workerPools", [])
+
+    logger.info(f"Retrieved {len(worker_pools_data)} Spacelift worker pools")
+    return worker_pools_data
+
+
+def transform_worker_pools(
+    worker_pools_data: list[dict[str, Any]], account_id: str
+) -> list[dict[str, Any]]:
+    result: list[dict[str, Any]] = []
+
+    for pool in worker_pools_data:
+        transformed_pool = {
+            "id": pool["id"],
+            "name": pool.get("name"),
+            "description": pool.get("description"),
+            "pool_type": None,  # WorkerPool type doesn't have a 'type' field in the API
+            "space_id": pool.get("space"),
+            "spacelift_account_id": account_id,
+        }
+
+        result.append(transformed_pool)
+
+    logger.info(f"Transformed {len(result)} worker pools")
+    return result
+
+
+def load_worker_pools(
+    neo4j_session: neo4j.Session,
+    worker_pools_data: list[dict[str, Any]],
+    update_tag: int,
+    account_id: str,
+) -> None:
+    """
+    Load Spacelift worker pools data into Neo4j using the data model.
+    """
+    load(
+        neo4j_session,
+        SpaceliftWorkerPoolSchema(),
+        worker_pools_data,
+        lastupdated=update_tag,
+        spacelift_account_id=account_id,
+    )
+
+    logger.info(f"Loaded {len(worker_pools_data)} Spacelift worker pools")
+
+
+@timeit
+def cleanup_worker_pools(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    """
+    Remove stale Spacelift worker pool data from Neo4j.
+    """
+    logger.debug("Running SpaceliftWorkerPool cleanup job")
+    GraphJob.from_node_schema(SpaceliftWorkerPoolSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync_worker_pools(
+    neo4j_session: neo4j.Session,
+    spacelift_session: requests.Session,
+    api_endpoint: str,
+    account_id: str,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    """
+    Sync Spacelift worker pools data to Neo4j.
+
+    :param neo4j_session: Neo4j session
+    :param spacelift_session: Authenticated requests session for Spacelift API
+    :param api_endpoint: Spacelift GraphQL API endpoint
+    :param account_id: The Spacelift account ID
+    :param common_job_parameters: Common job parameters containing UPDATE_TAG
+    """
+    # 1. GET - Fetch data from Spacelift API
+    worker_pools_raw_data = get_worker_pools(spacelift_session, api_endpoint)
+
+    # 2. TRANSFORM - Shape data for ingestion
+    transformed_worker_pools = transform_worker_pools(worker_pools_raw_data, account_id)
+
+    # 3. LOAD - Ingest to Neo4j using data model
+    load_worker_pools(
+        neo4j_session,
+        transformed_worker_pools,
+        common_job_parameters["UPDATE_TAG"],
+        account_id,
+    )
+
+    # 4. CLEANUP - Remove stale data
+    cleanup_worker_pools(neo4j_session, common_job_parameters)
+
+    logger.info(f"Synced {len(transformed_worker_pools)} Spacelift worker pools")

cartography/intel/spacelift/workers.py

@@ -0,0 +1,128 @@
+import logging
+from typing import Any
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.spacelift.util import call_spacelift_api
+from cartography.models.spacelift.worker import SpaceliftWorkerSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+# GraphQL query to fetch workers nested under workerPools
+# Note: Workers don't have a top-level query, they're nested under workerPools
+# Note: Worker type doesn't have a 'name' field, only 'id' and 'status'
+GET_WORKERS_QUERY = """
+query {
+    workerPools {
+        id
+        workers {
+            id
+            status
+        }
+    }
+}
+"""
+
+
+@timeit
+def get_workers(session: requests.Session, api_endpoint: str) -> list[dict[str, Any]]:
+    """
+    Fetch all Spacelift workers from the API.
+    Workers are nested under workerPools, so we query workerPools and flatten the workers.
+    """
+    logger.info("Fetching Spacelift workers")
+
+    response = call_spacelift_api(session, api_endpoint, GET_WORKERS_QUERY)
+    worker_pools = response.get("data", {}).get("workerPools", [])
+
+    # Flatten workers from all pools and add the pool ID to each worker
+    all_workers = []
+    for pool in worker_pools:
+        pool_id = pool["id"]
+        for worker in pool.get("workers", []):
+            # Add the pool ID to each worker
+            worker["workerPool"] = pool_id
+            all_workers.append(worker)
+
+    logger.info(
+        f"Retrieved {len(all_workers)} Spacelift workers from {len(worker_pools)} worker pools"
+    )
+    return all_workers
+
+
+def transform_workers(
+    workers_data: list[dict[str, Any]], account_id: str
+) -> list[dict[str, Any]]:
+    result: list[dict[str, Any]] = []
+
+    for worker in workers_data:
+        worker_id = worker["id"]
+
+        transformed_worker = {
+            "id": worker_id,
+            "name": worker_id,  # Use ID as name since Worker type doesn't have a name field
+            "status": worker.get("status"),
+            "worker_pool_id": worker.get("workerPool"),
+            "spacelift_account_id": account_id,
+        }
+
+        result.append(transformed_worker)
+
+    logger.info(f"Transformed {len(result)} workers")
+    return result
+
+
+def load_workers(
+    neo4j_session: neo4j.Session,
+    workers_data: list[dict[str, Any]],
+    update_tag: int,
+    account_id: str,
+) -> None:
+
+    load(
+        neo4j_session,
+        SpaceliftWorkerSchema(),
+        workers_data,
+        lastupdated=update_tag,
+        spacelift_account_id=account_id,
+    )
+
+    logger.info(f"Loaded {len(workers_data)} Spacelift workers")
+
+
+@timeit
+def cleanup_workers(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+) -> None:
+
+    logger.debug("Running SpaceliftWorker cleanup job")
+    GraphJob.from_node_schema(SpaceliftWorkerSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync_workers(
+    neo4j_session: neo4j.Session,
+    spacelift_session: requests.Session,
+    api_endpoint: str,
+    account_id: str,
+    common_job_parameters: dict[str, Any],
+) -> None:
+
+    workers_raw_data = get_workers(spacelift_session, api_endpoint)
+    transformed_workers = transform_workers(workers_raw_data, account_id)
+    load_workers(
+        neo4j_session,
+        transformed_workers,
+        common_job_parameters["UPDATE_TAG"],
+        account_id,
+    )
+    cleanup_workers(neo4j_session, common_job_parameters)
+
+    logger.info(f"Synced {len(transformed_workers)} Spacelift workers")

cartography/intel/tailscale/__init__.py

@@ -0,0 +1,77 @@
+import logging
+
+import neo4j
+import requests
+
+import cartography.intel.tailscale.acls
+import cartography.intel.tailscale.devices
+import cartography.intel.tailscale.postureintegrations
+import cartography.intel.tailscale.tailnets
+import cartography.intel.tailscale.users
+from cartography.config import Config
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def start_tailscale_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of Tailscale data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+
+    if not config.tailscale_token or not config.tailscale_org:
+        logger.info(
+            "Tailscale import is not configured - skipping this module. "
+            "See docs to configure.",
+        )
+        return
+
+    # Create requests sessions
+    api_session = requests.session()
+    api_session.headers.update({"Authorization": f"Bearer {config.tailscale_token}"})
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+        "BASE_URL": config.tailscale_base_url,
+        "org": config.tailscale_org,
+    }
+
+    cartography.intel.tailscale.tailnets.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+        org=config.tailscale_org,
+    )
+
+    users = cartography.intel.tailscale.users.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+        org=config.tailscale_org,
+    )
+
+    cartography.intel.tailscale.devices.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+        org=config.tailscale_org,
+    )
+
+    cartography.intel.tailscale.postureintegrations.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+        org=config.tailscale_org,
+    )
+
+    cartography.intel.tailscale.acls.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+        org=config.tailscale_org,
+        users=users,
+    )

cartography/intel/tailscale/acls.py

@@ -0,0 +1,146 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import Tuple
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.tailscale.utils import ACLParser
+from cartography.intel.tailscale.utils import role_to_group
+from cartography.models.tailscale.group import TailscaleGroupSchema
+from cartography.models.tailscale.tag import TailscaleTagSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: Dict[str, Any],
+    org: str,
+    users: List[Dict[str, Any]],
+) -> None:
+    raw_acl = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+        org,
+    )
+    groups, tags = transform(raw_acl, users)
+    load_groups(
+        neo4j_session,
+        groups,
+        common_job_parameters["UPDATE_TAG"],
+        org,
+    )
+    load_tags(
+        neo4j_session,
+        tags,
+        org,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+    org: str,
+) -> str:
+    req = api_session.get(
+        f"{base_url}/tailnet/{org}/acl",
+        timeout=_TIMEOUT,
+    )
+    req.raise_for_status()
+    return req.text
+
+
+def transform(
+    raw_acl: str,
+    users: List[Dict[str, Any]],
+) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
+    transformed_groups: Dict[str, Dict[str, Any]] = {}
+    transformed_tags: Dict[str, Dict[str, Any]] = {}
+
+    parser = ACLParser(raw_acl)
+    # Extract groups from the ACL
+    for group in parser.get_groups():
+        for dom in group["domain_members"]:
+            for user in users:
+                if user["loginName"].endswith(f"@{dom}"):
+                    group["members"].append(user["loginName"])
+        # Ensure domain members are unique
+        group["domain_members"] = list(set(group["domain_members"]))
+        transformed_groups[group["id"]] = group
+    # Extract tags from the ACL
+    for tag in parser.get_tags():
+        for dom in tag["domain_owners"]:
+            for user in users:
+                if user["loginName"].endswith(f"@{dom}"):
+                    tag["owners"].append(user["loginName"])
+        # Ensure domain owners are unique
+        tag["owners"] = list(set(tag["owners"]))
+        transformed_tags[tag["id"]] = tag
+
+    # Add autogroups based on user roles
+    for user in users:
+        for g in role_to_group(user["role"]):
+            if g not in transformed_groups:
+                transformed_groups[g] = {
+                    "id": g,
+                    "name": g.split(":")[-1],
+                    "members": [],
+                    "sub_groups": [],
+                    "domain_members": [],
+                }
+            transformed_groups[g]["members"].append(user["loginName"])
+
+    return list(transformed_groups.values()), list(transformed_tags.values())
+
+
+@timeit
+def load_groups(
+    neo4j_session: neo4j.Session,
+    groups: List[Dict[str, Any]],
+    update_tag: str,
+    org: str,
+) -> None:
+    logger.info(f"Loading {len(groups)} Tailscale Groups to the graph")
+    load(neo4j_session, TailscaleGroupSchema(), groups, lastupdated=update_tag, org=org)
+
+
+@timeit
+def load_tags(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    org: str,
+    update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(data)} Tailscale Tags to the graph")
+    load(
+        neo4j_session,
+        TailscaleTagSchema(),
+        data,
+        lastupdated=update_tag,
+        org=org,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(TailscaleGroupSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(TailscaleTagSchema(), common_job_parameters).run(
+        neo4j_session
+    )