cartography 0.103.0rc1__py3-none-any.whl → 0.104.0__py3-none-any.whl

cartography/intel/openai/serviceaccounts.py

@@ -0,0 +1,82 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.openai.util import paginated_get
+from cartography.models.openai.serviceaccount import OpenAIServiceAccountSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: Dict[str, Any],
+    project_id: str,
+) -> None:
+    serviceaccountss = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+        project_id,
+    )
+    load_serviceaccounts(
+        neo4j_session,
+        serviceaccountss,
+        project_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+    project_id: str,
+) -> List[Dict[str, Any]]:
+    return list(
+        paginated_get(
+            api_session,
+            "{base_url}/organization/projects/{project_id}/service_accounts".format(
+                base_url=base_url,
+                project_id=project_id,
+            ),
+            timeout=_TIMEOUT,
+        )
+    )
+
+
+@timeit
+def load_serviceaccounts(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    project_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d OpenAI ServiceAccount into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        OpenAIServiceAccountSchema(),
+        data,
+        lastupdated=update_tag,
+        project_id=project_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(OpenAIServiceAccountSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/openai/users.py

@@ -0,0 +1,75 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.openai.util import paginated_get
+from cartography.models.openai.organization import OpenAIOrganizationSchema
+from cartography.models.openai.user import OpenAIUserSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: Dict[str, Any],
+    ORG_ID: str,
+) -> None:
+    users = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+    )
+    load_users(neo4j_session, users, ORG_ID, common_job_parameters["UPDATE_TAG"])
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+) -> List[Dict[str, Any]]:
+    return list(
+        paginated_get(api_session, f"{base_url}/organization/users", timeout=_TIMEOUT)
+    )
+
+
+@timeit
+def load_users(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    ORG_ID: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        OpenAIOrganizationSchema(),
+        [{"id": ORG_ID}],
+        lastupdated=update_tag,
+    )
+    logger.info("Loading %d OpenAI User into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        OpenAIUserSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=ORG_ID,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(OpenAIUserSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/openai/util.py

@@ -0,0 +1,45 @@
+from typing import Any
+from typing import Generator
+
+import requests
+
+
+def paginated_get(
+    api_session: requests.Session,
+    url: str,
+    timeout: tuple[int, int],
+    after: str | None = None,
+) -> Generator[dict[str, Any], None, None]:
+    """Helper function to get paginated data from the OpenAI API.
+
+    This function handles the pagination of the API requests and returns
+    the results as a generator. It will continue to make requests until
+    all pages of data have been retrieved. The results are returned as a
+    list of dictionaries, where each dictionary represents a single
+    entity.
+
+    Args:
+        api_session (requests.Session): The requests session to use for making API calls.
+        url (str): The URL to make the API call to.
+        timeout (tuple[int, int]): The timeout for the API call.
+        after (str | None): The ID of the last item retrieved in the previous request.
+            If None, the first page of results will be retrieved.
+    Returns:
+        Generator[dict[str, Any], None, None]: A generator yielding dictionaries representing the results.
+    """
+    params = {"after": after} if after else {}
+    req = api_session.get(
+        url,
+        params=params,
+        timeout=timeout,
+    )
+    req.raise_for_status()
+    result = req.json()
+    yield from result.get("data", [])
+    if result.get("has_more"):
+        yield from paginated_get(
+            api_session,
+            url,
+            timeout=timeout,
+            after=result.get("last_id"),
+        )

cartography/intel/tailscale/__init__.py

@@ -0,0 +1,77 @@
+import logging
+
+import neo4j
+import requests
+
+import cartography.intel.tailscale.acls
+import cartography.intel.tailscale.devices
+import cartography.intel.tailscale.postureintegrations
+import cartography.intel.tailscale.tailnets
+import cartography.intel.tailscale.users
+from cartography.config import Config
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def start_tailscale_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of Tailscale data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+
+    if not config.tailscale_token or not config.tailscale_org:
+        logger.info(
+            "Tailscale import is not configured - skipping this module. "
+            "See docs to configure.",
+        )
+        return
+
+    # Create requests sessions
+    api_session = requests.session()
+    api_session.headers.update({"Authorization": f"Bearer {config.tailscale_token}"})
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+        "BASE_URL": config.tailscale_base_url,
+        "org": config.tailscale_org,
+    }
+
+    cartography.intel.tailscale.tailnets.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+        org=config.tailscale_org,
+    )
+
+    users = cartography.intel.tailscale.users.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+        org=config.tailscale_org,
+    )
+
+    cartography.intel.tailscale.devices.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+        org=config.tailscale_org,
+    )
+
+    cartography.intel.tailscale.postureintegrations.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+        org=config.tailscale_org,
+    )
+
+    cartography.intel.tailscale.acls.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+        org=config.tailscale_org,
+        users=users,
+    )

cartography/intel/tailscale/acls.py

@@ -0,0 +1,146 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import Tuple
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.tailscale.utils import ACLParser
+from cartography.intel.tailscale.utils import role_to_group
+from cartography.models.tailscale.group import TailscaleGroupSchema
+from cartography.models.tailscale.tag import TailscaleTagSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: Dict[str, Any],
+    org: str,
+    users: List[Dict[str, Any]],
+) -> None:
+    raw_acl = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+        org,
+    )
+    groups, tags = transform(raw_acl, users)
+    load_groups(
+        neo4j_session,
+        groups,
+        common_job_parameters["UPDATE_TAG"],
+        org,
+    )
+    load_tags(
+        neo4j_session,
+        tags,
+        org,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+    org: str,
+) -> str:
+    req = api_session.get(
+        f"{base_url}/tailnet/{org}/acl",
+        timeout=_TIMEOUT,
+    )
+    req.raise_for_status()
+    return req.text
+
+
+def transform(
+    raw_acl: str,
+    users: List[Dict[str, Any]],
+) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
+    transformed_groups: Dict[str, Dict[str, Any]] = {}
+    transformed_tags: Dict[str, Dict[str, Any]] = {}
+
+    parser = ACLParser(raw_acl)
+    # Extract groups from the ACL
+    for group in parser.get_groups():
+        for dom in group["domain_members"]:
+            for user in users:
+                if user["loginName"].endswith(f"@{dom}"):
+                    group["members"].append(user["loginName"])
+        # Ensure domain members are unique
+        group["domain_members"] = list(set(group["domain_members"]))
+        transformed_groups[group["id"]] = group
+    # Extract tags from the ACL
+    for tag in parser.get_tags():
+        for dom in tag["domain_owners"]:
+            for user in users:
+                if user["loginName"].endswith(f"@{dom}"):
+                    tag["owners"].append(user["loginName"])
+        # Ensure domain owners are unique
+        tag["owners"] = list(set(tag["owners"]))
+        transformed_tags[tag["id"]] = tag
+
+    # Add autogroups based on user roles
+    for user in users:
+        for g in role_to_group(user["role"]):
+            if g not in transformed_groups:
+                transformed_groups[g] = {
+                    "id": g,
+                    "name": g.split(":")[-1],
+                    "members": [],
+                    "sub_groups": [],
+                    "domain_members": [],
+                }
+            transformed_groups[g]["members"].append(user["loginName"])
+
+    return list(transformed_groups.values()), list(transformed_tags.values())
+
+
+@timeit
+def load_groups(
+    neo4j_session: neo4j.Session,
+    groups: List[Dict[str, Any]],
+    update_tag: str,
+    org: str,
+) -> None:
+    logger.info(f"Loading {len(groups)} Tailscale Groups to the graph")
+    load(neo4j_session, TailscaleGroupSchema(), groups, lastupdated=update_tag, org=org)
+
+
+@timeit
+def load_tags(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    org: str,
+    update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(data)} Tailscale Tags to the graph")
+    load(
+        neo4j_session,
+        TailscaleTagSchema(),
+        data,
+        lastupdated=update_tag,
+        org=org,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(TailscaleGroupSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(TailscaleTagSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/tailscale/devices.py

@@ -0,0 +1,127 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.tailscale.device import TailscaleDeviceSchema
+from cartography.models.tailscale.tag import TailscaleTagSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: Dict[str, Any],
+    org: str,
+) -> List[Dict]:
+    devices = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+        org,
+    )
+    tags = transform(devices)
+    load_devices(
+        neo4j_session,
+        devices,
+        org,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    load_tags(
+        neo4j_session,
+        tags,
+        org,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+    return devices
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+    org: str,
+) -> List[Dict[str, Any]]:
+    results: List[Dict[str, Any]] = []
+    req = api_session.get(
+        f"{base_url}/tailnet/{org}/devices",
+        timeout=_TIMEOUT,
+    )
+    req.raise_for_status()
+    results = req.json()["devices"]
+    return results
+
+
+def transform(
+    raw_data: List[Dict[str, Any]],
+) -> List[Dict[str, Any]]:
+    """Extracts tags from the raw data and returns a list of dictionaries"""
+    transformed_tags: Dict[str, Dict[str, Any]] = {}
+    # Transform the raw data into the format expected by the load function
+    for device in raw_data:
+        for raw_tag in device.get("tags", []):
+            if raw_tag not in transformed_tags:
+                transformed_tags[raw_tag] = {
+                    "id": raw_tag,
+                    "name": raw_tag.split(":")[-1],
+                    "devices": [device["nodeId"]],
+                }
+            else:
+                transformed_tags[raw_tag]["devices"].append(device["nodeId"])
+    return list(transformed_tags.values())
+
+
+@timeit
+def load_devices(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    org: str,
+    update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(data)} Tailscale Devices to the graph")
+    load(
+        neo4j_session,
+        TailscaleDeviceSchema(),
+        data,
+        lastupdated=update_tag,
+        org=org,
+    )
+
+
+@timeit
+def load_tags(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    org: str,
+    update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(data)} Tailscale Tags to the graph")
+    load(
+        neo4j_session,
+        TailscaleTagSchema(),
+        data,
+        lastupdated=update_tag,
+        org=org,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(TailscaleDeviceSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(TailscaleTagSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/tailscale/postureintegrations.py

@@ -0,0 +1,81 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.tailscale.postureintegration import (
+    TailscalePostureIntegrationSchema,
+)
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: Dict[str, Any],
+    org: str,
+) -> None:
+    postureintegrations = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+        org,
+    )
+    load_postureintegrations(
+        neo4j_session,
+        postureintegrations,
+        org,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+    org: str,
+) -> List[Dict[str, Any]]:
+    results: List[Dict[str, Any]] = []
+    req = api_session.get(
+        f"{base_url}/tailnet/{org}/posture/integrations",
+        timeout=_TIMEOUT,
+    )
+    req.raise_for_status()
+    results = req.json()["integrations"]
+    return results
+
+
+@timeit
+def load_postureintegrations(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    org: str,
+    update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(data)} Tailscale PostureIntegrations to the graph")
+    load(
+        neo4j_session,
+        TailscalePostureIntegrationSchema(),
+        data,
+        lastupdated=update_tag,
+        org=org,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(
+        TailscalePostureIntegrationSchema(), common_job_parameters
+    ).run(neo4j_session)

cartography/intel/tailscale/tailnets.py

@@ -0,0 +1,76 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.tailscale.tailnet import TailscaleTailnetSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: Dict[str, Any],
+    org: str,
+) -> None:
+    tailnet = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+        org,
+    )
+    load_tailnets(
+        neo4j_session,
+        [tailnet],
+        org,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+    org: str,
+) -> Dict[str, Any]:
+    req = api_session.get(
+        f"{base_url}/tailnet/{org}/settings",
+        timeout=_TIMEOUT,
+    )
+    req.raise_for_status()
+    return req.json()
+
+
+@timeit
+def load_tailnets(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    org: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        TailscaleTailnetSchema(),
+        data,
+        lastupdated=update_tag,
+        org=org,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(TailscaleTailnetSchema(), common_job_parameters).run(
+        neo4j_session
+    )