cartography 0.103.0rc1__py3-none-any.whl → 0.104.0rc2__py3-none-any.whl

cartography/intel/tailscale/__init__.py

@@ -0,0 +1,77 @@
+import logging
+
+import neo4j
+import requests
+
+import cartography.intel.tailscale.acls
+import cartography.intel.tailscale.devices
+import cartography.intel.tailscale.postureintegrations
+import cartography.intel.tailscale.tailnets
+import cartography.intel.tailscale.users
+from cartography.config import Config
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def start_tailscale_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of Tailscale data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+
+    if not config.tailscale_token or not config.tailscale_org:
+        logger.info(
+            "Tailscale import is not configured - skipping this module. "
+            "See docs to configure.",
+        )
+        return
+
+    # Create requests sessions
+    api_session = requests.session()
+    api_session.headers.update({"Authorization": f"Bearer {config.tailscale_token}"})
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+        "BASE_URL": config.tailscale_base_url,
+        "org": config.tailscale_org,
+    }
+
+    cartography.intel.tailscale.tailnets.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+        org=config.tailscale_org,
+    )
+
+    users = cartography.intel.tailscale.users.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+        org=config.tailscale_org,
+    )
+
+    cartography.intel.tailscale.devices.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+        org=config.tailscale_org,
+    )
+
+    cartography.intel.tailscale.postureintegrations.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+        org=config.tailscale_org,
+    )
+
+    cartography.intel.tailscale.acls.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+        org=config.tailscale_org,
+        users=users,
+    )

cartography/intel/tailscale/acls.py

@@ -0,0 +1,146 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import Tuple
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.tailscale.utils import ACLParser
+from cartography.intel.tailscale.utils import role_to_group
+from cartography.models.tailscale.group import TailscaleGroupSchema
+from cartography.models.tailscale.tag import TailscaleTagSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: Dict[str, Any],
+    org: str,
+    users: List[Dict[str, Any]],
+) -> None:
+    raw_acl = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+        org,
+    )
+    groups, tags = transform(raw_acl, users)
+    load_groups(
+        neo4j_session,
+        groups,
+        common_job_parameters["UPDATE_TAG"],
+        org,
+    )
+    load_tags(
+        neo4j_session,
+        tags,
+        org,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+    org: str,
+) -> str:
+    req = api_session.get(
+        f"{base_url}/tailnet/{org}/acl",
+        timeout=_TIMEOUT,
+    )
+    req.raise_for_status()
+    return req.text
+
+
+def transform(
+    raw_acl: str,
+    users: List[Dict[str, Any]],
+) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
+    transformed_groups: Dict[str, Dict[str, Any]] = {}
+    transformed_tags: Dict[str, Dict[str, Any]] = {}
+
+    parser = ACLParser(raw_acl)
+    # Extract groups from the ACL
+    for group in parser.get_groups():
+        for dom in group["domain_members"]:
+            for user in users:
+                if user["loginName"].endswith(f"@{dom}"):
+                    group["members"].append(user["loginName"])
+        # Ensure domain members are unique
+        group["domain_members"] = list(set(group["domain_members"]))
+        transformed_groups[group["id"]] = group
+    # Extract tags from the ACL
+    for tag in parser.get_tags():
+        for dom in tag["domain_owners"]:
+            for user in users:
+                if user["loginName"].endswith(f"@{dom}"):
+                    tag["owners"].append(user["loginName"])
+        # Ensure domain owners are unique
+        tag["owners"] = list(set(tag["owners"]))
+        transformed_tags[tag["id"]] = tag
+
+    # Add autogroups based on user roles
+    for user in users:
+        for g in role_to_group(user["role"]):
+            if g not in transformed_groups:
+                transformed_groups[g] = {
+                    "id": g,
+                    "name": g.split(":")[-1],
+                    "members": [],
+                    "sub_groups": [],
+                    "domain_members": [],
+                }
+            transformed_groups[g]["members"].append(user["loginName"])
+
+    return list(transformed_groups.values()), list(transformed_tags.values())
+
+
+@timeit
+def load_groups(
+    neo4j_session: neo4j.Session,
+    groups: List[Dict[str, Any]],
+    update_tag: str,
+    org: str,
+) -> None:
+    logger.info(f"Loading {len(groups)} Tailscale Groups to the graph")
+    load(neo4j_session, TailscaleGroupSchema(), groups, lastupdated=update_tag, org=org)
+
+
+@timeit
+def load_tags(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    org: str,
+    update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(data)} Tailscale Tags to the graph")
+    load(
+        neo4j_session,
+        TailscaleTagSchema(),
+        data,
+        lastupdated=update_tag,
+        org=org,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(TailscaleGroupSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(TailscaleTagSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/tailscale/devices.py

@@ -0,0 +1,127 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.tailscale.device import TailscaleDeviceSchema
+from cartography.models.tailscale.tag import TailscaleTagSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: Dict[str, Any],
+    org: str,
+) -> List[Dict]:
+    devices = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+        org,
+    )
+    tags = transform(devices)
+    load_devices(
+        neo4j_session,
+        devices,
+        org,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    load_tags(
+        neo4j_session,
+        tags,
+        org,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+    return devices
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+    org: str,
+) -> List[Dict[str, Any]]:
+    results: List[Dict[str, Any]] = []
+    req = api_session.get(
+        f"{base_url}/tailnet/{org}/devices",
+        timeout=_TIMEOUT,
+    )
+    req.raise_for_status()
+    results = req.json()["devices"]
+    return results
+
+
+def transform(
+    raw_data: List[Dict[str, Any]],
+) -> List[Dict[str, Any]]:
+    """Extracts tags from the raw data and returns a list of dictionaries"""
+    transformed_tags: Dict[str, Dict[str, Any]] = {}
+    # Transform the raw data into the format expected by the load function
+    for device in raw_data:
+        for raw_tag in device.get("tags", []):
+            if raw_tag not in transformed_tags:
+                transformed_tags[raw_tag] = {
+                    "id": raw_tag,
+                    "name": raw_tag.split(":")[-1],
+                    "devices": [device["nodeId"]],
+                }
+            else:
+                transformed_tags[raw_tag]["devices"].append(device["nodeId"])
+    return list(transformed_tags.values())
+
+
+@timeit
+def load_devices(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    org: str,
+    update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(data)} Tailscale Devices to the graph")
+    load(
+        neo4j_session,
+        TailscaleDeviceSchema(),
+        data,
+        lastupdated=update_tag,
+        org=org,
+    )
+
+
+@timeit
+def load_tags(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    org: str,
+    update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(data)} Tailscale Tags to the graph")
+    load(
+        neo4j_session,
+        TailscaleTagSchema(),
+        data,
+        lastupdated=update_tag,
+        org=org,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(TailscaleDeviceSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(TailscaleTagSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/tailscale/postureintegrations.py

@@ -0,0 +1,81 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.tailscale.postureintegration import (
+    TailscalePostureIntegrationSchema,
+)
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: Dict[str, Any],
+    org: str,
+) -> None:
+    postureintegrations = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+        org,
+    )
+    load_postureintegrations(
+        neo4j_session,
+        postureintegrations,
+        org,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+    org: str,
+) -> List[Dict[str, Any]]:
+    results: List[Dict[str, Any]] = []
+    req = api_session.get(
+        f"{base_url}/tailnet/{org}/posture/integrations",
+        timeout=_TIMEOUT,
+    )
+    req.raise_for_status()
+    results = req.json()["integrations"]
+    return results
+
+
+@timeit
+def load_postureintegrations(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    org: str,
+    update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(data)} Tailscale PostureIntegrations to the graph")
+    load(
+        neo4j_session,
+        TailscalePostureIntegrationSchema(),
+        data,
+        lastupdated=update_tag,
+        org=org,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(
+        TailscalePostureIntegrationSchema(), common_job_parameters
+    ).run(neo4j_session)

cartography/intel/tailscale/tailnets.py

@@ -0,0 +1,76 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.tailscale.tailnet import TailscaleTailnetSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: Dict[str, Any],
+    org: str,
+) -> None:
+    tailnet = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+        org,
+    )
+    load_tailnets(
+        neo4j_session,
+        [tailnet],
+        org,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+    org: str,
+) -> Dict[str, Any]:
+    req = api_session.get(
+        f"{base_url}/tailnet/{org}/settings",
+        timeout=_TIMEOUT,
+    )
+    req.raise_for_status()
+    return req.json()
+
+
+@timeit
+def load_tailnets(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    org: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        TailscaleTailnetSchema(),
+        data,
+        lastupdated=update_tag,
+        org=org,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(TailscaleTailnetSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/tailscale/users.py

@@ -0,0 +1,80 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.tailscale.user import TailscaleUserSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: Dict[str, Any],
+    org: str,
+) -> List[Dict]:
+    users = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+        org,
+    )
+    load_users(
+        neo4j_session,
+        users,
+        org,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+    return users
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+    org: str,
+) -> List[Dict[str, Any]]:
+    results: List[Dict[str, Any]] = []
+    req = api_session.get(
+        f"{base_url}/tailnet/{org}/users",
+        timeout=_TIMEOUT,
+    )
+    req.raise_for_status()
+    results = req.json()["users"]
+    return results
+
+
+@timeit
+def load_users(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    org: str,
+    update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(data)} Tailscale Users to the graph")
+    load(
+        neo4j_session,
+        TailscaleUserSchema(),
+        data,
+        lastupdated=update_tag,
+        org=org,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(TailscaleUserSchema(), common_job_parameters).run(
+        neo4j_session
+    )