cartography 0.102.0rc1__py3-none-any.whl → 0.103.0__py3-none-any.whl

cartography/intel/snipeit/asset.py

@@ -5,13 +5,13 @@ from typing import List
 
 import neo4j
 
-from .util import call_snipeit_api
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
 from cartography.models.snipeit.asset import SnipeitAssetSchema
 from cartography.models.snipeit.tenant import SnipeitTenantSchema
 from cartography.util import timeit
 
+from .util import call_snipeit_api
 
 logger = logging.getLogger(__name__)
 
@@ -24,9 +24,9 @@ def get(base_uri: str, token: str) -> List[Dict]:
         offset = len(results)
         api_endpoint = f"{api_endpoint}?order='asc'&offset={offset}"
         response = call_snipeit_api(api_endpoint, base_uri, token)
-        results.extend(response['rows'])
+        results.extend(response["rows"])
 
-        total = response['total']
+        total = response["total"]
         results_count = len(results)
         if results_count >= total:
             break
@@ -44,7 +44,7 @@ def load_assets(
     load(
         neo4j_session,
         SnipeitTenantSchema(),
-        [{'id': common_job_parameters["TENANT_ID"]}],
+        [{"id": common_job_parameters["TENANT_ID"]}],
         lastupdated=common_job_parameters["UPDATE_TAG"],
     )
 
@@ -59,7 +59,9 @@ def load_assets(
 
 @timeit
 def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
-    GraphJob.from_node_schema(SnipeitAssetSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(SnipeitAssetSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
 
 
 @timeit
@@ -70,5 +72,9 @@ def sync(
     token: str,
 ) -> None:
     assets = get(base_uri=base_uri, token=token)
-    load_assets(neo4j_session=neo4j_session, common_job_parameters=common_job_parameters, data=assets)
+    load_assets(
+        neo4j_session=neo4j_session,
+        common_job_parameters=common_job_parameters,
+        data=assets,
+    )
     cleanup(neo4j_session, common_job_parameters)

cartography/intel/snipeit/user.py

@@ -5,13 +5,14 @@ from typing import List
 
 import neo4j
 
-from .util import call_snipeit_api
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
 from cartography.models.snipeit.tenant import SnipeitTenantSchema
 from cartography.models.snipeit.user import SnipeitUserSchema
 from cartography.util import timeit
 
+from .util import call_snipeit_api
+
 logger = logging.getLogger(__name__)
 
 
@@ -23,9 +24,9 @@ def get(base_uri: str, token: str) -> List[Dict]:
         offset = len(results)
         api_endpoint = f"{api_endpoint}?order='asc'&offset={offset}"
         response = call_snipeit_api(api_endpoint, base_uri, token)
-        results.extend(response['rows'])
+        results.extend(response["rows"])
 
-        total = response['total']
+        total = response["total"]
         results_count = len(results)
         if results_count >= total:
             break
@@ -45,7 +46,7 @@ def load_users(
     load(
         neo4j_session,
         SnipeitTenantSchema(),
-        [{'id': common_job_parameters["TENANT_ID"]}],
+        [{"id": common_job_parameters["TENANT_ID"]}],
         lastupdated=common_job_parameters["UPDATE_TAG"],
     )
 
@@ -60,7 +61,9 @@ def load_users(
 
 @timeit
 def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
-    GraphJob.from_node_schema(SnipeitUserSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(SnipeitUserSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
 
 
 @timeit

cartography/intel/snipeit/util.py

@@ -12,17 +12,22 @@ _TIMEOUT = (60, 60)
 
 
 @timeit
-def call_snipeit_api(api_and_parameters: str, base_uri: str, token: str) -> Dict[str, Any]:
+def call_snipeit_api(
+    api_and_parameters: str,
+    base_uri: str,
+    token: str,
+) -> Dict[str, Any]:
     uri = base_uri + api_and_parameters
     try:
         logger.debug(
-            "SnipeIT: Get %s", uri,
+            "SnipeIT: Get %s",
+            uri,
         )
         response = requests.get(
             uri,
             headers={
-                'Accept': 'application/json',
-                'Authorization': f'Bearer {token}',
+                "Accept": "application/json",
+                "Authorization": f"Bearer {token}",
             },
             timeout=_TIMEOUT,
         )

cartography/intel/tailscale/__init__.py

@@ -0,0 +1,77 @@
+import logging
+
+import neo4j
+import requests
+
+import cartography.intel.tailscale.acls
+import cartography.intel.tailscale.devices
+import cartography.intel.tailscale.postureintegrations
+import cartography.intel.tailscale.tailnets
+import cartography.intel.tailscale.users
+from cartography.config import Config
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def start_tailscale_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of Tailscale data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+
+    if not config.tailscale_token or not config.tailscale_org:
+        logger.info(
+            "Tailscale import is not configured - skipping this module. "
+            "See docs to configure.",
+        )
+        return
+
+    # Create requests sessions
+    api_session = requests.session()
+    api_session.headers.update({"Authorization": f"Bearer {config.tailscale_token}"})
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+        "BASE_URL": config.tailscale_base_url,
+        "org": config.tailscale_org,
+    }
+
+    cartography.intel.tailscale.tailnets.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+        org=config.tailscale_org,
+    )
+
+    users = cartography.intel.tailscale.users.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+        org=config.tailscale_org,
+    )
+
+    cartography.intel.tailscale.devices.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+        org=config.tailscale_org,
+    )
+
+    cartography.intel.tailscale.postureintegrations.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+        org=config.tailscale_org,
+    )
+
+    cartography.intel.tailscale.acls.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+        org=config.tailscale_org,
+        users=users,
+    )

cartography/intel/tailscale/acls.py

@@ -0,0 +1,146 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import Tuple
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.tailscale.utils import ACLParser
+from cartography.intel.tailscale.utils import role_to_group
+from cartography.models.tailscale.group import TailscaleGroupSchema
+from cartography.models.tailscale.tag import TailscaleTagSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: Dict[str, Any],
+    org: str,
+    users: List[Dict[str, Any]],
+) -> None:
+    raw_acl = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+        org,
+    )
+    groups, tags = transform(raw_acl, users)
+    load_groups(
+        neo4j_session,
+        groups,
+        common_job_parameters["UPDATE_TAG"],
+        org,
+    )
+    load_tags(
+        neo4j_session,
+        tags,
+        org,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+    org: str,
+) -> str:
+    req = api_session.get(
+        f"{base_url}/tailnet/{org}/acl",
+        timeout=_TIMEOUT,
+    )
+    req.raise_for_status()
+    return req.text
+
+
+def transform(
+    raw_acl: str,
+    users: List[Dict[str, Any]],
+) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
+    transformed_groups: Dict[str, Dict[str, Any]] = {}
+    transformed_tags: Dict[str, Dict[str, Any]] = {}
+
+    parser = ACLParser(raw_acl)
+    # Extract groups from the ACL
+    for group in parser.get_groups():
+        for dom in group["domain_members"]:
+            for user in users:
+                if user["loginName"].endswith(f"@{dom}"):
+                    group["members"].append(user["loginName"])
+        # Ensure domain members are unique
+        group["domain_members"] = list(set(group["domain_members"]))
+        transformed_groups[group["id"]] = group
+    # Extract tags from the ACL
+    for tag in parser.get_tags():
+        for dom in tag["domain_owners"]:
+            for user in users:
+                if user["loginName"].endswith(f"@{dom}"):
+                    tag["owners"].append(user["loginName"])
+        # Ensure domain owners are unique
+        tag["owners"] = list(set(tag["owners"]))
+        transformed_tags[tag["id"]] = tag
+
+    # Add autogroups based on user roles
+    for user in users:
+        for g in role_to_group(user["role"]):
+            if g not in transformed_groups:
+                transformed_groups[g] = {
+                    "id": g,
+                    "name": g.split(":")[-1],
+                    "members": [],
+                    "sub_groups": [],
+                    "domain_members": [],
+                }
+            transformed_groups[g]["members"].append(user["loginName"])
+
+    return list(transformed_groups.values()), list(transformed_tags.values())
+
+
+@timeit
+def load_groups(
+    neo4j_session: neo4j.Session,
+    groups: List[Dict[str, Any]],
+    update_tag: str,
+    org: str,
+) -> None:
+    logger.info(f"Loading {len(groups)} Tailscale Groups to the graph")
+    load(neo4j_session, TailscaleGroupSchema(), groups, lastupdated=update_tag, org=org)
+
+
+@timeit
+def load_tags(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    org: str,
+    update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(data)} Tailscale Tags to the graph")
+    load(
+        neo4j_session,
+        TailscaleTagSchema(),
+        data,
+        lastupdated=update_tag,
+        org=org,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(TailscaleGroupSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(TailscaleTagSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/tailscale/devices.py

@@ -0,0 +1,127 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.tailscale.device import TailscaleDeviceSchema
+from cartography.models.tailscale.tag import TailscaleTagSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: Dict[str, Any],
+    org: str,
+) -> List[Dict]:
+    devices = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+        org,
+    )
+    tags = transform(devices)
+    load_devices(
+        neo4j_session,
+        devices,
+        org,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    load_tags(
+        neo4j_session,
+        tags,
+        org,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+    return devices
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+    org: str,
+) -> List[Dict[str, Any]]:
+    results: List[Dict[str, Any]] = []
+    req = api_session.get(
+        f"{base_url}/tailnet/{org}/devices",
+        timeout=_TIMEOUT,
+    )
+    req.raise_for_status()
+    results = req.json()["devices"]
+    return results
+
+
+def transform(
+    raw_data: List[Dict[str, Any]],
+) -> List[Dict[str, Any]]:
+    """Extracts tags from the raw data and returns a list of dictionaries"""
+    transformed_tags: Dict[str, Dict[str, Any]] = {}
+    # Transform the raw data into the format expected by the load function
+    for device in raw_data:
+        for raw_tag in device.get("tags", []):
+            if raw_tag not in transformed_tags:
+                transformed_tags[raw_tag] = {
+                    "id": raw_tag,
+                    "name": raw_tag.split(":")[-1],
+                    "devices": [device["nodeId"]],
+                }
+            else:
+                transformed_tags[raw_tag]["devices"].append(device["nodeId"])
+    return list(transformed_tags.values())
+
+
+@timeit
+def load_devices(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    org: str,
+    update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(data)} Tailscale Devices to the graph")
+    load(
+        neo4j_session,
+        TailscaleDeviceSchema(),
+        data,
+        lastupdated=update_tag,
+        org=org,
+    )
+
+
+@timeit
+def load_tags(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    org: str,
+    update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(data)} Tailscale Tags to the graph")
+    load(
+        neo4j_session,
+        TailscaleTagSchema(),
+        data,
+        lastupdated=update_tag,
+        org=org,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(TailscaleDeviceSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(TailscaleTagSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/tailscale/postureintegrations.py

@@ -0,0 +1,81 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.tailscale.postureintegration import (
+    TailscalePostureIntegrationSchema,
+)
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: Dict[str, Any],
+    org: str,
+) -> None:
+    postureintegrations = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+        org,
+    )
+    load_postureintegrations(
+        neo4j_session,
+        postureintegrations,
+        org,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+    org: str,
+) -> List[Dict[str, Any]]:
+    results: List[Dict[str, Any]] = []
+    req = api_session.get(
+        f"{base_url}/tailnet/{org}/posture/integrations",
+        timeout=_TIMEOUT,
+    )
+    req.raise_for_status()
+    results = req.json()["integrations"]
+    return results
+
+
+@timeit
+def load_postureintegrations(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    org: str,
+    update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(data)} Tailscale PostureIntegrations to the graph")
+    load(
+        neo4j_session,
+        TailscalePostureIntegrationSchema(),
+        data,
+        lastupdated=update_tag,
+        org=org,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(
+        TailscalePostureIntegrationSchema(), common_job_parameters
+    ).run(neo4j_session)

cartography/intel/tailscale/tailnets.py

@@ -0,0 +1,76 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.tailscale.tailnet import TailscaleTailnetSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: Dict[str, Any],
+    org: str,
+) -> None:
+    tailnet = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+        org,
+    )
+    load_tailnets(
+        neo4j_session,
+        [tailnet],
+        org,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+    org: str,
+) -> Dict[str, Any]:
+    req = api_session.get(
+        f"{base_url}/tailnet/{org}/settings",
+        timeout=_TIMEOUT,
+    )
+    req.raise_for_status()
+    return req.json()
+
+
+@timeit
+def load_tailnets(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    org: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        TailscaleTailnetSchema(),
+        data,
+        lastupdated=update_tag,
+        org=org,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(TailscaleTailnetSchema(), common_job_parameters).run(
+        neo4j_session
+    )