cartography 0.106.0rc1__py3-none-any.whl → 0.107.0__py3-none-any.whl

cartography/intel/sentinelone/agent.py

@@ -0,0 +1,139 @@
+import logging
+from typing import Any
+
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.sentinelone.api import get_paginated_results
+from cartography.models.sentinelone.agent import S1AgentSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_agents(api_url: str, api_token: str, account_id: str) -> list[dict[str, Any]]:
+    """
+    Get agent data from SentinelOne API
+    :param api_url: The SentinelOne API URL
+    :param api_token: The SentinelOne API token
+    :param account_id: The SentinelOne account ID
+    :return: Raw agent data from API
+    """
+    logger.info(f"Retrieving SentinelOne agent data for account {account_id}")
+
+    agents = get_paginated_results(
+        api_url=api_url,
+        endpoint="web/api/v2.1/agents",
+        api_token=api_token,
+        params={
+            "accountIds": account_id,
+            "limit": 1000,
+        },
+    )
+
+    logger.info(f"Retrieved {len(agents)} agents from SentinelOne account {account_id}")
+    return agents
+
+
+@timeit
+def transform_agents(agent_list: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """
+    Transform SentinelOne agent data for loading into Neo4j
+    :param agent_list: Raw agent data from the API
+    :return: Transformed agent data
+    """
+    result: list[dict[str, Any]] = []
+
+    for agent in agent_list:
+        transformed_agent = {
+            # Required fields - use direct access (will raise KeyError if missing)
+            "id": agent["id"],
+            # Optional fields - use .get() with None default
+            "uuid": agent.get("uuid"),
+            "computer_name": agent.get("computerName"),
+            "firewall_enabled": agent.get("firewallEnabled"),
+            "os_name": agent.get("osName"),
+            "os_revision": agent.get("osRevision"),
+            "domain": agent.get("domain"),
+            "last_active": agent.get("lastActiveDate"),
+            "last_successful_scan": agent.get("lastSuccessfulScanDate"),
+            "scan_status": agent.get("scanStatus"),
+            "serial_number": agent.get("serialNumber"),
+        }
+        result.append(transformed_agent)
+
+    return result
+
+
+@timeit
+def load_agents(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    account_id: str,
+    update_tag: int,
+) -> None:
+    """
+    Load SentinelOne agent data into Neo4j
+    :param neo4j_session: Neo4j session
+    :param data: The transformed agent data
+    :param account_id: The SentinelOne account ID
+    :param update_tag: Update tag for tracking data freshness
+    :return: None
+    """
+    logger.info(
+        f"Loading {len(data)} SentinelOne agents into Neo4j for account {account_id}"
+    )
+    load(
+        neo4j_session,
+        S1AgentSchema(),
+        data,
+        lastupdated=update_tag,
+        S1_ACCOUNT_ID=account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    """
+    Remove stale SentinelOne agent data from Neo4j
+    :param neo4j_session: Neo4j session
+    :param common_job_parameters: Common job parameters for cleanup
+    :return: None
+    """
+    logger.debug("Running S1Agent cleanup job")
+    GraphJob.from_node_schema(S1AgentSchema(), common_job_parameters).run(neo4j_session)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    """
+    Sync SentinelOne agents using the standard sync pattern
+    :param neo4j_session: Neo4j session
+    :param common_job_parameters: Common job parameters containing API_URL, API_TOKEN, S1_ACCOUNT_ID, UPDATE_TAG
+    :return: None
+    """
+    api_url = common_job_parameters["API_URL"]
+    api_token = common_job_parameters["API_TOKEN"]
+    account_id = common_job_parameters["S1_ACCOUNT_ID"]
+    update_tag = common_job_parameters["UPDATE_TAG"]
+
+    logger.info(f"Syncing SentinelOne agent data for account {account_id}")
+
+    # 1. GET - Fetch data from API
+    agents_raw_data = get_agents(api_url, api_token, account_id)
+
+    # 2. TRANSFORM - Shape data for ingestion
+    transformed_data = transform_agents(agents_raw_data)
+
+    # 3. LOAD - Ingest to Neo4j using data model
+    load_agents(neo4j_session, transformed_data, account_id, update_tag)
+
+    # 4. CLEANUP - Remove stale data
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/sentinelone/api.py

@@ -0,0 +1,113 @@
+import logging
+from typing import Any
+
+import requests
+
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def call_sentinelone_api(
+    api_url: str,
+    endpoint: str,
+    api_token: str,
+    method: str = "GET",
+    params: dict[str, Any] | None = None,
+    data: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """
+    Call the SentinelOne API
+    :param api_url: The base URL for the SentinelOne API
+    :param endpoint: The API endpoint to call
+    :param api_token: The API token for authentication
+    :param method: The HTTP method to use (default is GET)
+    :param params: Query parameters to include in the request
+    :param data: Data to include in the request body for POST/PUT methods
+    :return: The JSON response from the API
+    """
+    full_url = f"{api_url.rstrip('/')}/{endpoint.lstrip('/')}"
+
+    headers = {
+        "Accept": "application/json",
+        "Authorization": f"ApiToken {api_token}",
+        "Content-Type": "application/json",
+    }
+
+    try:
+        logger.debug(
+            "SentinelOne: %s %s",
+            method,
+            full_url,
+        )
+
+        response = requests.request(
+            method=method,
+            url=full_url,
+            headers=headers,
+            params=params,
+            json=data,
+            timeout=_TIMEOUT,
+        )
+
+        # Raise an exception for HTTP errors
+        response.raise_for_status()
+
+    except requests.exceptions.Timeout:
+        logger.warning(f"SentinelOne: Request to '{full_url}' timed out.")
+        raise
+    except requests.exceptions.RequestException as e:
+        logger.error(f"SentinelOne API request failed: {e}")
+        raise
+
+    return response.json()
+
+
+def get_paginated_results(
+    api_url: str,
+    endpoint: str,
+    api_token: str,
+    params: dict[str, Any] | None = None,
+) -> list[dict[str, Any]]:
+    """
+    Handle cursor-based pagination for SentinelOne API requests
+    :param api_url: The base URL for the SentinelOne API
+    :param endpoint: The API endpoint to call
+    :param api_token: The API token for authentication
+    :param params: Query parameters to include in the request
+    :return: A list of all items from all pages
+    """
+    query_params = params or {}
+
+    # Set default pagination parameters if not provided
+    if "limit" not in query_params:
+        query_params["limit"] = 100
+
+    next_cursor = None
+    total_items = []
+
+    while True:
+        if next_cursor:
+            query_params["cursor"] = next_cursor
+
+        response = call_sentinelone_api(
+            api_url=api_url,
+            endpoint=endpoint,
+            api_token=api_token,
+            params=query_params,
+        )
+
+        items = response.get("data", [])
+        if not items:
+            break
+
+        total_items.extend(items)
+        pagination = response.get("pagination", {})
+        next_cursor = pagination.get("nextCursor")
+        if not next_cursor:
+            break
+
+    return total_items

cartography/intel/sentinelone/application.py

@@ -0,0 +1,248 @@
+import logging
+from typing import Any
+
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.sentinelone.api import get_paginated_results
+from cartography.intel.sentinelone.utils import get_application_id
+from cartography.intel.sentinelone.utils import get_application_version_id
+from cartography.models.sentinelone.application import S1ApplicationSchema
+from cartography.models.sentinelone.application_version import (
+    S1ApplicationVersionSchema,
+)
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_application_data(
+    account_id: str, api_url: str, api_token: str
+) -> list[dict[str, Any]]:
+    """
+    Get application data from SentinelOne API
+    :param account_id: SentinelOne account ID
+    :param api_url: SentinelOne API URL
+    :param api_token: SentinelOne API token
+    :return: A list of application data dictionaries
+    """
+    logger.info(f"Retrieving SentinelOne application data for account {account_id}")
+    applications = get_paginated_results(
+        api_url=api_url,
+        endpoint="/web/api/v2.1/application-management/inventory",
+        api_token=api_token,
+        params={
+            "accountIds": account_id,
+            "limit": 1000,
+        },
+    )
+
+    logger.info(f"Retrieved {len(applications)} applications from SentinelOne")
+    return applications
+
+
+@timeit
+def get_application_installs(
+    app_inventory: list[dict[str, Any]], account_id: str, api_url: str, api_token: str
+) -> list[dict[str, Any]]:
+    """
+    Get application installs from SentinelOne API
+    :param app_inventory: List of applications to get installs for
+    :param account_id: SentinelOne account ID
+    :param api_url: SentinelOne API URL
+    :param api_token: SentinelOne API token
+    :return: A list of application installs data dictionaries
+    """
+    logger.info(
+        f"Retrieving SentinelOne application installs for "
+        f"{len(app_inventory)} applications in account "
+        f"{account_id}",
+    )
+
+    application_installs = []
+    for i, app in enumerate(app_inventory):
+        logger.info(
+            f"Retrieving SentinelOne installs for {app.get('applicationName')} "
+            f"{app.get('applicationVendor')} ({i + 1}/{len(app_inventory)})",
+        )
+        name = app["applicationName"]
+        vendor = app["applicationVendor"]
+        app_installs = get_paginated_results(
+            api_url=api_url,
+            endpoint="/web/api/v2.1/application-management/inventory/endpoints",
+            api_token=api_token,
+            params={
+                "accountIds": account_id,
+                "limit": 1000,
+                "applicationName": name,
+                "applicationVendor": vendor,
+            },
+        )
+
+        # Replace applicationVendor and applicationName with original values
+        # for consistency with the application data
+        for app_install in app_installs:
+            app_install["applicationVendor"] = vendor
+            app_install["applicationName"] = name
+        application_installs.extend(app_installs)
+
+    return application_installs
+
+
+def transform_applications(
+    applications_list: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+    """
+    Transform SentinelOne application data for loading into Neo4j
+    :param applications_list: Raw application data from the API
+    :return: Transformed application data
+    """
+    transformed_data = []
+    for app in applications_list:
+        vendor = app["applicationVendor"].strip()
+        name = app["applicationName"].strip()
+        app_id = get_application_id(name, vendor)
+        transformed_app = {
+            "id": app_id,
+            "name": name,
+            "vendor": vendor,
+        }
+        transformed_data.append(transformed_app)
+
+    return transformed_data
+
+
+def transform_application_versions(
+    application_installs_list: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+    """
+    Transform SentinelOne application installs for loading into Neo4j
+    :param application_installs_list: Raw application installs data from the API
+    :return: Transformed application installs data
+    """
+    transformed_data = []
+    for installed_version in application_installs_list:
+        app_name = installed_version["applicationName"]
+        app_vendor = installed_version["applicationVendor"]
+        version = installed_version["version"]
+
+        transformed_version = {
+            "id": get_application_version_id(
+                app_name,
+                app_vendor,
+                version,
+            ),
+            "application_id": get_application_id(
+                app_name,
+                app_vendor,
+            ),
+            "application_name": app_name,
+            "application_vendor": app_vendor,
+            "agent_uuid": installed_version.get("endpointUuid"),
+            "installation_path": installed_version.get("applicationInstallationPath"),
+            "installed_dt": installed_version.get("applicationInstallationDate"),
+            "version": version,
+        }
+        transformed_data.append(transformed_version)
+
+    return transformed_data
+
+
+@timeit
+def load_application_data(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    account_id: str,
+    update_tag: int,
+) -> None:
+    """
+    Load SentinelOne application data into Neo4j
+    :param neo4j_session: Neo4j session
+    :param data: The transformed application data
+    :param account_id: The SentinelOne account ID
+    :param update_tag: Update tag to set on the nodes
+    :return: None
+    """
+    logger.info(f"Loading {len(data)} SentinelOne applications into Neo4j")
+    load(
+        neo4j_session,
+        S1ApplicationSchema(),
+        data,
+        lastupdated=update_tag,
+        S1_ACCOUNT_ID=account_id,
+    )
+
+
+@timeit
+def load_application_versions(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    account_id: str,
+    update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(data)} SentinelOne application versions into Neo4j")
+    load(
+        neo4j_session,
+        S1ApplicationVersionSchema(),
+        data,
+        lastupdated=update_tag,
+        S1_ACCOUNT_ID=account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    logger.debug("Running SentinelOne S1Application cleanup")
+    GraphJob.from_node_schema(S1ApplicationSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    logger.debug("Running SentinelOne S1ApplicationVersion cleanup")
+    GraphJob.from_node_schema(S1ApplicationVersionSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    """
+    Sync SentinelOne applications
+    :param neo4j_session: Neo4j session
+    :param common_job_parameters: Common job parameters containing API_URL, API_TOKEN, etc.
+    :return: None
+    """
+    logger.info("Syncing SentinelOne application data")
+    account_id = str(common_job_parameters["S1_ACCOUNT_ID"])
+    api_url = str(common_job_parameters["API_URL"])
+    api_token = str(common_job_parameters["API_TOKEN"])
+
+    applications = get_application_data(account_id, api_url, api_token)
+    application_versions = get_application_installs(
+        applications, account_id, api_url, api_token
+    )
+    transformed_applications = transform_applications(applications)
+    transformed_application_versions = transform_application_versions(
+        application_versions
+    )
+
+    load_application_data(
+        neo4j_session,
+        transformed_applications,
+        account_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+
+    load_application_versions(
+        neo4j_session,
+        transformed_application_versions,
+        account_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/sentinelone/utils.py

@@ -0,0 +1,28 @@
+import re
+
+
+def get_application_id(name: str, vendor: str) -> str:
+    """
+    Generate a normalized unique identifier for an application.
+    :param name: Application name
+    :param vendor: Application vendor/publisher
+    :return: Normalized unique identifier in format 'vendor:name'
+    """
+    name_normalized = name.strip().lower().replace(" ", "_")
+    vendor_normalized = vendor.strip().lower().replace(" ", "_")
+    name_normalized = re.sub(r"[^\w]", "", name_normalized)
+    vendor_normalized = re.sub(r"[^\w]", "", vendor_normalized)
+    return f"{vendor_normalized}:{name_normalized}"
+
+
+def get_application_version_id(name: str, vendor: str, version: str) -> str:
+    """
+    Generate a unique identifier for an application version
+    :param name: Application name
+    :param vendor: Application vendor
+    :param version: Application version
+    :return: Unique identifier for the application version in format 'vendor:name:version'
+    """
+    app_id = get_application_id(name, vendor)
+    version_normalized = version.strip().lower().replace(" ", "_")
+    return f"{app_id}:{version_normalized}"

cartography/models/airbyte/connection.py

@@ -0,0 +1,138 @@
+from dataclasses import dataclass
+
+from cartography.models.core.common import PropertyRef
+from cartography.models.core.nodes import CartographyNodeProperties
+from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.relationships import CartographyRelProperties
+from cartography.models.core.relationships import CartographyRelSchema
+from cartography.models.core.relationships import LinkDirection
+from cartography.models.core.relationships import make_target_node_matcher
+from cartography.models.core.relationships import OtherRelationships
+from cartography.models.core.relationships import TargetNodeMatcher
+
+
+@dataclass(frozen=True)
+class AirbyteConnectionNodeProperties(CartographyNodeProperties):
+    id: PropertyRef = PropertyRef("connectionId")
+    name: PropertyRef = PropertyRef("name")
+    namespace_format: PropertyRef = PropertyRef("namespaceFormat")
+    prefix: PropertyRef = PropertyRef("prefix")
+    status: PropertyRef = PropertyRef("status")
+    data_residency: PropertyRef = PropertyRef("dataResidency")
+    non_breaking_schema_updates_behavior: PropertyRef = PropertyRef(
+        "nonBreakingSchemaUpdatesBehavior"
+    )
+    namespace_definition: PropertyRef = PropertyRef("namespaceDefinition")
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class AirbyteConnectionToOrganizationRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+# (:AirbyteOrganization)-[:RESOURCE]->(:AirbyteConnection)
+class AirbyteConnectionToOrganizationRel(CartographyRelSchema):
+    target_node_label: str = "AirbyteOrganization"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {"id": PropertyRef("ORG_ID", set_in_kwargs=True)},
+    )
+    direction: LinkDirection = LinkDirection.INWARD
+    rel_label: str = "RESOURCE"
+    properties: AirbyteConnectionToOrganizationRelProperties = (
+        AirbyteConnectionToOrganizationRelProperties()
+    )
+
+
+@dataclass(frozen=True)
+class AirbyteConnectionToWorkspaceRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+# (:AirbyteWorkspace)-[:CONTAINS]->(:AirbyteConnection)
+class AirbyteConnectionToWorkspaceRel(CartographyRelSchema):
+    target_node_label: str = "AirbyteWorkspace"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {"id": PropertyRef("workspaceId")},
+    )
+    direction: LinkDirection = LinkDirection.INWARD
+    rel_label: str = "CONTAINS"
+    properties: AirbyteConnectionToWorkspaceRelProperties = (
+        AirbyteConnectionToWorkspaceRelProperties()
+    )
+
+
+@dataclass(frozen=True)
+class AirbyteConnectionToSourceRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+# (:AirbyteSource)<-[:SYNC_FROM]-(:AirbyteConnection)
+class AirbyteConnectionToSourceRel(CartographyRelSchema):
+    target_node_label: str = "AirbyteSource"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {"id": PropertyRef("sourceId")},
+    )
+    direction: LinkDirection = LinkDirection.OUTWARD
+    rel_label: str = "SYNC_FROM"
+    properties: AirbyteConnectionToSourceRelProperties = (
+        AirbyteConnectionToSourceRelProperties()
+    )
+
+
+@dataclass(frozen=True)
+class AirbyteConnectionToDestinationRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+# (:AirbyteDestination)<-[:SYNC_TO]-(:AirbyteConnection)
+class AirbyteConnectionToDestinationRel(CartographyRelSchema):
+    target_node_label: str = "AirbyteDestination"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {"id": PropertyRef("destinationId")},
+    )
+    direction: LinkDirection = LinkDirection.OUTWARD
+    rel_label: str = "SYNC_TO"
+    properties: AirbyteConnectionToDestinationRelProperties = (
+        AirbyteConnectionToDestinationRelProperties()
+    )
+
+
+@dataclass(frozen=True)
+class AirbyteConnectionToTagRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+# (:AirbyteTag)<-[:TAGGED]-(:AirbyteConnection)
+class AirbyteConnectionToTagRel(CartographyRelSchema):
+    target_node_label: str = "AirbyteTag"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {"id": PropertyRef("tags_ids", one_to_many=True)},
+    )
+    direction: LinkDirection = LinkDirection.OUTWARD
+    rel_label: str = "TAGGED"
+    properties: AirbyteConnectionToTagRelProperties = (
+        AirbyteConnectionToTagRelProperties()
+    )
+
+
+@dataclass(frozen=True)
+class AirbyteConnectionSchema(CartographyNodeSchema):
+    label: str = "AirbyteConnection"
+    properties: AirbyteConnectionNodeProperties = AirbyteConnectionNodeProperties()
+    sub_resource_relationship: AirbyteConnectionToOrganizationRel = (
+        AirbyteConnectionToOrganizationRel()
+    )
+    other_relationships: OtherRelationships = OtherRelationships(
+        [
+            AirbyteConnectionToWorkspaceRel(),
+            AirbyteConnectionToSourceRel(),
+            AirbyteConnectionToDestinationRel(),
+            AirbyteConnectionToTagRel(),
+        ]
+    )