cartography 0.106.0rc2__py3-none-any.whl → 0.107.0rc2__py3-none-any.whl

cartography/intel/sentinelone/account.py

@@ -0,0 +1,140 @@
+import logging
+from typing import Any
+
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.intel.sentinelone.api import call_sentinelone_api
+from cartography.models.sentinelone.account import S1AccountSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_accounts(
+    api_url: str, api_token: str, account_ids: list[str] | None = None
+) -> list[dict[str, Any]]:
+    """
+    Get account data from SentinelOne API
+    :param api_url: The SentinelOne API URL
+    :param api_token: The SentinelOne API token
+    :param account_ids: Optional list of account IDs to filter for
+    :return: Raw account data from API
+    """
+    logger.info("Retrieving SentinelOne account data")
+
+    # Get accounts info
+    response = call_sentinelone_api(
+        api_url=api_url,
+        endpoint="web/api/v2.1/accounts",
+        api_token=api_token,
+    )
+
+    accounts_data = response.get("data", [])
+
+    # Filter accounts by ID if specified
+    if account_ids:
+        accounts_data = [
+            account for account in accounts_data if account.get("id") in account_ids
+        ]
+        logger.info(f"Filtered accounts data to {len(accounts_data)} matching accounts")
+
+    if accounts_data:
+        logger.info(
+            f"Retrieved SentinelOne account data: {len(accounts_data)} accounts"
+        )
+    else:
+        logger.warning("No SentinelOne accounts retrieved")
+
+    return accounts_data
+
+
+def transform_accounts(accounts_data: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """
+    Transform raw account data into standardized format for Neo4j ingestion
+    :param accounts_data: Raw account data from API
+    :return: List of transformed account data
+    """
+    result: list[dict[str, Any]] = []
+
+    for account in accounts_data:
+        transformed_account = {
+            # Required fields - use direct access (will raise KeyError if missing)
+            "id": account["id"],
+            # Optional fields - use .get() with None default
+            "name": account.get("name"),
+            "account_type": account.get("accountType"),
+            "active_agents": account.get("activeAgents"),
+            "created_at": account.get("createdAt"),
+            "expiration": account.get("expiration"),
+            "number_of_sites": account.get("numberOfSites"),
+            "state": account.get("state"),
+        }
+        result.append(transformed_account)
+
+    return result
+
+
+def load_accounts(
+    neo4j_session: neo4j.Session,
+    accounts_data: list[dict[str, Any]],
+    update_tag: int,
+) -> None:
+    """
+    Load SentinelOne account data into Neo4j using the data model
+    :param neo4j_session: Neo4j session
+    :param accounts_data: List of account data to load
+    :param update_tag: Update tag for tracking data freshness
+    """
+    if not accounts_data:
+        logger.warning("No account data provided to load_accounts")
+        return
+
+    load(
+        neo4j_session,
+        S1AccountSchema(),
+        accounts_data,
+        lastupdated=update_tag,
+        firstseen=update_tag,
+    )
+
+    logger.info(f"Loaded {len(accounts_data)} SentinelOne account nodes")
+
+
+@timeit
+def sync_accounts(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+    account_ids: list[str] | None = None,
+) -> list[str]:
+    """
+    Sync SentinelOne account data using the modern sync pattern
+    :param neo4j_session: Neo4j session
+    :param api_url: SentinelOne API URL
+    :param api_token: SentinelOne API token
+    :param update_tag: Update tag for tracking data freshness
+    :param common_job_parameters: Job parameters for cleanup
+    :param account_ids: Optional list of account IDs to filter for
+    :return: List of synced account IDs
+    """
+    # 1. GET - Fetch data from API
+    accounts_raw_data = get_accounts(
+        common_job_parameters["API_URL"],
+        common_job_parameters["API_TOKEN"],
+        account_ids,
+    )
+
+    # 2. TRANSFORM - Shape data for ingestion
+    transformed_accounts = transform_accounts(accounts_raw_data)
+
+    # 3. LOAD - Ingest to Neo4j using data model
+    load_accounts(
+        neo4j_session,
+        transformed_accounts,
+        common_job_parameters["UPDATE_TAG"],
+    )
+
+    synced_account_ids = [account["id"] for account in transformed_accounts]
+    logger.info(f"Synced {len(synced_account_ids)} SentinelOne accounts")
+    return synced_account_ids

cartography/intel/sentinelone/agent.py

@@ -0,0 +1,139 @@
+import logging
+from typing import Any
+
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.sentinelone.api import get_paginated_results
+from cartography.models.sentinelone.agent import S1AgentSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_agents(api_url: str, api_token: str, account_id: str) -> list[dict[str, Any]]:
+    """
+    Get agent data from SentinelOne API
+    :param api_url: The SentinelOne API URL
+    :param api_token: The SentinelOne API token
+    :param account_id: The SentinelOne account ID
+    :return: Raw agent data from API
+    """
+    logger.info(f"Retrieving SentinelOne agent data for account {account_id}")
+
+    agents = get_paginated_results(
+        api_url=api_url,
+        endpoint="web/api/v2.1/agents",
+        api_token=api_token,
+        params={
+            "accountIds": account_id,
+            "limit": 1000,
+        },
+    )
+
+    logger.info(f"Retrieved {len(agents)} agents from SentinelOne account {account_id}")
+    return agents
+
+
+@timeit
+def transform_agents(agent_list: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """
+    Transform SentinelOne agent data for loading into Neo4j
+    :param agent_list: Raw agent data from the API
+    :return: Transformed agent data
+    """
+    result: list[dict[str, Any]] = []
+
+    for agent in agent_list:
+        transformed_agent = {
+            # Required fields - use direct access (will raise KeyError if missing)
+            "id": agent["id"],
+            # Optional fields - use .get() with None default
+            "uuid": agent.get("uuid"),
+            "computer_name": agent.get("computerName"),
+            "firewall_enabled": agent.get("firewallEnabled"),
+            "os_name": agent.get("osName"),
+            "os_revision": agent.get("osRevision"),
+            "domain": agent.get("domain"),
+            "last_active": agent.get("lastActiveDate"),
+            "last_successful_scan": agent.get("lastSuccessfulScanDate"),
+            "scan_status": agent.get("scanStatus"),
+            "serial_number": agent.get("serialNumber"),
+        }
+        result.append(transformed_agent)
+
+    return result
+
+
+@timeit
+def load_agents(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    account_id: str,
+    update_tag: int,
+) -> None:
+    """
+    Load SentinelOne agent data into Neo4j
+    :param neo4j_session: Neo4j session
+    :param data: The transformed agent data
+    :param account_id: The SentinelOne account ID
+    :param update_tag: Update tag for tracking data freshness
+    :return: None
+    """
+    logger.info(
+        f"Loading {len(data)} SentinelOne agents into Neo4j for account {account_id}"
+    )
+    load(
+        neo4j_session,
+        S1AgentSchema(),
+        data,
+        lastupdated=update_tag,
+        S1_ACCOUNT_ID=account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    """
+    Remove stale SentinelOne agent data from Neo4j
+    :param neo4j_session: Neo4j session
+    :param common_job_parameters: Common job parameters for cleanup
+    :return: None
+    """
+    logger.debug("Running S1Agent cleanup job")
+    GraphJob.from_node_schema(S1AgentSchema(), common_job_parameters).run(neo4j_session)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    """
+    Sync SentinelOne agents using the standard sync pattern
+    :param neo4j_session: Neo4j session
+    :param common_job_parameters: Common job parameters containing API_URL, API_TOKEN, S1_ACCOUNT_ID, UPDATE_TAG
+    :return: None
+    """
+    api_url = common_job_parameters["API_URL"]
+    api_token = common_job_parameters["API_TOKEN"]
+    account_id = common_job_parameters["S1_ACCOUNT_ID"]
+    update_tag = common_job_parameters["UPDATE_TAG"]
+
+    logger.info(f"Syncing SentinelOne agent data for account {account_id}")
+
+    # 1. GET - Fetch data from API
+    agents_raw_data = get_agents(api_url, api_token, account_id)
+
+    # 2. TRANSFORM - Shape data for ingestion
+    transformed_data = transform_agents(agents_raw_data)
+
+    # 3. LOAD - Ingest to Neo4j using data model
+    load_agents(neo4j_session, transformed_data, account_id, update_tag)
+
+    # 4. CLEANUP - Remove stale data
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/sentinelone/api.py

@@ -0,0 +1,113 @@
+import logging
+from typing import Any
+
+import requests
+
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def call_sentinelone_api(
+    api_url: str,
+    endpoint: str,
+    api_token: str,
+    method: str = "GET",
+    params: dict[str, Any] | None = None,
+    data: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """
+    Call the SentinelOne API
+    :param api_url: The base URL for the SentinelOne API
+    :param endpoint: The API endpoint to call
+    :param api_token: The API token for authentication
+    :param method: The HTTP method to use (default is GET)
+    :param params: Query parameters to include in the request
+    :param data: Data to include in the request body for POST/PUT methods
+    :return: The JSON response from the API
+    """
+    full_url = f"{api_url.rstrip('/')}/{endpoint.lstrip('/')}"
+
+    headers = {
+        "Accept": "application/json",
+        "Authorization": f"ApiToken {api_token}",
+        "Content-Type": "application/json",
+    }
+
+    try:
+        logger.debug(
+            "SentinelOne: %s %s",
+            method,
+            full_url,
+        )
+
+        response = requests.request(
+            method=method,
+            url=full_url,
+            headers=headers,
+            params=params,
+            json=data,
+            timeout=_TIMEOUT,
+        )
+
+        # Raise an exception for HTTP errors
+        response.raise_for_status()
+
+    except requests.exceptions.Timeout:
+        logger.warning(f"SentinelOne: Request to '{full_url}' timed out.")
+        raise
+    except requests.exceptions.RequestException as e:
+        logger.error(f"SentinelOne API request failed: {e}")
+        raise
+
+    return response.json()
+
+
+def get_paginated_results(
+    api_url: str,
+    endpoint: str,
+    api_token: str,
+    params: dict[str, Any] | None = None,
+) -> list[dict[str, Any]]:
+    """
+    Handle cursor-based pagination for SentinelOne API requests
+    :param api_url: The base URL for the SentinelOne API
+    :param endpoint: The API endpoint to call
+    :param api_token: The API token for authentication
+    :param params: Query parameters to include in the request
+    :return: A list of all items from all pages
+    """
+    query_params = params or {}
+
+    # Set default pagination parameters if not provided
+    if "limit" not in query_params:
+        query_params["limit"] = 100
+
+    next_cursor = None
+    total_items = []
+
+    while True:
+        if next_cursor:
+            query_params["cursor"] = next_cursor
+
+        response = call_sentinelone_api(
+            api_url=api_url,
+            endpoint=endpoint,
+            api_token=api_token,
+            params=query_params,
+        )
+
+        items = response.get("data", [])
+        if not items:
+            break
+
+        total_items.extend(items)
+        pagination = response.get("pagination", {})
+        next_cursor = pagination.get("nextCursor")
+        if not next_cursor:
+            break
+
+    return total_items

cartography/intel/sentinelone/utils.py

@@ -0,0 +1,9 @@
+import re
+
+
+def get_application_id(name: str, vendor: str) -> str:
+    name_normalized = name.strip().lower().replace(" ", "_")
+    vendor_normalized = vendor.strip().lower().replace(" ", "_")
+    name_normalized = re.sub(r"[^\w]", "", name_normalized)
+    vendor_normalized = re.sub(r"[^\w\s]", "", vendor_normalized)
+    return f"{vendor_normalized}:{name_normalized}"

cartography/models/airbyte/connection.py

@@ -0,0 +1,138 @@
+from dataclasses import dataclass
+
+from cartography.models.core.common import PropertyRef
+from cartography.models.core.nodes import CartographyNodeProperties
+from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.relationships import CartographyRelProperties
+from cartography.models.core.relationships import CartographyRelSchema
+from cartography.models.core.relationships import LinkDirection
+from cartography.models.core.relationships import make_target_node_matcher
+from cartography.models.core.relationships import OtherRelationships
+from cartography.models.core.relationships import TargetNodeMatcher
+
+
+@dataclass(frozen=True)
+class AirbyteConnectionNodeProperties(CartographyNodeProperties):
+    id: PropertyRef = PropertyRef("connectionId")
+    name: PropertyRef = PropertyRef("name")
+    namespace_format: PropertyRef = PropertyRef("namespaceFormat")
+    prefix: PropertyRef = PropertyRef("prefix")
+    status: PropertyRef = PropertyRef("status")
+    data_residency: PropertyRef = PropertyRef("dataResidency")
+    non_breaking_schema_updates_behavior: PropertyRef = PropertyRef(
+        "nonBreakingSchemaUpdatesBehavior"
+    )
+    namespace_definition: PropertyRef = PropertyRef("namespaceDefinition")
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class AirbyteConnectionToOrganizationRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+# (:AirbyteOrganization)-[:RESOURCE]->(:AirbyteConnection)
+class AirbyteConnectionToOrganizationRel(CartographyRelSchema):
+    target_node_label: str = "AirbyteOrganization"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {"id": PropertyRef("ORG_ID", set_in_kwargs=True)},
+    )
+    direction: LinkDirection = LinkDirection.INWARD
+    rel_label: str = "RESOURCE"
+    properties: AirbyteConnectionToOrganizationRelProperties = (
+        AirbyteConnectionToOrganizationRelProperties()
+    )
+
+
+@dataclass(frozen=True)
+class AirbyteConnectionToWorkspaceRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+# (:AirbyteWorkspace)-[:CONTAINS]->(:AirbyteConnection)
+class AirbyteConnectionToWorkspaceRel(CartographyRelSchema):
+    target_node_label: str = "AirbyteWorkspace"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {"id": PropertyRef("workspaceId")},
+    )
+    direction: LinkDirection = LinkDirection.INWARD
+    rel_label: str = "CONTAINS"
+    properties: AirbyteConnectionToWorkspaceRelProperties = (
+        AirbyteConnectionToWorkspaceRelProperties()
+    )
+
+
+@dataclass(frozen=True)
+class AirbyteConnectionToSourceRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+# (:AirbyteSource)<-[:SYNC_FROM]-(:AirbyteConnection)
+class AirbyteConnectionToSourceRel(CartographyRelSchema):
+    target_node_label: str = "AirbyteSource"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {"id": PropertyRef("sourceId")},
+    )
+    direction: LinkDirection = LinkDirection.OUTWARD
+    rel_label: str = "SYNC_FROM"
+    properties: AirbyteConnectionToSourceRelProperties = (
+        AirbyteConnectionToSourceRelProperties()
+    )
+
+
+@dataclass(frozen=True)
+class AirbyteConnectionToDestinationRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+# (:AirbyteDestination)<-[:SYNC_TO]-(:AirbyteConnection)
+class AirbyteConnectionToDestinationRel(CartographyRelSchema):
+    target_node_label: str = "AirbyteDestination"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {"id": PropertyRef("destinationId")},
+    )
+    direction: LinkDirection = LinkDirection.OUTWARD
+    rel_label: str = "SYNC_TO"
+    properties: AirbyteConnectionToDestinationRelProperties = (
+        AirbyteConnectionToDestinationRelProperties()
+    )
+
+
+@dataclass(frozen=True)
+class AirbyteConnectionToTagRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+# (:AirbyteTag)<-[:TAGGED]-(:AirbyteConnection)
+class AirbyteConnectionToTagRel(CartographyRelSchema):
+    target_node_label: str = "AirbyteTag"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {"id": PropertyRef("tags_ids", one_to_many=True)},
+    )
+    direction: LinkDirection = LinkDirection.OUTWARD
+    rel_label: str = "TAGGED"
+    properties: AirbyteConnectionToTagRelProperties = (
+        AirbyteConnectionToTagRelProperties()
+    )
+
+
+@dataclass(frozen=True)
+class AirbyteConnectionSchema(CartographyNodeSchema):
+    label: str = "AirbyteConnection"
+    properties: AirbyteConnectionNodeProperties = AirbyteConnectionNodeProperties()
+    sub_resource_relationship: AirbyteConnectionToOrganizationRel = (
+        AirbyteConnectionToOrganizationRel()
+    )
+    other_relationships: OtherRelationships = OtherRelationships(
+        [
+            AirbyteConnectionToWorkspaceRel(),
+            AirbyteConnectionToSourceRel(),
+            AirbyteConnectionToDestinationRel(),
+            AirbyteConnectionToTagRel(),
+        ]
+    )

cartography/models/airbyte/destination.py

@@ -0,0 +1,75 @@
+from dataclasses import dataclass
+
+from cartography.models.core.common import PropertyRef
+from cartography.models.core.nodes import CartographyNodeProperties
+from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.relationships import CartographyRelProperties
+from cartography.models.core.relationships import CartographyRelSchema
+from cartography.models.core.relationships import LinkDirection
+from cartography.models.core.relationships import make_target_node_matcher
+from cartography.models.core.relationships import OtherRelationships
+from cartography.models.core.relationships import TargetNodeMatcher
+
+
+@dataclass(frozen=True)
+class AirbyteDestinationNodeProperties(CartographyNodeProperties):
+    id: PropertyRef = PropertyRef("destinationId")
+    name: PropertyRef = PropertyRef("name")
+    type: PropertyRef = PropertyRef("destinationType")
+    config_host: PropertyRef = PropertyRef("configuration.host")
+    config_port: PropertyRef = PropertyRef("configuration.port")
+    config_name: PropertyRef = PropertyRef("configuration.name")
+    config_region: PropertyRef = PropertyRef("configuration.region")
+    config_endpoint: PropertyRef = PropertyRef("configuration.endpoint")
+    config_account: PropertyRef = PropertyRef("configuration.account")
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class AirbyteDestinationToOrganizationRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+# (:AirbyteOrganization)-[:RESOURCE]->(:AirbyteDestination)
+class AirbyteDestinationToOrganizationRel(CartographyRelSchema):
+    target_node_label: str = "AirbyteOrganization"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {"id": PropertyRef("ORG_ID", set_in_kwargs=True)},
+    )
+    direction: LinkDirection = LinkDirection.INWARD
+    rel_label: str = "RESOURCE"
+    properties: AirbyteDestinationToOrganizationRelProperties = (
+        AirbyteDestinationToOrganizationRelProperties()
+    )
+
+
+@dataclass(frozen=True)
+class AirbyteDestinationToWorkspaceRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+# (:AirbyteWorkspace)-[:CONTAINS]->(:AirbyteDestination)
+class AirbyteDestinationToWorkspaceRel(CartographyRelSchema):
+    target_node_label: str = "AirbyteWorkspace"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {"id": PropertyRef("workspaceId")},
+    )
+    direction: LinkDirection = LinkDirection.INWARD
+    rel_label: str = "CONTAINS"
+    properties: AirbyteDestinationToWorkspaceRelProperties = (
+        AirbyteDestinationToWorkspaceRelProperties()
+    )
+
+
+@dataclass(frozen=True)
+class AirbyteDestinationSchema(CartographyNodeSchema):
+    label: str = "AirbyteDestination"
+    properties: AirbyteDestinationNodeProperties = AirbyteDestinationNodeProperties()
+    sub_resource_relationship: AirbyteDestinationToOrganizationRel = (
+        AirbyteDestinationToOrganizationRel()
+    )
+    other_relationships: OtherRelationships = OtherRelationships(
+        [AirbyteDestinationToWorkspaceRel()]
+    )

cartography/models/airbyte/organization.py

@@ -0,0 +1,19 @@
+from dataclasses import dataclass
+
+from cartography.models.core.common import PropertyRef
+from cartography.models.core.nodes import CartographyNodeProperties
+from cartography.models.core.nodes import CartographyNodeSchema
+
+
+@dataclass(frozen=True)
+class AirbyteOrganizationNodeProperties(CartographyNodeProperties):
+    id: PropertyRef = PropertyRef("organizationId")
+    name: PropertyRef = PropertyRef("organizationName")
+    email: PropertyRef = PropertyRef("email")
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class AirbyteOrganizationSchema(CartographyNodeSchema):
+    label: str = "AirbyteOrganization"
+    properties: AirbyteOrganizationNodeProperties = AirbyteOrganizationNodeProperties()