cartography 0.104.0rc2__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/sentinelone/cve.py

@@ -0,0 +1,119 @@
+import logging
+from typing import Any
+
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.sentinelone.api import get_paginated_results
+from cartography.intel.sentinelone.utils import get_application_version_id
+from cartography.models.sentinelone.cve import S1CVESchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get(api_url: str, api_token: str, account_id: str) -> list[dict[str, Any]]:
+    logger.info("Retrieving SentinelOne CVE data")
+    cves = get_paginated_results(
+        api_url=api_url,
+        endpoint="/web/api/v2.1/application-management/risks",
+        api_token=api_token,
+        params={
+            "limit": 1000,
+            "accountIds": account_id,
+        },
+    )
+
+    logger.info("Retrieved %d CVEs from SentinelOne", len(cves))
+    return cves
+
+
+def transform(cves_list: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """
+    Transform SentinelOne CVE data for loading into Neo4j
+    """
+    transformed_cves = []
+    for cve in cves_list:
+        app_version_id = get_application_version_id(
+            cve.get("applicationName", "unknown"),
+            cve.get("applicationVendor", "unknown"),
+            cve.get("applicationVersion", "unknown"),
+        )
+        transformed_cve = {
+            # Required fields - let them fail if missing
+            "id": f"S1|{cve['cveId']}",  # Use CVE ID as the unique identifier for the node
+            "cve_id": cve["cveId"],
+            # Optional fields - use .get() with None default
+            "application_version_id": app_version_id,
+            "base_score": cve.get("baseScore"),
+            "cvss_version": cve.get("cvssVersion"),
+            "published_date": cve.get("publishedDate"),
+            "severity": cve.get("severity"),
+            # Relationship properties
+            "days_detected": cve.get("daysDetected"),
+            "detection_date": cve.get("detectionDate"),
+            "last_scan_date": cve.get("lastScanDate"),
+            "last_scan_result": cve.get("lastScanResult"),
+            "status": cve.get("status"),
+        }
+
+        transformed_cves.append(transformed_cve)
+
+    return transformed_cves
+
+
+@timeit
+def load_cves(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    account_id: str,
+    update_tag: int,
+) -> None:
+    """
+    Load SentinelOne CVE data into Neo4j
+    """
+    logger.info(f"Loading {len(data)} SentinelOne CVEs into Neo4j")
+    load(
+        neo4j_session,
+        S1CVESchema(),
+        data,
+        lastupdated=update_tag,
+        S1_ACCOUNT_ID=account_id,  # Fixed parameter name to match model
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    """Remove CVE nodes that weren't updated in this sync run"""
+    logger.debug("Running S1CVEs cleanup")
+    GraphJob.from_node_schema(S1CVESchema(), common_job_parameters).run(neo4j_session)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    """
+    Sync SentinelOne CVEs following the standard pattern:
+    GET -> TRANSFORM -> LOAD -> CLEANUP
+    """
+    logger.info("Syncing SentinelOne CVE data")
+
+    api_url = common_job_parameters.get("API_URL", "")
+    api_token = common_job_parameters.get("API_TOKEN", "")
+    account_id = common_job_parameters.get("S1_ACCOUNT_ID", "")
+    update_tag = common_job_parameters.get("UPDATE_TAG", 0)
+
+    if not api_url or not api_token or not account_id or not update_tag:
+        logger.error("Missing required parameters for SentinelOne CVE sync")
+        return
+
+    cves = get(api_url, api_token, account_id)
+    transformed_cves = transform(cves)
+    load_cves(neo4j_session, transformed_cves, account_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/sentinelone/utils.py

@@ -0,0 +1,28 @@
+import re
+
+
+def get_application_id(name: str, vendor: str) -> str:
+    """
+    Generate a normalized unique identifier for an application.
+    :param name: Application name
+    :param vendor: Application vendor/publisher
+    :return: Normalized unique identifier in format 'vendor:name'
+    """
+    name_normalized = name.strip().lower().replace(" ", "_")
+    vendor_normalized = vendor.strip().lower().replace(" ", "_")
+    name_normalized = re.sub(r"[^\w]", "", name_normalized)
+    vendor_normalized = re.sub(r"[^\w]", "", vendor_normalized)
+    return f"{vendor_normalized}:{name_normalized}"
+
+
+def get_application_version_id(name: str, vendor: str, version: str) -> str:
+    """
+    Generate a unique identifier for an application version
+    :param name: Application name
+    :param vendor: Application vendor
+    :param version: Application version
+    :return: Unique identifier for the application version in format 'vendor:name:version'
+    """
+    app_id = get_application_id(name, vendor)
+    version_normalized = version.strip().lower().replace(" ", "_")
+    return f"{app_id}:{version_normalized}"

cartography/intel/slack/__init__.py

@@ -0,0 +1,78 @@
+import logging
+
+import neo4j
+from slack_sdk import WebClient
+from slack_sdk.http_retry.builtin_handlers import RateLimitErrorRetryHandler
+
+import cartography.intel.slack.channels
+import cartography.intel.slack.groups
+import cartography.intel.slack.teams
+import cartography.intel.slack.users
+from cartography.config import Config
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def start_slack_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of Slack data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+    if not config.slack_token:
+        logger.info(
+            "Slack import is not configured - skipping this module. "
+            "See docs to configure.",
+        )
+        return
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+        "CHANNELS_MEMBERSHIPS": config.slack_channels_memberships,
+    }
+
+    restricting_teams = []
+    if config.slack_teams:
+        for team in config.slack_teams.split(","):
+            restricting_teams.append(team.strip())
+
+    rate_limit_handler = RateLimitErrorRetryHandler(max_retry_count=1)
+    slack_client = WebClient(token=config.slack_token)
+    slack_client.retry_handlers.append(rate_limit_handler)
+
+    teams_id = cartography.intel.slack.teams.sync(
+        neo4j_session,
+        slack_client,
+        config.update_tag,
+        common_job_parameters,
+    )
+    for team_id in teams_id:
+        if restricting_teams and team_id not in restricting_teams:
+            logger.debug("Skipping team %s", team_id)
+            continue
+        logger.info("Syncing team %s", team_id)
+        common_job_parameters["TEAM_ID"] = team_id
+        cartography.intel.slack.users.sync(
+            neo4j_session,
+            slack_client,
+            team_id,
+            config.update_tag,
+            common_job_parameters,
+        )
+        cartography.intel.slack.channels.sync(
+            neo4j_session,
+            slack_client,
+            team_id,
+            config.update_tag,
+            common_job_parameters,
+        )
+        cartography.intel.slack.groups.sync(
+            neo4j_session,
+            slack_client,
+            team_id,
+            config.update_tag,
+            common_job_parameters,
+        )

cartography/intel/slack/channels.py

@@ -0,0 +1,80 @@
+import logging
+from typing import Any
+
+import neo4j
+from slack_sdk import WebClient
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.slack.utils import slack_paginate
+from cartography.models.slack.channels import SlackChannelSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    slack_client: WebClient,
+    team_id: str,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    channels = get(slack_client, team_id, common_job_parameters["CHANNELS_MEMBERSHIPS"])
+    load_channels(neo4j_session, channels, team_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    slack_client: WebClient, team_id: str, get_memberships: bool
+) -> list[dict[str, Any]]:
+    channels: list[dict[str, Any]] = []
+    for channel in slack_paginate(
+        slack_client,
+        "conversations_list",
+        "channels",
+        team_id=team_id,
+    ):
+        if channel["is_archived"]:
+            channels.append(channel)
+        elif get_memberships:
+            for member in slack_paginate(
+                slack_client,
+                "conversations_members",
+                "members",
+                channel=channel["id"],
+            ):
+                channel_m = channel.copy()
+                channel_m["member_id"] = member
+                channels.append(channel_m)
+        else:
+            channels.append(channel)
+    return channels
+
+
+@timeit
+def load_channels(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    team_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %s Slack channels into Neo4j", len(data))
+    load(
+        neo4j_session,
+        SlackChannelSchema(),
+        data,
+        lastupdated=update_tag,
+        TEAM_ID=team_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(SlackChannelSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/slack/groups.py

@@ -0,0 +1,90 @@
+import logging
+from itertools import zip_longest
+from typing import Any
+
+import neo4j
+from slack_sdk import WebClient
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.slack.utils import slack_paginate
+from cartography.models.slack.group import SlackGroupSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    slack_client: WebClient,
+    team_id: str,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    groups = get(slack_client, team_id)
+    formated_groups = transform(groups)
+    load_groups(neo4j_session, formated_groups, team_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(slack_client: WebClient, team_id: str) -> list[dict[str, Any]]:
+    return slack_paginate(
+        slack_client,
+        "usergroups_list",
+        "usergroups",
+        team_id=team_id,
+        include_count=True,
+        include_users=True,
+        include_disabled=True,
+    )
+
+
+def transform(groups: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    splitted_groups: list[dict[str, Any]] = []
+    for group in groups:
+        # Handle empty or None descriptions
+        if group["description"] is None or len(group["description"]) == 0:
+            group["description"] = None
+        # Ensure the loop runs at least once, even for groups with no members or channels
+        members_and_channels = list(
+            zip_longest(group["users"], group["prefs"]["channels"])
+        )
+        if not members_and_channels:
+            members_and_channels = [(None, None)]
+
+        for ms in members_and_channels:
+            formated_group = group.copy()
+            formated_group.pop("users")
+            formated_group.pop("prefs")
+            formated_group["member_id"] = ms[0]
+            formated_group["channel_id"] = ms[1]
+            splitted_groups.append(formated_group)
+    return splitted_groups
+
+
+@timeit
+def load_groups(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    team_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %s Slack groups into Neo4j", len(data))
+    load(
+        neo4j_session,
+        SlackGroupSchema(),
+        data,
+        lastupdated=update_tag,
+        TEAM_ID=team_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(SlackGroupSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/slack/teams.py

@@ -0,0 +1,65 @@
+import logging
+from typing import Any
+
+import neo4j
+from slack_sdk import WebClient
+
+from cartography.client.core.tx import load
+from cartography.models.slack.team import SlackTeamSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    slack_client: WebClient,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> list[str]:
+    # If no team ID is provided, we fetch the team info for the authenticated team
+    if not common_job_parameters.get("TEAM_ID"):
+        teams = get_teams(slack_client)
+        team_ids = [t["id"] for t in teams.get("teams", [])]
+    else:
+        team_ids = [common_job_parameters["TEAM_ID"]]
+    teams_details = get_teams_details(slack_client, team_ids)
+    load_team(neo4j_session, teams_details, update_tag)
+
+    return team_ids
+
+
+@timeit
+def get_teams(slack_client: WebClient) -> dict[str, Any]:
+    response = slack_client.auth_teams_list()
+    return response.data
+
+
+@timeit
+def get_teams_details(
+    slack_client: WebClient, teams_ids: list[str]
+) -> list[dict[str, Any]]:
+    # Get teams
+    teams: list[dict[str, Any]] = []
+    for team_id in teams_ids:
+        response = slack_client.team_info(team=team_id)
+        team = response.data.get("team")
+        if team:
+            teams.append(team)
+    return teams
+
+
+@timeit
+def load_team(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    update_tag: int,
+) -> None:
+    logger.info("Loading %s Slack teams into Neo4j", len(data))
+    load(
+        neo4j_session,
+        SlackTeamSchema(),
+        data,
+        lastupdated=update_tag,
+    )

cartography/intel/slack/users.py

@@ -0,0 +1,57 @@
+import logging
+from typing import Any
+
+import neo4j
+from slack_sdk import WebClient
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.slack.utils import slack_paginate
+from cartography.models.slack.user import SlackUserSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    slack_client: WebClient,
+    team_id: str,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    users = get(slack_client, team_id)
+    load_users(neo4j_session, users, team_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(slack_client: WebClient, team_id: str) -> list[dict[str, Any]]:
+    return slack_paginate(slack_client, "users_list", "members", team_id=team_id)
+
+
+@timeit
+def load_users(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    team_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %s Slack users into Neo4j", len(data))
+    load(
+        neo4j_session,
+        SlackUserSchema(),
+        data,
+        lastupdated=update_tag,
+        TEAM_ID=team_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(SlackUserSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/slack/utils.py

@@ -0,0 +1,29 @@
+from typing import Any
+
+from slack_sdk import WebClient
+
+
+def slack_paginate(
+    slack_client: WebClient,
+    endpoint: str,
+    data_key: str,
+    **kwargs: Any,
+) -> list[dict[str, Any]]:
+    items: list[dict[str, Any]] = []
+    endpoint_method = getattr(slack_client, endpoint)
+
+    # First query
+    response = endpoint_method(**kwargs)
+    for item in response[data_key]:
+        items.append(item)
+
+    # Iterate over the cursor
+    cursor = response.get("response_metadata", {}).get("next_cursor", "")
+    while cursor != "":
+        kwargs["cursor"] = cursor
+        response = endpoint_method(**kwargs)
+        for item in response[data_key]:
+            items.append(item)
+        cursor = response.get("response_metadata", {}).get("next_cursor", "")
+
+    return items