cartography 0.106.0rc2__py3-none-any.whl → 0.107.0rc1__py3-none-any.whl

cartography/intel/scaleway/instances/instances.py

@@ -0,0 +1,92 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.instance.v1 import InstanceV1API
+from scaleway.instance.v1 import Server
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import DEFAULT_ZONE
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.instance.instance import ScalewayInstanceSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    projects_id: list[str],
+    update_tag: int,
+) -> None:
+    instances = get(client, org_id)
+    instances_by_project = transform_instances(instances)
+    load_instances(neo4j_session, instances_by_project, update_tag)
+    cleanup(neo4j_session, projects_id, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Server]:
+    api = InstanceV1API(client)
+    return api.list_servers_all(organization=org_id, zone=DEFAULT_ZONE)
+
+
+def transform_instances(
+    instances: list[Server],
+) -> dict[str, list[dict[str, Any]]]:
+    result: dict[str, list[dict[str, Any]]] = {}
+    for instance in instances:
+        project_id = instance.project
+        formatted_instance = scaleway_obj_to_dict(instance)
+        formatted_instance["public_ips"] = [
+            ip["id"] for ip in formatted_instance.get("public_ips", [])
+        ]
+        formatted_instance["volumes_id"] = [
+            volume["id"] for volume in formatted_instance.get("volumes", {}).values()
+        ]
+        result.setdefault(project_id, []).append(formatted_instance)
+    return result
+
+
+@timeit
+def load_instances(
+    neo4j_session: neo4j.Session,
+    data: dict[str, list[dict[str, Any]]],
+    update_tag: int,
+) -> None:
+    for project_id, instances in data.items():
+        logger.info(
+            "Loading %d Scaleway Instance in project '%s' into Neo4j.",
+            len(instances),
+            project_id,
+        )
+        load(
+            neo4j_session,
+            ScalewayInstanceSchema(),
+            instances,
+            lastupdated=update_tag,
+            PROJECT_ID=project_id,
+        )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    projects_id: list[str],
+    common_job_parameters: dict[str, Any],
+) -> None:
+    for project_id in projects_id:
+        scopped_job_parameters = common_job_parameters.copy()
+        scopped_job_parameters["PROJECT_ID"] = project_id
+        GraphJob.from_node_schema(ScalewayInstanceSchema(), scopped_job_parameters).run(
+            neo4j_session
+        )

cartography/intel/scaleway/projects.py

@@ -0,0 +1,79 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.account.v3 import AccountV3ProjectAPI
+from scaleway.account.v3 import Project
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.organization import ScalewayOrganizationSchema
+from cartography.models.scaleway.project import ScalewayProjectSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    update_tag: int,
+) -> list[dict]:
+    projects = get(client, org_id)
+    formatted_projects = transform_projects(projects)
+    load_projects(neo4j_session, formatted_projects, org_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+    return formatted_projects
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Project]:
+    api = AccountV3ProjectAPI(client)
+    return api.list_projects_all(organization_id=org_id)
+
+
+def transform_projects(projects: list[Project]) -> list[dict[str, Any]]:
+    formatted_projects = []
+    for project in projects:
+        formatted_projects.append(scaleway_obj_to_dict(project))
+    return formatted_projects
+
+
+@timeit
+def load_projects(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        ScalewayOrganizationSchema(),
+        [{"id": org_id}],
+        lastupdated=update_tag,
+    )
+    logger.info("Loading %d Scaleway Projects into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        ScalewayProjectSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=org_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(ScalewayProjectSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/scaleway/storage/snapshots.py

@@ -0,0 +1,86 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.instance.v1 import InstanceV1API
+from scaleway.instance.v1 import Snapshot
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import DEFAULT_ZONE
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.storage.snapshot import ScalewayVolumeSnapshotSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    projects_id: list[str],
+    update_tag: int,
+) -> None:
+    snapshots = get(client, org_id)
+    snapshots_by_project = transform_snapshots(snapshots)
+    load_snapshots(neo4j_session, snapshots_by_project, update_tag)
+    cleanup(neo4j_session, projects_id, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Snapshot]:
+    api = InstanceV1API(client)
+    return api.list_snapshots_all(organization=org_id, zone=DEFAULT_ZONE)
+
+
+def transform_snapshots(
+    snapshots: list[Snapshot],
+) -> dict[str, list[dict[str, Any]]]:
+    result: dict[str, list[dict[str, Any]]] = {}
+    for snapshot in snapshots:
+        project_id = snapshot.project
+        formatted_snapshot = scaleway_obj_to_dict(snapshot)
+        result.setdefault(project_id, []).append(formatted_snapshot)
+    return result
+
+
+@timeit
+def load_snapshots(
+    neo4j_session: neo4j.Session,
+    data: dict[str, list[dict[str, Any]]],
+    update_tag: int,
+) -> None:
+    for project_id, snapshots in data.items():
+        logger.info(
+            "Loading %d Scaleway InstanceSnapshots in project '%s' into Neo4j.",
+            len(snapshots),
+            project_id,
+        )
+        load(
+            neo4j_session,
+            ScalewayVolumeSnapshotSchema(),
+            snapshots,
+            lastupdated=update_tag,
+            PROJECT_ID=project_id,
+        )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    projects_id: list[str],
+    common_job_parameters: dict[str, Any],
+) -> None:
+    for project_id in projects_id:
+        scoped_job_parameters = common_job_parameters.copy()
+        scoped_job_parameters["PROJECT_ID"] = project_id
+        GraphJob.from_node_schema(
+            ScalewayVolumeSnapshotSchema(), scoped_job_parameters
+        ).run(neo4j_session)

cartography/intel/scaleway/storage/volumes.py

@@ -0,0 +1,84 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.instance.v1 import InstanceV1API
+from scaleway.instance.v1 import Volume
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import DEFAULT_ZONE
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.storage.volume import ScalewayVolumeSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    projects_id: list[str],
+    update_tag: int,
+) -> None:
+    volumes = get(client, org_id)
+    volumes_by_project = transform_volumes(volumes)
+    load_volumes(neo4j_session, volumes_by_project, update_tag)
+    cleanup(neo4j_session, projects_id, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Volume]:
+    api = InstanceV1API(client)
+    return api.list_volumes_all(organization=org_id, zone=DEFAULT_ZONE)
+
+
+def transform_volumes(volumes: list[Volume]) -> dict[str, list[dict[str, Any]]]:
+    result: dict[str, list[dict[str, Any]]] = {}
+    for volume in volumes:
+        project_id = volume.project
+        formatted_volume = scaleway_obj_to_dict(volume)
+        result.setdefault(project_id, []).append(formatted_volume)
+    return result
+
+
+@timeit
+def load_volumes(
+    neo4j_session: neo4j.Session,
+    data: dict[str, list[dict[str, Any]]],
+    update_tag: int,
+) -> None:
+    for project_id, volumes in data.items():
+        logger.info(
+            "Loading %d Scaleway InstanceVolumes in project '%s' into Neo4j.",
+            len(volumes),
+            project_id,
+        )
+        load(
+            neo4j_session,
+            ScalewayVolumeSchema(),
+            volumes,
+            lastupdated=update_tag,
+            PROJECT_ID=project_id,
+        )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    projects_id: list[str],
+    common_job_parameters: dict[str, Any],
+) -> None:
+    for project_id in projects_id:
+        scoped_job_parameters = common_job_parameters.copy()
+        scoped_job_parameters["PROJECT_ID"] = project_id
+        GraphJob.from_node_schema(ScalewayVolumeSchema(), scoped_job_parameters).run(
+            neo4j_session
+        )

cartography/intel/scaleway/utils.py

@@ -0,0 +1,37 @@
+import dataclasses
+from typing import Any
+
+# Zone does not really matter for readonly access, but we need to set it
+DEFAULT_ZONE = "fr-par-1"
+
+
+def scaleway_obj_to_dict(obj: Any) -> dict[str, Any]:
+    """Transform a Scaleway object (dataclass, dict, or list) into a dictionary."""
+    if isinstance(obj, type) or not dataclasses.is_dataclass(obj):
+        raise TypeError(f"Expected a dataclass, got {type(obj).__name__} instead.")
+    result: dict[str, Any] = dataclasses.asdict(obj)
+
+    for k in list(result.keys()):
+        result[k] = _scaleway_element_sanitize(result[k])
+    return result
+
+
+def _scaleway_element_sanitize(element: Any) -> Any:
+    """Sanitize a Scaleway element by removing empty strings and lists."""
+    if isinstance(element, str) and element == "":
+        return None
+    elif isinstance(element, list):
+        if len(element) == 0:
+            return None
+        return [
+            _scaleway_element_sanitize(item) for item in element if item is not None
+        ]
+    elif isinstance(element, dict):
+        return {
+            k: _scaleway_element_sanitize(v)
+            for k, v in element.items()
+            if v is not None
+        }
+    elif dataclasses.is_dataclass(element):
+        return scaleway_obj_to_dict(element)
+    return element

cartography/intel/sentinelone/__init__.py

@@ -0,0 +1,63 @@
+import logging
+
+import neo4j
+
+import cartography.intel.sentinelone.agent
+from cartography.config import Config
+from cartography.intel.sentinelone.account import sync_accounts
+from cartography.stats import get_stats_client
+from cartography.util import merge_module_sync_metadata
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+stat_handler = get_stats_client(__name__)
+
+
+@timeit
+def start_sentinelone_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    Perform ingestion of SentinelOne data.
+    :param neo4j_session: Neo4j session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+    if not config.sentinelone_api_token or not config.sentinelone_api_url:
+        logger.info("SentinelOne API configuration not found - skipping this module.")
+        return
+
+    # Set up common job parameters
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+        "API_URL": config.sentinelone_api_url,
+        "API_TOKEN": config.sentinelone_api_token,
+    }
+
+    # Sync SentinelOne account data (needs to be done first to establish the account nodes)
+    synced_account_ids = sync_accounts(
+        neo4j_session,
+        common_job_parameters,
+        config.sentinelone_account_ids,
+    )
+
+    # Sync agents for each account
+    for account_id in synced_account_ids:
+        # Add account-specific parameter
+        common_job_parameters["S1_ACCOUNT_ID"] = account_id
+
+        cartography.intel.sentinelone.agent.sync(
+            neo4j_session,
+            common_job_parameters,
+        )
+
+        # Clean up account-specific parameter
+        del common_job_parameters["S1_ACCOUNT_ID"]
+
+    # Record that the sync is complete
+    merge_module_sync_metadata(
+        neo4j_session,
+        group_type="SentinelOne",
+        group_id="sentinelone",
+        synced_type="SentinelOneData",
+        update_tag=config.update_tag,
+        stat_handler=stat_handler,
+    )

cartography/intel/sentinelone/account.py

@@ -0,0 +1,140 @@
+import logging
+from typing import Any
+
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.intel.sentinelone.api import call_sentinelone_api
+from cartography.models.sentinelone.account import S1AccountSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_accounts(
+    api_url: str, api_token: str, account_ids: list[str] | None = None
+) -> list[dict[str, Any]]:
+    """
+    Get account data from SentinelOne API
+    :param api_url: The SentinelOne API URL
+    :param api_token: The SentinelOne API token
+    :param account_ids: Optional list of account IDs to filter for
+    :return: Raw account data from API
+    """
+    logger.info("Retrieving SentinelOne account data")
+
+    # Get accounts info
+    response = call_sentinelone_api(
+        api_url=api_url,
+        endpoint="web/api/v2.1/accounts",
+        api_token=api_token,
+    )
+
+    accounts_data = response.get("data", [])
+
+    # Filter accounts by ID if specified
+    if account_ids:
+        accounts_data = [
+            account for account in accounts_data if account.get("id") in account_ids
+        ]
+        logger.info(f"Filtered accounts data to {len(accounts_data)} matching accounts")
+
+    if accounts_data:
+        logger.info(
+            f"Retrieved SentinelOne account data: {len(accounts_data)} accounts"
+        )
+    else:
+        logger.warning("No SentinelOne accounts retrieved")
+
+    return accounts_data
+
+
+def transform_accounts(accounts_data: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """
+    Transform raw account data into standardized format for Neo4j ingestion
+    :param accounts_data: Raw account data from API
+    :return: List of transformed account data
+    """
+    result: list[dict[str, Any]] = []
+
+    for account in accounts_data:
+        transformed_account = {
+            # Required fields - use direct access (will raise KeyError if missing)
+            "id": account["id"],
+            # Optional fields - use .get() with None default
+            "name": account.get("name"),
+            "account_type": account.get("accountType"),
+            "active_agents": account.get("activeAgents"),
+            "created_at": account.get("createdAt"),
+            "expiration": account.get("expiration"),
+            "number_of_sites": account.get("numberOfSites"),
+            "state": account.get("state"),
+        }
+        result.append(transformed_account)
+
+    return result
+
+
+def load_accounts(
+    neo4j_session: neo4j.Session,
+    accounts_data: list[dict[str, Any]],
+    update_tag: int,
+) -> None:
+    """
+    Load SentinelOne account data into Neo4j using the data model
+    :param neo4j_session: Neo4j session
+    :param accounts_data: List of account data to load
+    :param update_tag: Update tag for tracking data freshness
+    """
+    if not accounts_data:
+        logger.warning("No account data provided to load_accounts")
+        return
+
+    load(
+        neo4j_session,
+        S1AccountSchema(),
+        accounts_data,
+        lastupdated=update_tag,
+        firstseen=update_tag,
+    )
+
+    logger.info(f"Loaded {len(accounts_data)} SentinelOne account nodes")
+
+
+@timeit
+def sync_accounts(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+    account_ids: list[str] | None = None,
+) -> list[str]:
+    """
+    Sync SentinelOne account data using the modern sync pattern
+    :param neo4j_session: Neo4j session
+    :param api_url: SentinelOne API URL
+    :param api_token: SentinelOne API token
+    :param update_tag: Update tag for tracking data freshness
+    :param common_job_parameters: Job parameters for cleanup
+    :param account_ids: Optional list of account IDs to filter for
+    :return: List of synced account IDs
+    """
+    # 1. GET - Fetch data from API
+    accounts_raw_data = get_accounts(
+        common_job_parameters["API_URL"],
+        common_job_parameters["API_TOKEN"],
+        account_ids,
+    )
+
+    # 2. TRANSFORM - Shape data for ingestion
+    transformed_accounts = transform_accounts(accounts_raw_data)
+
+    # 3. LOAD - Ingest to Neo4j using data model
+    load_accounts(
+        neo4j_session,
+        transformed_accounts,
+        common_job_parameters["UPDATE_TAG"],
+    )
+
+    synced_account_ids = [account["id"] for account in transformed_accounts]
+    logger.info(f"Synced {len(synced_account_ids)} SentinelOne accounts")
+    return synced_account_ids