cartography 0.104.0rc2__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/scaleway/iam/groups.py

@@ -0,0 +1,71 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.iam.v1alpha1 import Group
+from scaleway.iam.v1alpha1 import IamV1Alpha1API
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.iam.group import ScalewayGroupSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    groups = get(client, org_id)
+    formatted_groups = transform_groups(groups)
+    load_groups(neo4j_session, formatted_groups, org_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Group]:
+    api = IamV1Alpha1API(client)
+    return api.list_groups_all(organization_id=org_id)
+
+
+def transform_groups(groups: list[Group]) -> list[dict[str, Any]]:
+    formatted_groups = []
+    for group in groups:
+        formatted_groups.append(scaleway_obj_to_dict(group))
+    return formatted_groups
+
+
+@timeit
+def load_groups(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Scaleway Groups into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        ScalewayGroupSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=org_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(ScalewayGroupSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/scaleway/iam/users.py

@@ -0,0 +1,71 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.iam.v1alpha1 import IamV1Alpha1API
+from scaleway.iam.v1alpha1 import User
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.iam.user import ScalewayUserSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    users = get(client, org_id)
+    formatted_users = transform_users(users)
+    load_users(neo4j_session, formatted_users, org_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[User]:
+    api = IamV1Alpha1API(client)
+    return api.list_users_all(organization_id=org_id)
+
+
+def transform_users(users: list[User]) -> list[dict[str, Any]]:
+    formatted_users = []
+    for user in users:
+        formatted_users.append(scaleway_obj_to_dict(user))
+    return formatted_users
+
+
+@timeit
+def load_users(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Scaleway Users into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        ScalewayUserSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=org_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(ScalewayUserSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/scaleway/instances/flexibleips.py

@@ -0,0 +1,86 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.instance.v1 import InstanceV1API
+from scaleway.instance.v1 import Ip
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import DEFAULT_ZONE
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.instance.flexibleip import ScalewayFlexibleIpSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    projects_id: list[str],
+    update_tag: int,
+) -> None:
+    flexibleips = get(client, org_id)
+    flexibleips_by_project = transform_flexibleips(flexibleips)
+    load_flexibleips(neo4j_session, flexibleips_by_project, update_tag)
+    cleanup(neo4j_session, projects_id, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Ip]:
+    api = InstanceV1API(client)
+    return api.list_ips_all(organization=org_id, zone=DEFAULT_ZONE)
+
+
+def transform_flexibleips(
+    flexibleips: list[Ip],
+) -> dict[str, list[dict[str, Any]]]:
+    result: dict[str, list[dict[str, Any]]] = {}
+    for flexibleip in flexibleips:
+        project_id = flexibleip.project
+        formatted_flexibleip = scaleway_obj_to_dict(flexibleip)
+        result.setdefault(project_id, []).append(formatted_flexibleip)
+    return result
+
+
+@timeit
+def load_flexibleips(
+    neo4j_session: neo4j.Session,
+    data: dict[str, list[dict[str, Any]]],
+    update_tag: int,
+) -> None:
+    for project_id, flexibleips in data.items():
+        logger.info(
+            "Loading %d Scaleway Flexible IPs in project '%s' into Neo4j.",
+            len(flexibleips),
+            project_id,
+        )
+        load(
+            neo4j_session,
+            ScalewayFlexibleIpSchema(),
+            flexibleips,
+            lastupdated=update_tag,
+            PROJECT_ID=project_id,
+        )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    projects_id: list[str],
+    common_job_parameters: dict[str, Any],
+) -> None:
+    for project_id in projects_id:
+        scopped_job_parameters = common_job_parameters.copy()
+        scopped_job_parameters["PROJECT_ID"] = project_id
+        GraphJob.from_node_schema(
+            ScalewayFlexibleIpSchema(), scopped_job_parameters
+        ).run(neo4j_session)

cartography/intel/scaleway/instances/instances.py

@@ -0,0 +1,92 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.instance.v1 import InstanceV1API
+from scaleway.instance.v1 import Server
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import DEFAULT_ZONE
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.instance.instance import ScalewayInstanceSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    projects_id: list[str],
+    update_tag: int,
+) -> None:
+    instances = get(client, org_id)
+    instances_by_project = transform_instances(instances)
+    load_instances(neo4j_session, instances_by_project, update_tag)
+    cleanup(neo4j_session, projects_id, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Server]:
+    api = InstanceV1API(client)
+    return api.list_servers_all(organization=org_id, zone=DEFAULT_ZONE)
+
+
+def transform_instances(
+    instances: list[Server],
+) -> dict[str, list[dict[str, Any]]]:
+    result: dict[str, list[dict[str, Any]]] = {}
+    for instance in instances:
+        project_id = instance.project
+        formatted_instance = scaleway_obj_to_dict(instance)
+        formatted_instance["public_ips"] = [
+            ip["id"] for ip in formatted_instance.get("public_ips", [])
+        ]
+        formatted_instance["volumes_id"] = [
+            volume["id"] for volume in formatted_instance.get("volumes", {}).values()
+        ]
+        result.setdefault(project_id, []).append(formatted_instance)
+    return result
+
+
+@timeit
+def load_instances(
+    neo4j_session: neo4j.Session,
+    data: dict[str, list[dict[str, Any]]],
+    update_tag: int,
+) -> None:
+    for project_id, instances in data.items():
+        logger.info(
+            "Loading %d Scaleway Instance in project '%s' into Neo4j.",
+            len(instances),
+            project_id,
+        )
+        load(
+            neo4j_session,
+            ScalewayInstanceSchema(),
+            instances,
+            lastupdated=update_tag,
+            PROJECT_ID=project_id,
+        )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    projects_id: list[str],
+    common_job_parameters: dict[str, Any],
+) -> None:
+    for project_id in projects_id:
+        scopped_job_parameters = common_job_parameters.copy()
+        scopped_job_parameters["PROJECT_ID"] = project_id
+        GraphJob.from_node_schema(ScalewayInstanceSchema(), scopped_job_parameters).run(
+            neo4j_session
+        )

cartography/intel/scaleway/projects.py

@@ -0,0 +1,79 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.account.v3 import AccountV3ProjectAPI
+from scaleway.account.v3 import Project
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.organization import ScalewayOrganizationSchema
+from cartography.models.scaleway.project import ScalewayProjectSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    update_tag: int,
+) -> list[dict]:
+    projects = get(client, org_id)
+    formatted_projects = transform_projects(projects)
+    load_projects(neo4j_session, formatted_projects, org_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+    return formatted_projects
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Project]:
+    api = AccountV3ProjectAPI(client)
+    return api.list_projects_all(organization_id=org_id)
+
+
+def transform_projects(projects: list[Project]) -> list[dict[str, Any]]:
+    formatted_projects = []
+    for project in projects:
+        formatted_projects.append(scaleway_obj_to_dict(project))
+    return formatted_projects
+
+
+@timeit
+def load_projects(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        ScalewayOrganizationSchema(),
+        [{"id": org_id}],
+        lastupdated=update_tag,
+    )
+    logger.info("Loading %d Scaleway Projects into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        ScalewayProjectSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=org_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(ScalewayProjectSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/scaleway/storage/snapshots.py

@@ -0,0 +1,86 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.instance.v1 import InstanceV1API
+from scaleway.instance.v1 import Snapshot
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import DEFAULT_ZONE
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.storage.snapshot import ScalewayVolumeSnapshotSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    projects_id: list[str],
+    update_tag: int,
+) -> None:
+    snapshots = get(client, org_id)
+    snapshots_by_project = transform_snapshots(snapshots)
+    load_snapshots(neo4j_session, snapshots_by_project, update_tag)
+    cleanup(neo4j_session, projects_id, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Snapshot]:
+    api = InstanceV1API(client)
+    return api.list_snapshots_all(organization=org_id, zone=DEFAULT_ZONE)
+
+
+def transform_snapshots(
+    snapshots: list[Snapshot],
+) -> dict[str, list[dict[str, Any]]]:
+    result: dict[str, list[dict[str, Any]]] = {}
+    for snapshot in snapshots:
+        project_id = snapshot.project
+        formatted_snapshot = scaleway_obj_to_dict(snapshot)
+        result.setdefault(project_id, []).append(formatted_snapshot)
+    return result
+
+
+@timeit
+def load_snapshots(
+    neo4j_session: neo4j.Session,
+    data: dict[str, list[dict[str, Any]]],
+    update_tag: int,
+) -> None:
+    for project_id, snapshots in data.items():
+        logger.info(
+            "Loading %d Scaleway InstanceSnapshots in project '%s' into Neo4j.",
+            len(snapshots),
+            project_id,
+        )
+        load(
+            neo4j_session,
+            ScalewayVolumeSnapshotSchema(),
+            snapshots,
+            lastupdated=update_tag,
+            PROJECT_ID=project_id,
+        )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    projects_id: list[str],
+    common_job_parameters: dict[str, Any],
+) -> None:
+    for project_id in projects_id:
+        scoped_job_parameters = common_job_parameters.copy()
+        scoped_job_parameters["PROJECT_ID"] = project_id
+        GraphJob.from_node_schema(
+            ScalewayVolumeSnapshotSchema(), scoped_job_parameters
+        ).run(neo4j_session)

cartography/intel/scaleway/storage/volumes.py

@@ -0,0 +1,84 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.instance.v1 import InstanceV1API
+from scaleway.instance.v1 import Volume
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import DEFAULT_ZONE
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.storage.volume import ScalewayVolumeSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    projects_id: list[str],
+    update_tag: int,
+) -> None:
+    volumes = get(client, org_id)
+    volumes_by_project = transform_volumes(volumes)
+    load_volumes(neo4j_session, volumes_by_project, update_tag)
+    cleanup(neo4j_session, projects_id, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Volume]:
+    api = InstanceV1API(client)
+    return api.list_volumes_all(organization=org_id, zone=DEFAULT_ZONE)
+
+
+def transform_volumes(volumes: list[Volume]) -> dict[str, list[dict[str, Any]]]:
+    result: dict[str, list[dict[str, Any]]] = {}
+    for volume in volumes:
+        project_id = volume.project
+        formatted_volume = scaleway_obj_to_dict(volume)
+        result.setdefault(project_id, []).append(formatted_volume)
+    return result
+
+
+@timeit
+def load_volumes(
+    neo4j_session: neo4j.Session,
+    data: dict[str, list[dict[str, Any]]],
+    update_tag: int,
+) -> None:
+    for project_id, volumes in data.items():
+        logger.info(
+            "Loading %d Scaleway InstanceVolumes in project '%s' into Neo4j.",
+            len(volumes),
+            project_id,
+        )
+        load(
+            neo4j_session,
+            ScalewayVolumeSchema(),
+            volumes,
+            lastupdated=update_tag,
+            PROJECT_ID=project_id,
+        )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    projects_id: list[str],
+    common_job_parameters: dict[str, Any],
+) -> None:
+    for project_id in projects_id:
+        scoped_job_parameters = common_job_parameters.copy()
+        scoped_job_parameters["PROJECT_ID"] = project_id
+        GraphJob.from_node_schema(ScalewayVolumeSchema(), scoped_job_parameters).run(
+            neo4j_session
+        )

cartography/intel/scaleway/utils.py

@@ -0,0 +1,37 @@
+import dataclasses
+from typing import Any
+
+# Zone does not really matter for readonly access, but we need to set it
+DEFAULT_ZONE = "fr-par-1"
+
+
+def scaleway_obj_to_dict(obj: Any) -> dict[str, Any]:
+    """Transform a Scaleway object (dataclass, dict, or list) into a dictionary."""
+    if isinstance(obj, type) or not dataclasses.is_dataclass(obj):
+        raise TypeError(f"Expected a dataclass, got {type(obj).__name__} instead.")
+    result: dict[str, Any] = dataclasses.asdict(obj)
+
+    for k in list(result.keys()):
+        result[k] = _scaleway_element_sanitize(result[k])
+    return result
+
+
+def _scaleway_element_sanitize(element: Any) -> Any:
+    """Sanitize a Scaleway element by removing empty strings and lists."""
+    if isinstance(element, str) and element == "":
+        return None
+    elif isinstance(element, list):
+        if len(element) == 0:
+            return None
+        return [
+            _scaleway_element_sanitize(item) for item in element if item is not None
+        ]
+    elif isinstance(element, dict):
+        return {
+            k: _scaleway_element_sanitize(v)
+            for k, v in element.items()
+            if v is not None
+        }
+    elif dataclasses.is_dataclass(element):
+        return scaleway_obj_to_dict(element)
+    return element

cartography/intel/sentinelone/__init__.py

@@ -0,0 +1,75 @@
+import logging
+
+import neo4j
+
+import cartography.intel.sentinelone.agent
+import cartography.intel.sentinelone.application
+import cartography.intel.sentinelone.cve
+from cartography.config import Config
+from cartography.intel.sentinelone.account import sync_accounts
+from cartography.stats import get_stats_client
+from cartography.util import merge_module_sync_metadata
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+stat_handler = get_stats_client(__name__)
+
+
+@timeit
+def start_sentinelone_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    Perform ingestion of SentinelOne data.
+    :param neo4j_session: Neo4j session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+    if not config.sentinelone_api_token or not config.sentinelone_api_url:
+        logger.info("SentinelOne API configuration not found - skipping this module.")
+        return
+
+    # Set up common job parameters
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+        "API_URL": config.sentinelone_api_url,
+        "API_TOKEN": config.sentinelone_api_token,
+    }
+
+    # Sync SentinelOne account data (needs to be done first to establish the account nodes)
+    synced_account_ids = sync_accounts(
+        neo4j_session,
+        common_job_parameters,
+        config.sentinelone_account_ids,
+    )
+
+    # Sync agents and applications for each account
+    for account_id in synced_account_ids:
+        # Add account-specific parameter
+        common_job_parameters["S1_ACCOUNT_ID"] = account_id
+
+        cartography.intel.sentinelone.agent.sync(
+            neo4j_session,
+            common_job_parameters,
+        )
+
+        cartography.intel.sentinelone.application.sync(
+            neo4j_session,
+            common_job_parameters,
+        )
+
+        cartography.intel.sentinelone.cve.sync(
+            neo4j_session,
+            common_job_parameters,
+        )
+
+        # Clean up account-specific parameters
+        del common_job_parameters["S1_ACCOUNT_ID"]
+
+    # Record that the sync is complete
+    merge_module_sync_metadata(
+        neo4j_session,
+        group_type="SentinelOne",
+        group_id="sentinelone",
+        synced_type="SentinelOneData",
+        update_tag=config.update_tag,
+        stat_handler=stat_handler,
+    )