cartography 0.106.0rc1__py3-none-any.whl → 0.107.0rc1__py3-none-any.whl

cartography/intel/scaleway/iam/applications.py

@@ -0,0 +1,71 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.iam.v1alpha1 import Application
+from scaleway.iam.v1alpha1 import IamV1Alpha1API
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.iam.application import ScalewayApplicationSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    applications = get(client, org_id)
+    formatted_applications = transform_applications(applications)
+    load_applications(neo4j_session, formatted_applications, org_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Application]:
+    api = IamV1Alpha1API(client)
+    return api.list_applications_all(organization_id=org_id)
+
+
+def transform_applications(applications: list[Application]) -> list[dict[str, Any]]:
+    formatted_applications = []
+    for application in applications:
+        formatted_applications.append(scaleway_obj_to_dict(application))
+    return formatted_applications
+
+
+@timeit
+def load_applications(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Scaleway Applications into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        ScalewayApplicationSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=org_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(ScalewayApplicationSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/scaleway/iam/groups.py

@@ -0,0 +1,71 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.iam.v1alpha1 import Group
+from scaleway.iam.v1alpha1 import IamV1Alpha1API
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.iam.group import ScalewayGroupSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    groups = get(client, org_id)
+    formatted_groups = transform_groups(groups)
+    load_groups(neo4j_session, formatted_groups, org_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Group]:
+    api = IamV1Alpha1API(client)
+    return api.list_groups_all(organization_id=org_id)
+
+
+def transform_groups(groups: list[Group]) -> list[dict[str, Any]]:
+    formatted_groups = []
+    for group in groups:
+        formatted_groups.append(scaleway_obj_to_dict(group))
+    return formatted_groups
+
+
+@timeit
+def load_groups(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Scaleway Groups into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        ScalewayGroupSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=org_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(ScalewayGroupSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/scaleway/iam/users.py

@@ -0,0 +1,71 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.iam.v1alpha1 import IamV1Alpha1API
+from scaleway.iam.v1alpha1 import User
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.iam.user import ScalewayUserSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    users = get(client, org_id)
+    formatted_users = transform_users(users)
+    load_users(neo4j_session, formatted_users, org_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[User]:
+    api = IamV1Alpha1API(client)
+    return api.list_users_all(organization_id=org_id)
+
+
+def transform_users(users: list[User]) -> list[dict[str, Any]]:
+    formatted_users = []
+    for user in users:
+        formatted_users.append(scaleway_obj_to_dict(user))
+    return formatted_users
+
+
+@timeit
+def load_users(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Scaleway Users into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        ScalewayUserSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=org_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(ScalewayUserSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/scaleway/instances/flexibleips.py

@@ -0,0 +1,86 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.instance.v1 import InstanceV1API
+from scaleway.instance.v1 import Ip
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import DEFAULT_ZONE
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.instance.flexibleip import ScalewayFlexibleIpSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    projects_id: list[str],
+    update_tag: int,
+) -> None:
+    flexibleips = get(client, org_id)
+    flexibleips_by_project = transform_flexibleips(flexibleips)
+    load_flexibleips(neo4j_session, flexibleips_by_project, update_tag)
+    cleanup(neo4j_session, projects_id, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Ip]:
+    api = InstanceV1API(client)
+    return api.list_ips_all(organization=org_id, zone=DEFAULT_ZONE)
+
+
+def transform_flexibleips(
+    flexibleips: list[Ip],
+) -> dict[str, list[dict[str, Any]]]:
+    result: dict[str, list[dict[str, Any]]] = {}
+    for flexibleip in flexibleips:
+        project_id = flexibleip.project
+        formatted_flexibleip = scaleway_obj_to_dict(flexibleip)
+        result.setdefault(project_id, []).append(formatted_flexibleip)
+    return result
+
+
+@timeit
+def load_flexibleips(
+    neo4j_session: neo4j.Session,
+    data: dict[str, list[dict[str, Any]]],
+    update_tag: int,
+) -> None:
+    for project_id, flexibleips in data.items():
+        logger.info(
+            "Loading %d Scaleway Flexible IPs in project '%s' into Neo4j.",
+            len(flexibleips),
+            project_id,
+        )
+        load(
+            neo4j_session,
+            ScalewayFlexibleIpSchema(),
+            flexibleips,
+            lastupdated=update_tag,
+            PROJECT_ID=project_id,
+        )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    projects_id: list[str],
+    common_job_parameters: dict[str, Any],
+) -> None:
+    for project_id in projects_id:
+        scopped_job_parameters = common_job_parameters.copy()
+        scopped_job_parameters["PROJECT_ID"] = project_id
+        GraphJob.from_node_schema(
+            ScalewayFlexibleIpSchema(), scopped_job_parameters
+        ).run(neo4j_session)

cartography/intel/scaleway/instances/instances.py

@@ -0,0 +1,92 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.instance.v1 import InstanceV1API
+from scaleway.instance.v1 import Server
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import DEFAULT_ZONE
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.instance.instance import ScalewayInstanceSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    projects_id: list[str],
+    update_tag: int,
+) -> None:
+    instances = get(client, org_id)
+    instances_by_project = transform_instances(instances)
+    load_instances(neo4j_session, instances_by_project, update_tag)
+    cleanup(neo4j_session, projects_id, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Server]:
+    api = InstanceV1API(client)
+    return api.list_servers_all(organization=org_id, zone=DEFAULT_ZONE)
+
+
+def transform_instances(
+    instances: list[Server],
+) -> dict[str, list[dict[str, Any]]]:
+    result: dict[str, list[dict[str, Any]]] = {}
+    for instance in instances:
+        project_id = instance.project
+        formatted_instance = scaleway_obj_to_dict(instance)
+        formatted_instance["public_ips"] = [
+            ip["id"] for ip in formatted_instance.get("public_ips", [])
+        ]
+        formatted_instance["volumes_id"] = [
+            volume["id"] for volume in formatted_instance.get("volumes", {}).values()
+        ]
+        result.setdefault(project_id, []).append(formatted_instance)
+    return result
+
+
+@timeit
+def load_instances(
+    neo4j_session: neo4j.Session,
+    data: dict[str, list[dict[str, Any]]],
+    update_tag: int,
+) -> None:
+    for project_id, instances in data.items():
+        logger.info(
+            "Loading %d Scaleway Instance in project '%s' into Neo4j.",
+            len(instances),
+            project_id,
+        )
+        load(
+            neo4j_session,
+            ScalewayInstanceSchema(),
+            instances,
+            lastupdated=update_tag,
+            PROJECT_ID=project_id,
+        )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    projects_id: list[str],
+    common_job_parameters: dict[str, Any],
+) -> None:
+    for project_id in projects_id:
+        scopped_job_parameters = common_job_parameters.copy()
+        scopped_job_parameters["PROJECT_ID"] = project_id
+        GraphJob.from_node_schema(ScalewayInstanceSchema(), scopped_job_parameters).run(
+            neo4j_session
+        )

cartography/intel/scaleway/projects.py

@@ -0,0 +1,79 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.account.v3 import AccountV3ProjectAPI
+from scaleway.account.v3 import Project
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.organization import ScalewayOrganizationSchema
+from cartography.models.scaleway.project import ScalewayProjectSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    update_tag: int,
+) -> list[dict]:
+    projects = get(client, org_id)
+    formatted_projects = transform_projects(projects)
+    load_projects(neo4j_session, formatted_projects, org_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+    return formatted_projects
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Project]:
+    api = AccountV3ProjectAPI(client)
+    return api.list_projects_all(organization_id=org_id)
+
+
+def transform_projects(projects: list[Project]) -> list[dict[str, Any]]:
+    formatted_projects = []
+    for project in projects:
+        formatted_projects.append(scaleway_obj_to_dict(project))
+    return formatted_projects
+
+
+@timeit
+def load_projects(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        ScalewayOrganizationSchema(),
+        [{"id": org_id}],
+        lastupdated=update_tag,
+    )
+    logger.info("Loading %d Scaleway Projects into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        ScalewayProjectSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=org_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(ScalewayProjectSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/scaleway/storage/snapshots.py

@@ -0,0 +1,86 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.instance.v1 import InstanceV1API
+from scaleway.instance.v1 import Snapshot
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import DEFAULT_ZONE
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.storage.snapshot import ScalewayVolumeSnapshotSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    projects_id: list[str],
+    update_tag: int,
+) -> None:
+    snapshots = get(client, org_id)
+    snapshots_by_project = transform_snapshots(snapshots)
+    load_snapshots(neo4j_session, snapshots_by_project, update_tag)
+    cleanup(neo4j_session, projects_id, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Snapshot]:
+    api = InstanceV1API(client)
+    return api.list_snapshots_all(organization=org_id, zone=DEFAULT_ZONE)
+
+
+def transform_snapshots(
+    snapshots: list[Snapshot],
+) -> dict[str, list[dict[str, Any]]]:
+    result: dict[str, list[dict[str, Any]]] = {}
+    for snapshot in snapshots:
+        project_id = snapshot.project
+        formatted_snapshot = scaleway_obj_to_dict(snapshot)
+        result.setdefault(project_id, []).append(formatted_snapshot)
+    return result
+
+
+@timeit
+def load_snapshots(
+    neo4j_session: neo4j.Session,
+    data: dict[str, list[dict[str, Any]]],
+    update_tag: int,
+) -> None:
+    for project_id, snapshots in data.items():
+        logger.info(
+            "Loading %d Scaleway InstanceSnapshots in project '%s' into Neo4j.",
+            len(snapshots),
+            project_id,
+        )
+        load(
+            neo4j_session,
+            ScalewayVolumeSnapshotSchema(),
+            snapshots,
+            lastupdated=update_tag,
+            PROJECT_ID=project_id,
+        )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    projects_id: list[str],
+    common_job_parameters: dict[str, Any],
+) -> None:
+    for project_id in projects_id:
+        scoped_job_parameters = common_job_parameters.copy()
+        scoped_job_parameters["PROJECT_ID"] = project_id
+        GraphJob.from_node_schema(
+            ScalewayVolumeSnapshotSchema(), scoped_job_parameters
+        ).run(neo4j_session)

cartography/intel/scaleway/storage/volumes.py

@@ -0,0 +1,84 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.instance.v1 import InstanceV1API
+from scaleway.instance.v1 import Volume
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import DEFAULT_ZONE
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.storage.volume import ScalewayVolumeSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    projects_id: list[str],
+    update_tag: int,
+) -> None:
+    volumes = get(client, org_id)
+    volumes_by_project = transform_volumes(volumes)
+    load_volumes(neo4j_session, volumes_by_project, update_tag)
+    cleanup(neo4j_session, projects_id, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Volume]:
+    api = InstanceV1API(client)
+    return api.list_volumes_all(organization=org_id, zone=DEFAULT_ZONE)
+
+
+def transform_volumes(volumes: list[Volume]) -> dict[str, list[dict[str, Any]]]:
+    result: dict[str, list[dict[str, Any]]] = {}
+    for volume in volumes:
+        project_id = volume.project
+        formatted_volume = scaleway_obj_to_dict(volume)
+        result.setdefault(project_id, []).append(formatted_volume)
+    return result
+
+
+@timeit
+def load_volumes(
+    neo4j_session: neo4j.Session,
+    data: dict[str, list[dict[str, Any]]],
+    update_tag: int,
+) -> None:
+    for project_id, volumes in data.items():
+        logger.info(
+            "Loading %d Scaleway InstanceVolumes in project '%s' into Neo4j.",
+            len(volumes),
+            project_id,
+        )
+        load(
+            neo4j_session,
+            ScalewayVolumeSchema(),
+            volumes,
+            lastupdated=update_tag,
+            PROJECT_ID=project_id,
+        )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    projects_id: list[str],
+    common_job_parameters: dict[str, Any],
+) -> None:
+    for project_id in projects_id:
+        scoped_job_parameters = common_job_parameters.copy()
+        scoped_job_parameters["PROJECT_ID"] = project_id
+        GraphJob.from_node_schema(ScalewayVolumeSchema(), scoped_job_parameters).run(
+            neo4j_session
+        )

cartography/intel/scaleway/utils.py

@@ -0,0 +1,37 @@
+import dataclasses
+from typing import Any
+
+# Zone does not really matter for readonly access, but we need to set it
+DEFAULT_ZONE = "fr-par-1"
+
+
+def scaleway_obj_to_dict(obj: Any) -> dict[str, Any]:
+    """Transform a Scaleway object (dataclass, dict, or list) into a dictionary."""
+    if isinstance(obj, type) or not dataclasses.is_dataclass(obj):
+        raise TypeError(f"Expected a dataclass, got {type(obj).__name__} instead.")
+    result: dict[str, Any] = dataclasses.asdict(obj)
+
+    for k in list(result.keys()):
+        result[k] = _scaleway_element_sanitize(result[k])
+    return result
+
+
+def _scaleway_element_sanitize(element: Any) -> Any:
+    """Sanitize a Scaleway element by removing empty strings and lists."""
+    if isinstance(element, str) and element == "":
+        return None
+    elif isinstance(element, list):
+        if len(element) == 0:
+            return None
+        return [
+            _scaleway_element_sanitize(item) for item in element if item is not None
+        ]
+    elif isinstance(element, dict):
+        return {
+            k: _scaleway_element_sanitize(v)
+            for k, v in element.items()
+            if v is not None
+        }
+    elif dataclasses.is_dataclass(element):
+        return scaleway_obj_to_dict(element)
+    return element