cartography 0.104.0rc3__py3-none-any.whl → 0.106.0__py3-none-any.whl

cartography/intel/scaleway/instances/instances.py

@@ -0,0 +1,92 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.instance.v1 import InstanceV1API
+from scaleway.instance.v1 import Server
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import DEFAULT_ZONE
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.instance.instance import ScalewayInstanceSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    projects_id: list[str],
+    update_tag: int,
+) -> None:
+    instances = get(client, org_id)
+    instances_by_project = transform_instances(instances)
+    load_instances(neo4j_session, instances_by_project, update_tag)
+    cleanup(neo4j_session, projects_id, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Server]:
+    api = InstanceV1API(client)
+    return api.list_servers_all(organization=org_id, zone=DEFAULT_ZONE)
+
+
+def transform_instances(
+    instances: list[Server],
+) -> dict[str, list[dict[str, Any]]]:
+    result: dict[str, list[dict[str, Any]]] = {}
+    for instance in instances:
+        project_id = instance.project
+        formatted_instance = scaleway_obj_to_dict(instance)
+        formatted_instance["public_ips"] = [
+            ip["id"] for ip in formatted_instance.get("public_ips", [])
+        ]
+        formatted_instance["volumes_id"] = [
+            volume["id"] for volume in formatted_instance.get("volumes", {}).values()
+        ]
+        result.setdefault(project_id, []).append(formatted_instance)
+    return result
+
+
+@timeit
+def load_instances(
+    neo4j_session: neo4j.Session,
+    data: dict[str, list[dict[str, Any]]],
+    update_tag: int,
+) -> None:
+    for project_id, instances in data.items():
+        logger.info(
+            "Loading %d Scaleway Instance in project '%s' into Neo4j.",
+            len(instances),
+            project_id,
+        )
+        load(
+            neo4j_session,
+            ScalewayInstanceSchema(),
+            instances,
+            lastupdated=update_tag,
+            PROJECT_ID=project_id,
+        )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    projects_id: list[str],
+    common_job_parameters: dict[str, Any],
+) -> None:
+    for project_id in projects_id:
+        scopped_job_parameters = common_job_parameters.copy()
+        scopped_job_parameters["PROJECT_ID"] = project_id
+        GraphJob.from_node_schema(ScalewayInstanceSchema(), scopped_job_parameters).run(
+            neo4j_session
+        )

cartography/intel/scaleway/projects.py

@@ -0,0 +1,79 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.account.v3 import AccountV3ProjectAPI
+from scaleway.account.v3 import Project
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.organization import ScalewayOrganizationSchema
+from cartography.models.scaleway.project import ScalewayProjectSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    update_tag: int,
+) -> list[dict]:
+    projects = get(client, org_id)
+    formatted_projects = transform_projects(projects)
+    load_projects(neo4j_session, formatted_projects, org_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+    return formatted_projects
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Project]:
+    api = AccountV3ProjectAPI(client)
+    return api.list_projects_all(organization_id=org_id)
+
+
+def transform_projects(projects: list[Project]) -> list[dict[str, Any]]:
+    formatted_projects = []
+    for project in projects:
+        formatted_projects.append(scaleway_obj_to_dict(project))
+    return formatted_projects
+
+
+@timeit
+def load_projects(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        ScalewayOrganizationSchema(),
+        [{"id": org_id}],
+        lastupdated=update_tag,
+    )
+    logger.info("Loading %d Scaleway Projects into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        ScalewayProjectSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=org_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(ScalewayProjectSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/scaleway/storage/snapshots.py

@@ -0,0 +1,86 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.instance.v1 import InstanceV1API
+from scaleway.instance.v1 import Snapshot
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import DEFAULT_ZONE
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.storage.snapshot import ScalewayVolumeSnapshotSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    projects_id: list[str],
+    update_tag: int,
+) -> None:
+    snapshots = get(client, org_id)
+    snapshots_by_project = transform_snapshots(snapshots)
+    load_snapshots(neo4j_session, snapshots_by_project, update_tag)
+    cleanup(neo4j_session, projects_id, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Snapshot]:
+    api = InstanceV1API(client)
+    return api.list_snapshots_all(organization=org_id, zone=DEFAULT_ZONE)
+
+
+def transform_snapshots(
+    snapshots: list[Snapshot],
+) -> dict[str, list[dict[str, Any]]]:
+    result: dict[str, list[dict[str, Any]]] = {}
+    for snapshot in snapshots:
+        project_id = snapshot.project
+        formatted_snapshot = scaleway_obj_to_dict(snapshot)
+        result.setdefault(project_id, []).append(formatted_snapshot)
+    return result
+
+
+@timeit
+def load_snapshots(
+    neo4j_session: neo4j.Session,
+    data: dict[str, list[dict[str, Any]]],
+    update_tag: int,
+) -> None:
+    for project_id, snapshots in data.items():
+        logger.info(
+            "Loading %d Scaleway InstanceSnapshots in project '%s' into Neo4j.",
+            len(snapshots),
+            project_id,
+        )
+        load(
+            neo4j_session,
+            ScalewayVolumeSnapshotSchema(),
+            snapshots,
+            lastupdated=update_tag,
+            PROJECT_ID=project_id,
+        )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    projects_id: list[str],
+    common_job_parameters: dict[str, Any],
+) -> None:
+    for project_id in projects_id:
+        scoped_job_parameters = common_job_parameters.copy()
+        scoped_job_parameters["PROJECT_ID"] = project_id
+        GraphJob.from_node_schema(
+            ScalewayVolumeSnapshotSchema(), scoped_job_parameters
+        ).run(neo4j_session)

cartography/intel/scaleway/storage/volumes.py

@@ -0,0 +1,84 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.instance.v1 import InstanceV1API
+from scaleway.instance.v1 import Volume
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import DEFAULT_ZONE
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.storage.volume import ScalewayVolumeSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    projects_id: list[str],
+    update_tag: int,
+) -> None:
+    volumes = get(client, org_id)
+    volumes_by_project = transform_volumes(volumes)
+    load_volumes(neo4j_session, volumes_by_project, update_tag)
+    cleanup(neo4j_session, projects_id, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Volume]:
+    api = InstanceV1API(client)
+    return api.list_volumes_all(organization=org_id, zone=DEFAULT_ZONE)
+
+
+def transform_volumes(volumes: list[Volume]) -> dict[str, list[dict[str, Any]]]:
+    result: dict[str, list[dict[str, Any]]] = {}
+    for volume in volumes:
+        project_id = volume.project
+        formatted_volume = scaleway_obj_to_dict(volume)
+        result.setdefault(project_id, []).append(formatted_volume)
+    return result
+
+
+@timeit
+def load_volumes(
+    neo4j_session: neo4j.Session,
+    data: dict[str, list[dict[str, Any]]],
+    update_tag: int,
+) -> None:
+    for project_id, volumes in data.items():
+        logger.info(
+            "Loading %d Scaleway InstanceVolumes in project '%s' into Neo4j.",
+            len(volumes),
+            project_id,
+        )
+        load(
+            neo4j_session,
+            ScalewayVolumeSchema(),
+            volumes,
+            lastupdated=update_tag,
+            PROJECT_ID=project_id,
+        )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    projects_id: list[str],
+    common_job_parameters: dict[str, Any],
+) -> None:
+    for project_id in projects_id:
+        scoped_job_parameters = common_job_parameters.copy()
+        scoped_job_parameters["PROJECT_ID"] = project_id
+        GraphJob.from_node_schema(ScalewayVolumeSchema(), scoped_job_parameters).run(
+            neo4j_session
+        )

cartography/intel/scaleway/utils.py

@@ -0,0 +1,37 @@
+import dataclasses
+from typing import Any
+
+# Zone does not really matter for readonly access, but we need to set it
+DEFAULT_ZONE = "fr-par-1"
+
+
+def scaleway_obj_to_dict(obj: Any) -> dict[str, Any]:
+    """Transform a Scaleway object (dataclass, dict, or list) into a dictionary."""
+    if isinstance(obj, type) or not dataclasses.is_dataclass(obj):
+        raise TypeError(f"Expected a dataclass, got {type(obj).__name__} instead.")
+    result: dict[str, Any] = dataclasses.asdict(obj)
+
+    for k in list(result.keys()):
+        result[k] = _scaleway_element_sanitize(result[k])
+    return result
+
+
+def _scaleway_element_sanitize(element: Any) -> Any:
+    """Sanitize a Scaleway element by removing empty strings and lists."""
+    if isinstance(element, str) and element == "":
+        return None
+    elif isinstance(element, list):
+        if len(element) == 0:
+            return None
+        return [
+            _scaleway_element_sanitize(item) for item in element if item is not None
+        ]
+    elif isinstance(element, dict):
+        return {
+            k: _scaleway_element_sanitize(v)
+            for k, v in element.items()
+            if v is not None
+        }
+    elif dataclasses.is_dataclass(element):
+        return scaleway_obj_to_dict(element)
+    return element

cartography/intel/trivy/__init__.py

@@ -0,0 +1,161 @@
+import logging
+from typing import Any
+
+import boto3
+from neo4j import Session
+
+from cartography.client.aws import list_accounts
+from cartography.client.aws.ecr import get_ecr_images
+from cartography.config import Config
+from cartography.intel.trivy.scanner import cleanup
+from cartography.intel.trivy.scanner import get_json_files_in_s3
+from cartography.intel.trivy.scanner import sync_single_image_from_s3
+from cartography.stats import get_stats_client
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+stat_handler = get_stats_client("trivy.scanner")
+
+
+@timeit
+def get_scan_targets(
+    neo4j_session: Session,
+    account_ids: list[str] | None = None,
+) -> set[str]:
+    """
+    Return list of ECR images from all accounts in the graph.
+    """
+    if not account_ids:
+        aws_accounts = list_accounts(neo4j_session)
+    else:
+        aws_accounts = account_ids
+
+    ecr_images: set[str] = set()
+    for account_id in aws_accounts:
+        for _, _, image_uri, _, _ in get_ecr_images(neo4j_session, account_id):
+            ecr_images.add(image_uri)
+
+    return ecr_images
+
+
+def _get_intersection(
+    images_in_graph: set[str], json_files: set[str], trivy_s3_prefix: str
+) -> list[tuple[str, str]]:
+    """
+    Get the intersection of ECR images in the graph and S3 scan results.
+
+    Args:
+        images_in_graph: Set of ECR images in the graph
+        json_files: Set of S3 object keys for JSON files
+        trivy_s3_prefix: S3 prefix path containing scan results
+
+    Returns:
+        List of tuples (image_uri, s3_object_key)
+    """
+    intersection = []
+    prefix_len = len(trivy_s3_prefix)
+    for s3_object_key in json_files:
+        # Sample key "123456789012.dkr.ecr.us-west-2.amazonaws.com/other-repo:v1.0.json"
+        # Sample key "folder/derp/123456789012.dkr.ecr.us-west-2.amazonaws.com/other-repo:v1.0.json"
+        # Remove the prefix and the .json suffix
+        image_uri = s3_object_key[prefix_len:-5]
+
+        if image_uri in images_in_graph:
+            intersection.append((image_uri, s3_object_key))
+
+    return intersection
+
+
+@timeit
+def sync_trivy_aws_ecr_from_s3(
+    neo4j_session: Session,
+    trivy_s3_bucket: str,
+    trivy_s3_prefix: str,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+    boto3_session: boto3.Session,
+) -> None:
+    """
+    Sync Trivy scan results from S3 for AWS ECR images.
+
+    Args:
+        neo4j_session: Neo4j session for database operations
+        trivy_s3_bucket: S3 bucket containing scan results
+        trivy_s3_prefix: S3 prefix path containing scan results
+        update_tag: Update tag for tracking
+        common_job_parameters: Common job parameters for cleanup
+        boto3_session: boto3 session for S3 operations
+    """
+    logger.info(
+        f"Using Trivy scan results from s3://{trivy_s3_bucket}/{trivy_s3_prefix}"
+    )
+
+    images_in_graph: set[str] = get_scan_targets(neo4j_session)
+    json_files: set[str] = get_json_files_in_s3(
+        trivy_s3_bucket, trivy_s3_prefix, boto3_session
+    )
+    intersection: list[tuple[str, str]] = _get_intersection(
+        images_in_graph, json_files, trivy_s3_prefix
+    )
+
+    if len(intersection) == 0:
+        logger.error(
+            f"Trivy sync was configured, but there are no ECR images with S3 json scan results in bucket "
+            f"'{trivy_s3_bucket}' with prefix '{trivy_s3_prefix}'. "
+            "Skipping Trivy sync to avoid potential data loss. "
+            "Please check the S3 bucket and prefix configuration. We expect the json files in s3 to be named "
+            f"`<image_uri>.json` and to be in the same bucket and prefix as the scan results. If the prefix is "
+            "a folder, it MUST end with a trailing slash '/'. "
+        )
+        logger.error(f"JSON files in S3: {json_files}")
+        raise ValueError("No ECR images with S3 json scan results found.")
+
+    logger.info(f"Processing {len(intersection)} ECR images with S3 scan results")
+    for image_uri, s3_object_key in intersection:
+        sync_single_image_from_s3(
+            neo4j_session,
+            image_uri,
+            update_tag,
+            trivy_s3_bucket,
+            s3_object_key,
+            boto3_session,
+        )
+
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def start_trivy_ingestion(neo4j_session: Session, config: Config) -> None:
+    """
+    Start Trivy scan ingestion from S3.
+
+    Args:
+        neo4j_session: Neo4j session for database operations
+        config: Configuration object containing S3 settings
+    """
+    # Check if S3 configuration is provided
+    if not config.trivy_s3_bucket:
+        logger.info("Trivy S3 configuration not provided. Skipping Trivy ingestion.")
+        return
+
+    # Default to empty string if s3 prefix is not provided
+    if config.trivy_s3_prefix is None:
+        config.trivy_s3_prefix = ""
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+    }
+
+    # Get ECR images to scan
+    boto3_session = boto3.Session()
+
+    sync_trivy_aws_ecr_from_s3(
+        neo4j_session,
+        config.trivy_s3_bucket,
+        config.trivy_s3_prefix,
+        config.update_tag,
+        common_job_parameters,
+        boto3_session,
+    )
+
+    # Support other Trivy resource types here e.g. if Google Cloud has images.