cartography 0.93.0rc1__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/scaleway/instances/instances.py

@@ -0,0 +1,92 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.instance.v1 import InstanceV1API
+from scaleway.instance.v1 import Server
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import DEFAULT_ZONE
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.instance.instance import ScalewayInstanceSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    projects_id: list[str],
+    update_tag: int,
+) -> None:
+    instances = get(client, org_id)
+    instances_by_project = transform_instances(instances)
+    load_instances(neo4j_session, instances_by_project, update_tag)
+    cleanup(neo4j_session, projects_id, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Server]:
+    api = InstanceV1API(client)
+    return api.list_servers_all(organization=org_id, zone=DEFAULT_ZONE)
+
+
+def transform_instances(
+    instances: list[Server],
+) -> dict[str, list[dict[str, Any]]]:
+    result: dict[str, list[dict[str, Any]]] = {}
+    for instance in instances:
+        project_id = instance.project
+        formatted_instance = scaleway_obj_to_dict(instance)
+        formatted_instance["public_ips"] = [
+            ip["id"] for ip in formatted_instance.get("public_ips", [])
+        ]
+        formatted_instance["volumes_id"] = [
+            volume["id"] for volume in formatted_instance.get("volumes", {}).values()
+        ]
+        result.setdefault(project_id, []).append(formatted_instance)
+    return result
+
+
+@timeit
+def load_instances(
+    neo4j_session: neo4j.Session,
+    data: dict[str, list[dict[str, Any]]],
+    update_tag: int,
+) -> None:
+    for project_id, instances in data.items():
+        logger.info(
+            "Loading %d Scaleway Instance in project '%s' into Neo4j.",
+            len(instances),
+            project_id,
+        )
+        load(
+            neo4j_session,
+            ScalewayInstanceSchema(),
+            instances,
+            lastupdated=update_tag,
+            PROJECT_ID=project_id,
+        )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    projects_id: list[str],
+    common_job_parameters: dict[str, Any],
+) -> None:
+    for project_id in projects_id:
+        scopped_job_parameters = common_job_parameters.copy()
+        scopped_job_parameters["PROJECT_ID"] = project_id
+        GraphJob.from_node_schema(ScalewayInstanceSchema(), scopped_job_parameters).run(
+            neo4j_session
+        )

cartography/intel/scaleway/projects.py

@@ -0,0 +1,79 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.account.v3 import AccountV3ProjectAPI
+from scaleway.account.v3 import Project
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.organization import ScalewayOrganizationSchema
+from cartography.models.scaleway.project import ScalewayProjectSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    update_tag: int,
+) -> list[dict]:
+    projects = get(client, org_id)
+    formatted_projects = transform_projects(projects)
+    load_projects(neo4j_session, formatted_projects, org_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+    return formatted_projects
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Project]:
+    api = AccountV3ProjectAPI(client)
+    return api.list_projects_all(organization_id=org_id)
+
+
+def transform_projects(projects: list[Project]) -> list[dict[str, Any]]:
+    formatted_projects = []
+    for project in projects:
+        formatted_projects.append(scaleway_obj_to_dict(project))
+    return formatted_projects
+
+
+@timeit
+def load_projects(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        ScalewayOrganizationSchema(),
+        [{"id": org_id}],
+        lastupdated=update_tag,
+    )
+    logger.info("Loading %d Scaleway Projects into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        ScalewayProjectSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=org_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(ScalewayProjectSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/scaleway/storage/snapshots.py

@@ -0,0 +1,86 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.instance.v1 import InstanceV1API
+from scaleway.instance.v1 import Snapshot
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import DEFAULT_ZONE
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.storage.snapshot import ScalewayVolumeSnapshotSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    projects_id: list[str],
+    update_tag: int,
+) -> None:
+    snapshots = get(client, org_id)
+    snapshots_by_project = transform_snapshots(snapshots)
+    load_snapshots(neo4j_session, snapshots_by_project, update_tag)
+    cleanup(neo4j_session, projects_id, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Snapshot]:
+    api = InstanceV1API(client)
+    return api.list_snapshots_all(organization=org_id, zone=DEFAULT_ZONE)
+
+
+def transform_snapshots(
+    snapshots: list[Snapshot],
+) -> dict[str, list[dict[str, Any]]]:
+    result: dict[str, list[dict[str, Any]]] = {}
+    for snapshot in snapshots:
+        project_id = snapshot.project
+        formatted_snapshot = scaleway_obj_to_dict(snapshot)
+        result.setdefault(project_id, []).append(formatted_snapshot)
+    return result
+
+
+@timeit
+def load_snapshots(
+    neo4j_session: neo4j.Session,
+    data: dict[str, list[dict[str, Any]]],
+    update_tag: int,
+) -> None:
+    for project_id, snapshots in data.items():
+        logger.info(
+            "Loading %d Scaleway InstanceSnapshots in project '%s' into Neo4j.",
+            len(snapshots),
+            project_id,
+        )
+        load(
+            neo4j_session,
+            ScalewayVolumeSnapshotSchema(),
+            snapshots,
+            lastupdated=update_tag,
+            PROJECT_ID=project_id,
+        )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    projects_id: list[str],
+    common_job_parameters: dict[str, Any],
+) -> None:
+    for project_id in projects_id:
+        scoped_job_parameters = common_job_parameters.copy()
+        scoped_job_parameters["PROJECT_ID"] = project_id
+        GraphJob.from_node_schema(
+            ScalewayVolumeSnapshotSchema(), scoped_job_parameters
+        ).run(neo4j_session)

cartography/intel/scaleway/storage/volumes.py

@@ -0,0 +1,84 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.instance.v1 import InstanceV1API
+from scaleway.instance.v1 import Volume
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import DEFAULT_ZONE
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.storage.volume import ScalewayVolumeSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    projects_id: list[str],
+    update_tag: int,
+) -> None:
+    volumes = get(client, org_id)
+    volumes_by_project = transform_volumes(volumes)
+    load_volumes(neo4j_session, volumes_by_project, update_tag)
+    cleanup(neo4j_session, projects_id, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Volume]:
+    api = InstanceV1API(client)
+    return api.list_volumes_all(organization=org_id, zone=DEFAULT_ZONE)
+
+
+def transform_volumes(volumes: list[Volume]) -> dict[str, list[dict[str, Any]]]:
+    result: dict[str, list[dict[str, Any]]] = {}
+    for volume in volumes:
+        project_id = volume.project
+        formatted_volume = scaleway_obj_to_dict(volume)
+        result.setdefault(project_id, []).append(formatted_volume)
+    return result
+
+
+@timeit
+def load_volumes(
+    neo4j_session: neo4j.Session,
+    data: dict[str, list[dict[str, Any]]],
+    update_tag: int,
+) -> None:
+    for project_id, volumes in data.items():
+        logger.info(
+            "Loading %d Scaleway InstanceVolumes in project '%s' into Neo4j.",
+            len(volumes),
+            project_id,
+        )
+        load(
+            neo4j_session,
+            ScalewayVolumeSchema(),
+            volumes,
+            lastupdated=update_tag,
+            PROJECT_ID=project_id,
+        )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    projects_id: list[str],
+    common_job_parameters: dict[str, Any],
+) -> None:
+    for project_id in projects_id:
+        scoped_job_parameters = common_job_parameters.copy()
+        scoped_job_parameters["PROJECT_ID"] = project_id
+        GraphJob.from_node_schema(ScalewayVolumeSchema(), scoped_job_parameters).run(
+            neo4j_session
+        )

cartography/intel/scaleway/utils.py

@@ -0,0 +1,37 @@
+import dataclasses
+from typing import Any
+
+# Zone does not really matter for readonly access, but we need to set it
+DEFAULT_ZONE = "fr-par-1"
+
+
+def scaleway_obj_to_dict(obj: Any) -> dict[str, Any]:
+    """Transform a Scaleway object (dataclass, dict, or list) into a dictionary."""
+    if isinstance(obj, type) or not dataclasses.is_dataclass(obj):
+        raise TypeError(f"Expected a dataclass, got {type(obj).__name__} instead.")
+    result: dict[str, Any] = dataclasses.asdict(obj)
+
+    for k in list(result.keys()):
+        result[k] = _scaleway_element_sanitize(result[k])
+    return result
+
+
+def _scaleway_element_sanitize(element: Any) -> Any:
+    """Sanitize a Scaleway element by removing empty strings and lists."""
+    if isinstance(element, str) and element == "":
+        return None
+    elif isinstance(element, list):
+        if len(element) == 0:
+            return None
+        return [
+            _scaleway_element_sanitize(item) for item in element if item is not None
+        ]
+    elif isinstance(element, dict):
+        return {
+            k: _scaleway_element_sanitize(v)
+            for k, v in element.items()
+            if v is not None
+        }
+    elif dataclasses.is_dataclass(element):
+        return scaleway_obj_to_dict(element)
+    return element

cartography/intel/semgrep/__init__.py

@@ -3,21 +3,46 @@ import logging
 import neo4j
 
 from cartography.config import Config
-from cartography.intel.semgrep.findings import sync
+from cartography.intel.semgrep.dependencies import sync_dependencies
+from cartography.intel.semgrep.deployment import sync_deployment
+from cartography.intel.semgrep.findings import sync_findings
 from cartography.util import timeit
 
-
 logger = logging.getLogger(__name__)
 
 
 @timeit
 def start_semgrep_ingestion(
-    neo4j_session: neo4j.Session, config: Config,
+    neo4j_session: neo4j.Session,
+    config: Config,
 ) -> None:
     common_job_parameters = {
         "UPDATE_TAG": config.update_tag,
     }
     if not config.semgrep_app_token:
-        logger.info('Semgrep import is not configured - skipping this module. See docs to configure.')
+        logger.info(
+            "Semgrep import is not configured - skipping this module. See docs to configure.",
+        )
         return
-    sync(neo4j_session, config.semgrep_app_token, config.update_tag, common_job_parameters)
+
+    # sync_deployment must be called first since it populates common_job_parameters
+    # with the deployment ID and slug, which are required by the other sync functions
+    sync_deployment(
+        neo4j_session,
+        config.semgrep_app_token,
+        config.update_tag,
+        common_job_parameters,
+    )
+    sync_dependencies(
+        neo4j_session,
+        config.semgrep_app_token,
+        config.semgrep_dependency_ecosystems,
+        config.update_tag,
+        common_job_parameters,
+    )  # noqa: E501
+    sync_findings(
+        neo4j_session,
+        config.semgrep_app_token,
+        config.update_tag,
+        common_job_parameters,
+    )

cartography/intel/semgrep/dependencies.py

@@ -0,0 +1,255 @@
+import logging
+from typing import Any
+from typing import Callable
+from typing import Dict
+from typing import List
+
+import neo4j
+import requests
+from requests.exceptions import HTTPError
+from requests.exceptions import ReadTimeout
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.semgrep.dependencies import SemgrepGoLibrarySchema
+from cartography.models.semgrep.dependencies import SemgrepNpmLibrarySchema
+from cartography.stats import get_stats_client
+from cartography.util import merge_module_sync_metadata
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+stat_handler = get_stats_client(__name__)
+_PAGE_SIZE = 10000
+_TIMEOUT = (60, 60)
+_MAX_RETRIES = 3
+
+# The keys in this dictionary must be in Semgrep's list of supported ecosystems, defined here:
+# https://semgrep.dev/api/v1/docs/#tag/SupplyChainService/operation/semgrep_app.products.sca.handlers.dependency.list_dependencies_conexxion
+ECOSYSTEM_TO_SCHEMA: Dict = {
+    "gomod": SemgrepGoLibrarySchema,
+    "npm": SemgrepNpmLibrarySchema,
+}
+
+
+def parse_and_validate_semgrep_ecosystems(ecosystems: str) -> List[str]:
+    validated_ecosystems: List[str] = []
+    for ecosystem in ecosystems.split(","):
+        ecosystem = ecosystem.strip().lower()
+
+        if ecosystem in ECOSYSTEM_TO_SCHEMA:
+            validated_ecosystems.append(ecosystem)
+        else:
+            valid_ecosystems: str = ",".join(ECOSYSTEM_TO_SCHEMA.keys())
+            raise ValueError(
+                f'Error parsing `semgrep-dependency-ecosystems`. You specified "{ecosystems}". '
+                f'Please check that your input is formatted as comma-separated values, e.g. "gomod,npm". '
+                f"Full list of supported ecosystems: {valid_ecosystems}.",
+            )
+    return validated_ecosystems
+
+
+@timeit
+def get_dependencies(
+    semgrep_app_token: str,
+    deployment_id: str,
+    ecosystem: str,
+) -> List[Dict[str, Any]]:
+    """
+    Gets all dependencies for the given ecosystem within the given Semgrep deployment ID.
+    param: semgrep_app_token: The Semgrep App token to use for authentication.
+    param: deployment_id: The Semgrep deployment ID to use for retrieving dependencies.
+    param: ecosystem: The ecosystem to import dependencies from, e.g. "gomod" or "npm".
+    """
+    all_deps = []
+    deps_url = f"https://semgrep.dev/api/v1/deployments/{deployment_id}/dependencies"
+    has_more = True
+    page = 0
+    retries = 0
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": f"Bearer {semgrep_app_token}",
+    }
+
+    request_data: dict[str, Any] = {
+        "pageSize": _PAGE_SIZE,
+        "dependencyFilter": {
+            "ecosystem": [ecosystem],
+        },
+    }
+
+    logger.info(
+        f"Retrieving Semgrep {ecosystem} dependencies for deployment '{deployment_id}'.",
+    )
+    while has_more:
+        try:
+            response = requests.post(
+                deps_url,
+                json=request_data,
+                headers=headers,
+                timeout=_TIMEOUT,
+            )
+            response.raise_for_status()
+            data = response.json()
+        except (ReadTimeout, HTTPError):
+            logger.warning(
+                f"Failed to retrieve Semgrep {ecosystem} dependencies for page {page}. Retrying...",
+            )
+            retries += 1
+            if retries >= _MAX_RETRIES:
+                raise
+            continue
+        deps = data.get("dependencies", [])
+        has_more = data.get("hasMore", False)
+        logger.info(f"Processed page {page} of Semgrep {ecosystem} dependencies.")
+        all_deps.extend(deps)
+        retries = 0
+        page += 1
+        request_data["cursor"] = data.get("cursor")
+
+    logger.info(
+        f"Retrieved {len(all_deps)} Semgrep {ecosystem} dependencies in {page} pages.",
+    )
+    return all_deps
+
+
+def transform_dependencies(raw_deps: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """
+    Transforms the raw dependencies response from Semgrep API into a list of dicts
+    that can be used to create the Dependency nodes.
+    """
+
+    """
+    sample raw_dep as of November 2024:
+    {
+        "repositoryId": "123456",
+        "definedAt": {
+            "path": "go.mod",
+            "startLine": "6",
+            "endLine": "6",
+            "url": "https://github.com/org/repo-name/blob/00000000000000000000000000000000/go.mod#L6",
+            "committedAt": "1970-01-01T00:00:00Z",
+            "startCol": "0",
+            "endCol": "0"
+        },
+        "transitivity": "DIRECT",
+        "package": {
+            "name": "github.com/foo/bar",
+            "versionSpecifier": "1.2.3"
+        },
+        "ecosystem": "gomod",
+        "licenses": [],
+        "pathToTransitivity": []
+    },
+    """
+    deps = []
+    for raw_dep in raw_deps:
+
+        # We could call a different endpoint to get all repo IDs and store a mapping of repo ID to URL,
+        # but it's much simpler to just extract the URL from the definedAt field.
+        repo_url = raw_dep["definedAt"]["url"].split("/blob/", 1)[0]
+
+        name = raw_dep["package"]["name"]
+        version = raw_dep["package"]["versionSpecifier"]
+        id = f"{name}|{version}"
+
+        # As of November 2024, Semgrep does not import dependencies with version specifiers such as >, <, etc.
+        # For now, hardcode the specifier to ==<version> to align with GitHub-sourced Python dependencies.
+        # If Semgrep eventually supports version specifiers, update this line accordingly.
+        specifier = f"=={version}"
+
+        deps.append(
+            {
+                # existing dependency properties:
+                "id": id,
+                "name": name,
+                "specifier": specifier,
+                "version": version,
+                "repo_url": repo_url,
+                # Semgrep-specific properties:
+                "ecosystem": raw_dep["ecosystem"],
+                "transitivity": raw_dep["transitivity"].lower(),
+                "url": raw_dep["definedAt"]["url"],
+            },
+        )
+
+    return deps
+
+
+@timeit
+def load_dependencies(
+    neo4j_session: neo4j.Session,
+    dependency_schema: Callable,
+    dependencies: List[Dict],
+    deployment_id: str,
+    update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading {len(dependencies)} {dependency_schema().label} objects into the graph.",
+    )
+    load(
+        neo4j_session,
+        dependency_schema(),
+        dependencies,
+        lastupdated=update_tag,
+        DEPLOYMENT_ID=deployment_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    dependency_schema: Callable,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.info(
+        f"Running Semgrep Dependencies cleanup job for {dependency_schema().label}.",
+    )
+    GraphJob.from_node_schema(dependency_schema(), common_job_parameters).run(
+        neo4j_session,
+    )
+
+
+@timeit
+def sync_dependencies(
+    neo4j_session: neo4j.Session,
+    semgrep_app_token: str,
+    ecosystems_str: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+
+    deployment_id = common_job_parameters.get("DEPLOYMENT_ID")
+    if not deployment_id:
+        logger.warning(
+            "Missing Semgrep deployment ID, ensure that sync_deployment() has been called. "
+            "Skipping Semgrep dependencies sync job.",
+        )
+        return
+
+    if not ecosystems_str:
+        logger.warning(
+            "Semgrep is not configured to import dependencies for any ecosystems, see docs to configure. "
+            "Skipping Semgrep dependencies sync job.",
+        )
+        return
+
+    # We don't expect an error here since we've already validated the input in cli.py
+    ecosystems = parse_and_validate_semgrep_ecosystems(ecosystems_str)
+
+    logger.info("Running Semgrep dependencies sync job.")
+
+    for ecosystem in ecosystems:
+        schema = ECOSYSTEM_TO_SCHEMA[ecosystem]
+        raw_deps = get_dependencies(semgrep_app_token, deployment_id, ecosystem)
+        deps = transform_dependencies(raw_deps)
+        load_dependencies(neo4j_session, schema, deps, deployment_id, update_tag)
+        cleanup(neo4j_session, schema, common_job_parameters)
+
+    merge_module_sync_metadata(
+        neo4j_session=neo4j_session,
+        group_type="Semgrep",
+        group_id=deployment_id,
+        synced_type="SemgrepDependency",
+        update_tag=update_tag,
+        stat_handler=stat_handler,
+    )