cartography 0.104.0rc2__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/ontology/utils.py

@@ -0,0 +1,176 @@
+import logging
+from dataclasses import asdict
+from typing import Any
+
+import neo4j
+
+from cartography.client.core.tx import read_list_of_dicts_tx
+from cartography.graph.job import GraphJob
+from cartography.models.ontology.mapping import ONTOLOGY_MODELS
+from cartography.models.ontology.mapping import ONTOLOGY_NODES_MAPPING
+from cartography.models.ontology.mapping.specs import OntologyNodeMapping
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def _run_source_node_single_query(
+    module_name: str,
+    node: OntologyNodeMapping,
+    neo4j_session: neo4j.Session,
+    query: str,
+    results: dict[str, dict[str, Any]],
+    **kwargs: Any,
+) -> dict[str, dict[str, Any]]:
+    # DOC
+    for row in neo4j_session.execute_read(read_list_of_dicts_tx, query, **kwargs):
+        node_data = row["n"]
+        result: dict[str, Any] = {}
+        skip_node: bool = False
+
+        # Extract only the fields defined in the ontology mapping
+        for field in node.fields:
+            value = node_data.get(field.node_field)
+            # Skip nodes missing required fields
+            if field.required and not value:
+                logger.debug(
+                    "Skipping node with label '%s' due to missing required field '%s'.",
+                    node.node_label,
+                    field.node_field,
+                )
+                skip_node = True
+                break
+            result[field.ontology_field] = value
+        if skip_node:
+            continue
+
+        # Merge results based on the node's id field to avoid duplicates
+        ontology_model = ONTOLOGY_MODELS[module_name]
+        if ontology_model is None:
+            # Should not happen as we skip non-eligible nodes above
+            logger.warning(
+                "No ontology model found for module '%s'. Skipping node label '%s'.",
+                module_name,
+                node.node_label,
+            )
+            continue
+        id_field = ontology_model().properties.id.name
+        existing = results.get(result[id_field])
+        if existing:
+            logger.debug("Merging node: %s to %s", result[id_field], existing[id_field])
+            # Merge existing data with new data, prioritizing non-None values
+            for key, value in result.items():
+                if existing.get(key) is None and value is not None:
+                    existing[key] = value
+        else:
+            logger.debug("Adding new node: %s", result[id_field])
+            results[result[id_field]] = result
+    return results
+
+
+@timeit
+def get_source_nodes_from_graph(
+    neo4j_session: neo4j.Session,
+    source_of_truth: list[str],
+    module_name: str,
+) -> list[dict[str, Any]]:
+    """Retrieve source nodes from the Neo4j graph database based on the ontology mapping.
+
+    This function queries the Neo4j database for nodes that match the labels
+    defined in the ontology mapping for the specified module and source of truth.
+    It returns a list of dictionaries containing the relevant fields for each node.
+
+    If no source of truth is provided, default to all sources defined in the mapping.
+
+    Args:
+        neo4j_session (neo4j.Session): The Neo4j session to use for querying the database.
+        source_of_truth (list[str]): A list of source of truth identifiers to filter the modules.
+        module_name (str): The name of the ontology module to use for the mapping (eg. users, devices, etc.).
+
+    Returns:
+        list[dict[str, Any]]: A list of dictionaries, each containing a node details formatted according to the ontology mapping.
+    """
+    results: dict[str, dict[str, Any]] = {}
+    modules_mapping = ONTOLOGY_NODES_MAPPING[module_name]
+    if len(source_of_truth) == 0:
+        source_of_truth = list(modules_mapping.keys())
+    # Check if ontology nodes are used in mapping
+    _has_ontology = False
+    if modules_mapping.get("ontology") is not None:
+        _has_ontology = True
+        for node in modules_mapping["ontology"].nodes:
+            if not node.eligible_for_source:
+                logger.debug(
+                    "Skipping ontology node with label '%s' as it is not eligible for source of truth.",
+                    node.node_label,
+                )
+                continue
+            # Run the query for every source
+            for source in source_of_truth:
+                # Use parameterized query to prevent Cypher injection attacks
+                query = f"MATCH (n:{node.node_label} {{_ont_source: $source}}) RETURN n"
+                results = _run_source_node_single_query(
+                    module_name, node, neo4j_session, query, results, source=source
+                )
+
+    # Run queries for each source of truth
+    for source in source_of_truth:
+        if source not in modules_mapping:
+            if not _has_ontology:
+                logger.warning(
+                    "Source of truth '%s' is not supported for '%s'.",
+                    source,
+                    module_name,
+                )
+            continue
+        for node in modules_mapping[source].nodes:
+            if not node.eligible_for_source:
+                logger.debug(
+                    "Skipping node with label '%s' as it is not eligible for source of truth '%s'.",
+                    node.node_label,
+                    source,
+                )
+                continue
+            query = f"MATCH (n:{node.node_label}) RETURN n"
+            results = _run_source_node_single_query(
+                module_name, node, neo4j_session, query, results
+            )
+
+    return list(results.values())
+
+
+@timeit
+def link_ontology_nodes(
+    neo4j_session: neo4j.Session,
+    module_name: str,
+    update_tag: int,
+) -> None:
+    """Link ontology nodes in the Neo4j graph database based on the ontology mapping.
+
+    This function retrieves the ontology mapping for the specified module and
+    executes the relationship statements defined in the mapping to link nodes
+    in the Neo4j graph database.
+
+    Args:
+        neo4j_session (neo4j.Session): The Neo4j session to use for executing the relationship statements.
+        module_name (str): The name of the ontology module for which to link nodes (eg. users, devices, etc.).
+        update_tag (int): The update tag of the current run, used to tag the changes in the graph.
+    """
+    modules_mapping = ONTOLOGY_NODES_MAPPING.get(module_name)
+    if modules_mapping is None:
+        logger.warning("No ontology mapping found for module '%s'.", module_name)
+        return
+    for source, mapping in modules_mapping.items():
+        if len(mapping.rels) == 0:
+            continue
+        formated_json = {
+            "name": f"Linking ontology nodes for {module_name} for source {source}",
+            "statements": [asdict(rel) for rel in mapping.rels],
+        }
+        GraphJob.run_from_json(
+            neo4j_session,
+            formated_json,
+            {"UPDATE_TAG": update_tag},
+            short_name=f"ontology.{module_name}.{source}.linking",
+        )

cartography/intel/pagerduty/escalation_policies.py

@@ -6,6 +6,7 @@ from typing import List
 import neo4j
 from pdpyras import APISession
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -72,7 +73,8 @@ def load_escalation_policy_data(
             for team in policy["teams"]:
                 teams.append({"escalation_policy": policy["id"], "team": team["id"]})
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         EscalationPolicies=data,
         update_tag=update_tag,
@@ -115,7 +117,8 @@ def _attach_rules(
                 elif target["type"] == "schedule":
                     schedules.append({"rule": rule["id"], "schedule": target["id"]})
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Rules=data,
         update_tag=update_tag,
@@ -140,7 +143,8 @@ def _attach_user_targets(
         MERGE (p)-[r:ASSOCIATED_WITH]->(u)
         ON CREATE SET r.firstseen = timestamp()
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Relations=data,
         update_tag=update_tag,
@@ -162,7 +166,8 @@ def _attach_schedule_targets(
         MERGE (p)-[r:ASSOCIATED_WITH]->(s)
         ON CREATE SET r.firstseen = timestamp()
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Relations=data,
         update_tag=update_tag,
@@ -184,7 +189,8 @@ def _attach_services(
         MERGE (s)-[r:ASSOCIATED_WITH]->(p)
         ON CREATE SET r.firstseen = timestamp()
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Relations=data,
         update_tag=update_tag,
@@ -206,7 +212,8 @@ def _attach_teams(
         MERGE (t)-[r:ASSOCIATED_WITH]->(p)
         ON CREATE SET r.firstseen = timestamp()
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Relations=data,
         update_tag=update_tag,

cartography/intel/pagerduty/schedules.py

@@ -7,6 +7,7 @@ import dateutil.parser
 import neo4j
 from pdpyras import APISession
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -63,7 +64,8 @@ def load_schedule_data(
                 layer["_schedule_id"] = schedule["id"]
                 layers.append(layer)
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Schedules=data,
         update_tag=update_tag,
@@ -87,7 +89,8 @@ def _attach_users(
         MERGE (u)-[r:MEMBER_OF]->(s)
         ON CREATE SET r.firstseen = timestamp()
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Relations=data,
         update_tag=update_tag,
@@ -129,7 +132,8 @@ def _attach_layers(
                 users.append(
                     {"layer_id": layer["_layer_id"], "user": user["user"]["id"]},
                 )
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Layers=data,
         update_tag=update_tag,
@@ -152,7 +156,8 @@ def _attach_layer_users(
         MERGE (u)-[r:MEMBER_OF]->(l)
         ON CREATE SET r.firstseen = timestamp()
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Relations=data,
         update_tag=update_tag,

cartography/intel/pagerduty/services.py

@@ -7,6 +7,7 @@ import dateutil.parser
 import neo4j
 from pdpyras import APISession
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -96,7 +97,8 @@ def load_service_data(
             for team in service["teams"]:
                 team_relations.append({"service": service["id"], "team": team["id"]})
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Services=data,
         update_tag=update_tag,
@@ -120,7 +122,8 @@ def _attach_teams(
         MERGE (t)-[r:ASSOCIATED_WITH]->(s)
         ON CREATE SET r.firstseen = timestamp()
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Relations=data,
         update_tag=update_tag,
@@ -162,7 +165,8 @@ def load_integration_data(
         created_at = dateutil.parser.parse(integration["created_at"])
         integration["created_at"] = int(created_at.timestamp())
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Integrations=data,
         update_tag=update_tag,

cartography/intel/pagerduty/teams.py

@@ -6,6 +6,7 @@ from typing import List
 import neo4j
 from pdpyras import APISession
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -68,7 +69,8 @@ def load_team_data(
     """
     logger.info(f"Loading {len(data)} pagerduty teams.")
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Teams=data,
         update_tag=update_tag,
@@ -90,7 +92,8 @@ def load_team_relations(
         ON CREATE SET r.firstseen = timestamp()
         SET r.role = relation.role
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Relations=data,
         update_tag=update_tag,

cartography/intel/pagerduty/users.py

@@ -6,6 +6,7 @@ from typing import List
 import neo4j
 from pdpyras import APISession
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -57,7 +58,8 @@ def load_user_data(
     """
     logger.info(f"Loading {len(data)} pagerduty users.")
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Users=data,
         update_tag=update_tag,

cartography/intel/pagerduty/vendors.py

@@ -6,6 +6,7 @@ from typing import List
 import neo4j
 from pdpyras import APISession
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -53,7 +54,8 @@ def load_vendor_data(
     """
     logger.info(f"Loading {len(data)} pagerduty vendors.")
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Vendors=data,
         update_tag=update_tag,

cartography/intel/scaleway/__init__.py

@@ -0,0 +1,127 @@
+import logging
+
+import neo4j
+import scaleway
+
+import cartography.intel.scaleway.iam.apikeys
+import cartography.intel.scaleway.iam.applications
+import cartography.intel.scaleway.iam.groups
+import cartography.intel.scaleway.iam.users
+import cartography.intel.scaleway.instances.flexibleips
+import cartography.intel.scaleway.instances.instances
+import cartography.intel.scaleway.projects
+import cartography.intel.scaleway.storage.snapshots
+import cartography.intel.scaleway.storage.volumes
+from cartography.config import Config
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def start_scaleway_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of Scaleway data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+
+    if (
+        not config.scaleway_access_key
+        or not config.scaleway_secret_key
+        or not config.scaleway_org
+    ):
+        logger.info(
+            "Tailscale import is not configured - skipping this module. "
+            "See docs to configure.",
+        )
+        return
+
+    # Create client
+    client = scaleway.Client(
+        access_key=config.scaleway_access_key,
+        secret_key=config.scaleway_secret_key,
+    )
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+        "ORG_ID": config.scaleway_org,
+    }
+
+    # Organization level
+    projects = cartography.intel.scaleway.projects.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        update_tag=config.update_tag,
+    )
+    projects_id = [project["id"] for project in projects]
+    cartography.intel.scaleway.iam.users.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        update_tag=config.update_tag,
+    )
+    cartography.intel.scaleway.iam.applications.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        update_tag=config.update_tag,
+    )
+    cartography.intel.scaleway.iam.groups.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        update_tag=config.update_tag,
+    )
+    cartography.intel.scaleway.iam.apikeys.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        update_tag=config.update_tag,
+    )
+
+    # Storage
+    cartography.intel.scaleway.storage.volumes.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        projects_id=projects_id,
+        update_tag=config.update_tag,
+    )
+    cartography.intel.scaleway.storage.snapshots.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        projects_id=projects_id,
+        update_tag=config.update_tag,
+    )
+
+    # Instances
+    # DISABLED due to https://github.com/scaleway/scaleway-sdk-python/issues/1040
+    """
+    cartography.intel.scaleway.instances.flexibleips.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        projects_id=projects_id,
+        update_tag=config.update_tag,
+    )
+    """
+    cartography.intel.scaleway.instances.instances.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        projects_id=projects_id,
+        update_tag=config.update_tag,
+    )

cartography/intel/scaleway/iam/apikeys.py

@@ -0,0 +1,71 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.iam.v1alpha1 import APIKey
+from scaleway.iam.v1alpha1 import IamV1Alpha1API
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.iam.apikey import ScalewayApiKeySchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    apikeys = get(client, org_id)
+    formatted_apikeys = transform_apikeys(apikeys)
+    load_apikeys(neo4j_session, formatted_apikeys, org_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[APIKey]:
+    api = IamV1Alpha1API(client)
+    return api.list_api_keys_all(organization_id=org_id)
+
+
+def transform_apikeys(apikeys: list[APIKey]) -> list[dict[str, Any]]:
+    formatted_apikeys = []
+    for apikey in apikeys:
+        formatted_apikeys.append(scaleway_obj_to_dict(apikey))
+    return formatted_apikeys
+
+
+@timeit
+def load_apikeys(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Scaleway ApiKeys into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        ScalewayApiKeySchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=org_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(ScalewayApiKeySchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/scaleway/iam/applications.py

@@ -0,0 +1,71 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.iam.v1alpha1 import Application
+from scaleway.iam.v1alpha1 import IamV1Alpha1API
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.iam.application import ScalewayApplicationSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    applications = get(client, org_id)
+    formatted_applications = transform_applications(applications)
+    load_applications(neo4j_session, formatted_applications, org_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Application]:
+    api = IamV1Alpha1API(client)
+    return api.list_applications_all(organization_id=org_id)
+
+
+def transform_applications(applications: list[Application]) -> list[dict[str, Any]]:
+    formatted_applications = []
+    for application in applications:
+        formatted_applications.append(scaleway_obj_to_dict(application))
+    return formatted_applications
+
+
+@timeit
+def load_applications(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Scaleway Applications into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        ScalewayApplicationSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=org_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(ScalewayApplicationSchema(), common_job_parameters).run(
+        neo4j_session
+    )