cartography 0.117.0__py3-none-any.whl → 0.119.0__py3-none-any.whl

cartography/intel/ontology/__init__.py

@@ -0,0 +1,44 @@
+import logging
+
+import neo4j
+
+import cartography.intel.ontology.devices
+import cartography.intel.ontology.users
+from cartography.config import Config
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def run(neo4j_session: neo4j.Session, config: Config) -> None:
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+    }
+
+    # Get source of truth from config
+    if config.ontology_users_source:
+        users_source_of_truth = [
+            source.strip() for source in config.ontology_users_source.split(",")
+        ]
+    else:
+        users_source_of_truth = []
+    if config.ontology_devices_source:
+        computers_source_of_truth = [
+            source.strip() for source in config.ontology_devices_source.split(",")
+        ]
+    else:
+        computers_source_of_truth = []
+
+    cartography.intel.ontology.users.sync(
+        neo4j_session,
+        users_source_of_truth,
+        config.update_tag,
+        common_job_parameters,
+    )
+    cartography.intel.ontology.devices.sync(
+        neo4j_session,
+        computers_source_of_truth,
+        config.update_tag,
+        common_job_parameters,
+    )

cartography/intel/ontology/devices.py

@@ -0,0 +1,54 @@
+import logging
+from typing import Any
+
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.ontology.utils import get_source_nodes_from_graph
+from cartography.intel.ontology.utils import link_ontology_nodes
+from cartography.models.ontology.device import DeviceSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    source_of_truth: list[str],
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    data = get_source_nodes_from_graph(neo4j_session, source_of_truth, "devices")
+    load_devices(
+        neo4j_session,
+        data,
+        update_tag,
+    )
+    link_ontology_nodes(neo4j_session, "devices", update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def load_devices(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        DeviceSchema(),
+        data,
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    GraphJob.from_node_schema(DeviceSchema(), common_job_parameters).run(
+        neo4j_session,
+    )

cartography/intel/ontology/users.py

@@ -0,0 +1,54 @@
+import logging
+from typing import Any
+
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.ontology.utils import get_source_nodes_from_graph
+from cartography.intel.ontology.utils import link_ontology_nodes
+from cartography.models.ontology.user import UserSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    source_of_truth: list[str],
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    data = get_source_nodes_from_graph(neo4j_session, source_of_truth, "users")
+    load_users(
+        neo4j_session,
+        data,
+        update_tag,
+    )
+    link_ontology_nodes(neo4j_session, "users", update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def load_users(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        UserSchema(),
+        data,
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    GraphJob.from_node_schema(UserSchema(), common_job_parameters).run(
+        neo4j_session,
+    )

cartography/intel/ontology/utils.py

@@ -0,0 +1,121 @@
+import logging
+from dataclasses import asdict
+from typing import Any
+
+import neo4j
+
+from cartography.client.core.tx import read_list_of_dicts_tx
+from cartography.graph.job import GraphJob
+from cartography.models.ontology.mapping import ONTOLOGY_MAPPING
+from cartography.models.ontology.mapping import ONTOLOGY_MODELS
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_source_nodes_from_graph(
+    neo4j_session: neo4j.Session,
+    source_of_truth: list[str],
+    module_name: str,
+) -> list[dict[str, Any]]:
+    """Retrieve source nodes from the Neo4j graph database based on the ontology mapping.
+
+    This function queries the Neo4j database for nodes that match the labels
+    defined in the ontology mapping for the specified module and source of truth.
+    It returns a list of dictionaries containing the relevant fields for each node.
+
+    If no source of truth is provided, default to all sources defined in the mapping.
+
+    Args:
+        neo4j_session (neo4j.Session): The Neo4j session to use for querying the database.
+        source_of_truth (list[str]): A list of source of truth identifiers to filter the modules.
+        module_name (str): The name of the ontology module to use for the mapping (eg. users, devices, etc.).
+
+    Returns:
+        list[dict[str, Any]]: A list of dictionaries, each containing a node details formatted according to the ontology mapping.
+    """
+    results: dict[str, dict[str, Any]] = {}
+    modules_mapping = ONTOLOGY_MAPPING[module_name]
+    if len(source_of_truth) == 0:
+        source_of_truth = list(modules_mapping.keys())
+    for source in source_of_truth:
+        if source not in modules_mapping:
+            logger.warning(
+                "Source of truth '%s' is not supported for '%s'.", source, module_name
+            )
+            continue
+        for node in modules_mapping[source].nodes:
+            query = f"MATCH (n:{node.node_label}) RETURN n"
+            for row in neo4j_session.execute_read(read_list_of_dicts_tx, query):
+                node_data = row["n"]
+                result: dict[str, Any] = {}
+                skip_node: bool = False
+
+                # Extract only the fields defined in the ontology mapping
+                for field in node.fields:
+                    value = node_data.get(field.node_field)
+                    # Skip nodes missing required fields
+                    if field.required and value is None:
+                        logger.debug(
+                            "Skipping node with label '%s' due to missing required field '%s'.",
+                            node.node_label,
+                            field.node_field,
+                        )
+                        skip_node = True
+                        break
+                    result[field.ontology_field] = value
+                if skip_node:
+                    continue
+
+                # Merge results based on the node's id field to avoid duplicates
+                id_field = ONTOLOGY_MODELS[module_name]().properties.id.name
+                existing = results.get(result[id_field])
+                if existing:
+                    logger.debug(
+                        "Merging node: %s to %s", result[id_field], existing[id_field]
+                    )
+                    # Merge existing data with new data, prioritizing non-None values
+                    for key, value in result.items():
+                        if existing.get(key) is None and value is not None:
+                            existing[key] = value
+                else:
+                    logger.debug("Adding new node: %s", result[id_field])
+                    results[result[id_field]] = result
+    return list(results.values())
+
+
+@timeit
+def link_ontology_nodes(
+    neo4j_session: neo4j.Session,
+    module_name: str,
+    update_tag: int,
+) -> None:
+    """Link ontology nodes in the Neo4j graph database based on the ontology mapping.
+
+    This function retrieves the ontology mapping for the specified module and
+    executes the relationship statements defined in the mapping to link nodes
+    in the Neo4j graph database.
+
+    Args:
+        neo4j_session (neo4j.Session): The Neo4j session to use for executing the relationship statements.
+        module_name (str): The name of the ontology module for which to link nodes (eg. users, devices, etc.).
+        update_tag (int): The update tag of the current run, used to tag the changes in the graph.
+    """
+    modules_mapping = ONTOLOGY_MAPPING.get(module_name)
+    if modules_mapping is None:
+        logger.warning("No ontology mapping found for module '%s'.", module_name)
+        return
+    for source, mapping in modules_mapping.items():
+        if len(mapping.rels) == 0:
+            continue
+        formated_json = {
+            "name": f"Linking ontology nodes for {module_name} for source {source}",
+            "statements": [asdict(rel) for rel in mapping.rels],
+        }
+        GraphJob.run_from_json(
+            neo4j_session,
+            formated_json,
+            {"UPDATE_TAG": update_tag},
+            short_name=f"ontology.{module_name}.{source}.linking",
+        )

cartography/intel/pagerduty/escalation_policies.py

@@ -6,6 +6,7 @@ from typing import List
 import neo4j
 from pdpyras import APISession
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -72,7 +73,8 @@ def load_escalation_policy_data(
             for team in policy["teams"]:
                 teams.append({"escalation_policy": policy["id"], "team": team["id"]})
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         EscalationPolicies=data,
         update_tag=update_tag,
@@ -115,7 +117,8 @@ def _attach_rules(
                 elif target["type"] == "schedule":
                     schedules.append({"rule": rule["id"], "schedule": target["id"]})
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Rules=data,
         update_tag=update_tag,
@@ -140,7 +143,8 @@ def _attach_user_targets(
         MERGE (p)-[r:ASSOCIATED_WITH]->(u)
         ON CREATE SET r.firstseen = timestamp()
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Relations=data,
         update_tag=update_tag,
@@ -162,7 +166,8 @@ def _attach_schedule_targets(
         MERGE (p)-[r:ASSOCIATED_WITH]->(s)
         ON CREATE SET r.firstseen = timestamp()
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Relations=data,
         update_tag=update_tag,
@@ -184,7 +189,8 @@ def _attach_services(
         MERGE (s)-[r:ASSOCIATED_WITH]->(p)
         ON CREATE SET r.firstseen = timestamp()
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Relations=data,
         update_tag=update_tag,
@@ -206,7 +212,8 @@ def _attach_teams(
         MERGE (t)-[r:ASSOCIATED_WITH]->(p)
         ON CREATE SET r.firstseen = timestamp()
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Relations=data,
         update_tag=update_tag,

cartography/intel/pagerduty/schedules.py

@@ -7,6 +7,7 @@ import dateutil.parser
 import neo4j
 from pdpyras import APISession
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -63,7 +64,8 @@ def load_schedule_data(
                 layer["_schedule_id"] = schedule["id"]
                 layers.append(layer)
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Schedules=data,
         update_tag=update_tag,
@@ -87,7 +89,8 @@ def _attach_users(
         MERGE (u)-[r:MEMBER_OF]->(s)
         ON CREATE SET r.firstseen = timestamp()
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Relations=data,
         update_tag=update_tag,
@@ -129,7 +132,8 @@ def _attach_layers(
                 users.append(
                     {"layer_id": layer["_layer_id"], "user": user["user"]["id"]},
                 )
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Layers=data,
         update_tag=update_tag,
@@ -152,7 +156,8 @@ def _attach_layer_users(
         MERGE (u)-[r:MEMBER_OF]->(l)
         ON CREATE SET r.firstseen = timestamp()
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Relations=data,
         update_tag=update_tag,

cartography/intel/pagerduty/services.py

@@ -7,6 +7,7 @@ import dateutil.parser
 import neo4j
 from pdpyras import APISession
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -96,7 +97,8 @@ def load_service_data(
             for team in service["teams"]:
                 team_relations.append({"service": service["id"], "team": team["id"]})
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Services=data,
         update_tag=update_tag,
@@ -120,7 +122,8 @@ def _attach_teams(
         MERGE (t)-[r:ASSOCIATED_WITH]->(s)
         ON CREATE SET r.firstseen = timestamp()
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Relations=data,
         update_tag=update_tag,
@@ -162,7 +165,8 @@ def load_integration_data(
         created_at = dateutil.parser.parse(integration["created_at"])
         integration["created_at"] = int(created_at.timestamp())
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Integrations=data,
         update_tag=update_tag,

cartography/intel/pagerduty/teams.py

@@ -6,6 +6,7 @@ from typing import List
 import neo4j
 from pdpyras import APISession
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -68,7 +69,8 @@ def load_team_data(
     """
     logger.info(f"Loading {len(data)} pagerduty teams.")
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Teams=data,
         update_tag=update_tag,
@@ -90,7 +92,8 @@ def load_team_relations(
         ON CREATE SET r.firstseen = timestamp()
         SET r.role = relation.role
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Relations=data,
         update_tag=update_tag,

cartography/intel/pagerduty/users.py

@@ -6,6 +6,7 @@ from typing import List
 import neo4j
 from pdpyras import APISession
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -57,7 +58,8 @@ def load_user_data(
     """
     logger.info(f"Loading {len(data)} pagerduty users.")
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Users=data,
         update_tag=update_tag,

cartography/intel/pagerduty/vendors.py

@@ -6,6 +6,7 @@ from typing import List
 import neo4j
 from pdpyras import APISession
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -53,7 +54,8 @@ def load_vendor_data(
     """
     logger.info(f"Loading {len(data)} pagerduty vendors.")
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Vendors=data,
         update_tag=update_tag,