cartography 0.102.0rc2__py3-none-any.whl → 0.103.0rc1__py3-none-any.whl

cartography/models/snipeit/asset.py

@@ -16,18 +16,19 @@ class SnipeitAssetNodeProperties(CartographyNodeProperties):
     """
     https://snipe-it.readme.io/reference/hardware-list
     """
+
     # Common properties
-    id: PropertyRef = PropertyRef('id')
-    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+    id: PropertyRef = PropertyRef("id")
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
 
     # SnipeIT specific properties
-    asset_tag: PropertyRef = PropertyRef('asset_tag')
-    assigned_to: PropertyRef = PropertyRef('assigned_to.email')
-    category: PropertyRef = PropertyRef('category.name')
-    company: PropertyRef = PropertyRef('company.name')
-    manufacturer: PropertyRef = PropertyRef('manufacturer.name')
-    model: PropertyRef = PropertyRef('model.name')
-    serial: PropertyRef = PropertyRef('serial', extra_index=True)
+    asset_tag: PropertyRef = PropertyRef("asset_tag")
+    assigned_to: PropertyRef = PropertyRef("assigned_to.email")
+    category: PropertyRef = PropertyRef("category.name")
+    company: PropertyRef = PropertyRef("company.name")
+    manufacturer: PropertyRef = PropertyRef("manufacturer.name")
+    model: PropertyRef = PropertyRef("model.name")
+    serial: PropertyRef = PropertyRef("serial", extra_index=True)
 
 
 ###
@@ -35,45 +36,53 @@ class SnipeitAssetNodeProperties(CartographyNodeProperties):
 ###
 @dataclass(frozen=True)
 class SnipeitTenantToSnipeitAssetRelProperties(CartographyRelProperties):
-    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
 
 
 @dataclass(frozen=True)
 class SnipeitTenantToSnipeitAssetRel(CartographyRelSchema):
-    target_node_label: str = 'SnipeitTenant'
+    target_node_label: str = "SnipeitTenant"
     target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
-        {'id': PropertyRef('TENANT_ID', set_in_kwargs=True)},
+        {"id": PropertyRef("TENANT_ID", set_in_kwargs=True)},
     )
     direction: LinkDirection = LinkDirection.INWARD
     rel_label: str = "HAS_ASSET"
-    properties: SnipeitTenantToSnipeitAssetRelProperties = SnipeitTenantToSnipeitAssetRelProperties()
+    properties: SnipeitTenantToSnipeitAssetRelProperties = (
+        SnipeitTenantToSnipeitAssetRelProperties()
+    )
 
 
 ###
 # (:SnipeitUser)-[:HAS_CHECKED_OUT]->(:SnipeitAsset)
 ###
 @dataclass(frozen=True)
-class SnipeitUserToSnipeitAssetProperties(CartographyRelProperties):
-    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+class SnipeitUserToSnipeitAssetRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
 
 
 @dataclass(frozen=True)
 class SnipeitUserToSnipeitAssetRel(CartographyRelSchema):
-    target_node_label: str = 'SnipeitUser'
+    target_node_label: str = "SnipeitUser"
     target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
-        {'email': PropertyRef('assigned_to.email')},
+        {"email": PropertyRef("assigned_to.email")},
     )
     direction: LinkDirection = LinkDirection.INWARD
     rel_label: str = "HAS_CHECKED_OUT"
-    properties: SnipeitUserToSnipeitAssetProperties = SnipeitUserToSnipeitAssetProperties()
+    properties: SnipeitUserToSnipeitAssetRelProperties = (
+        SnipeitUserToSnipeitAssetRelProperties()
+    )
 
 
 ###
 @dataclass(frozen=True)
 class SnipeitAssetSchema(CartographyNodeSchema):
-    label: str = 'SnipeitAsset'  # The label of the node
-    properties: SnipeitAssetNodeProperties = SnipeitAssetNodeProperties()  # An object representing all properties
-    sub_resource_relationship: SnipeitTenantToSnipeitAssetRel = SnipeitTenantToSnipeitAssetRel()
+    label: str = "SnipeitAsset"  # The label of the node
+    properties: SnipeitAssetNodeProperties = (
+        SnipeitAssetNodeProperties()
+    )  # An object representing all properties
+    sub_resource_relationship: SnipeitTenantToSnipeitAssetRel = (
+        SnipeitTenantToSnipeitAssetRel()
+    )
     other_relationships: OtherRelationships = OtherRelationships(
         [
             SnipeitUserToSnipeitAssetRel(),

cartography/models/snipeit/tenant.py

@@ -7,11 +7,13 @@ from cartography.models.core.nodes import CartographyNodeSchema
 
 @dataclass(frozen=True)
 class SnipeitTenantNodeProperties(CartographyNodeProperties):
-    id: PropertyRef = PropertyRef('id')
-    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+    id: PropertyRef = PropertyRef("id")
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
 
 
 @dataclass(frozen=True)
 class SnipeitTenantSchema(CartographyNodeSchema):
-    label: str = 'SnipeitTenant'  # The label of the node
-    properties: SnipeitTenantNodeProperties = SnipeitTenantNodeProperties()  # An object representing all properties
+    label: str = "SnipeitTenant"  # The label of the node
+    properties: SnipeitTenantNodeProperties = (
+        SnipeitTenantNodeProperties()
+    )  # An object representing all properties

cartography/models/snipeit/user.py

@@ -15,35 +15,42 @@ class SnipeitUserNodeProperties(CartographyNodeProperties):
     """
     Ref: https://snipe-it.readme.io/reference/users
     """
+
     # Common properties
-    id: PropertyRef = PropertyRef('id')
-    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+    id: PropertyRef = PropertyRef("id")
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
 
     # SnipeIT specific properties
-    company: PropertyRef = PropertyRef('company_id.name', extra_index=True)
-    email: PropertyRef = PropertyRef('email', extra_index=True)
-    username: PropertyRef = PropertyRef('username')
+    company: PropertyRef = PropertyRef("company_id.name", extra_index=True)
+    email: PropertyRef = PropertyRef("email", extra_index=True)
+    username: PropertyRef = PropertyRef("username")
 
 
 @dataclass(frozen=True)
 class SnipeitTenantToSnipeitUserRelProperties(CartographyRelProperties):
-    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
 
 
 @dataclass(frozen=True)
 # (:SnipeitTenant)-[:HAS_USER]->(:SnipeitUser)
 class SnipeitTenantToSnipeitUserRel(CartographyRelSchema):
-    target_node_label: str = 'SnipeitTenant'
+    target_node_label: str = "SnipeitTenant"
     target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
-        {'id': PropertyRef('TENANT_ID', set_in_kwargs=True)},
+        {"id": PropertyRef("TENANT_ID", set_in_kwargs=True)},
     )
     direction: LinkDirection = LinkDirection.INWARD
     rel_label: str = "HAS_USER"
-    properties: SnipeitTenantToSnipeitUserRelProperties = SnipeitTenantToSnipeitUserRelProperties()
+    properties: SnipeitTenantToSnipeitUserRelProperties = (
+        SnipeitTenantToSnipeitUserRelProperties()
+    )
 
 
 @dataclass(frozen=True)
 class SnipeitUserSchema(CartographyNodeSchema):
-    label: str = 'SnipeitUser'  # The label of the node
-    properties: SnipeitUserNodeProperties = SnipeitUserNodeProperties()  # An object representing all properties
-    sub_resource_relationship: SnipeitTenantToSnipeitUserRel = SnipeitTenantToSnipeitUserRel()
+    label: str = "SnipeitUser"  # The label of the node
+    properties: SnipeitUserNodeProperties = (
+        SnipeitUserNodeProperties()
+    )  # An object representing all properties
+    sub_resource_relationship: SnipeitTenantToSnipeitUserRel = (
+        SnipeitTenantToSnipeitUserRel()
+    )

cartography/stats.py

@@ -17,11 +17,11 @@ class ScopedStatsClient:
 
     _client: StatsClient = None
 
-    def __init__(self, prefix: Optional[str], root: 'ScopedStatsClient'):
+    def __init__(self, prefix: Optional[str], root: "ScopedStatsClient"):
         self._scope_prefix = prefix
         self._root = root
 
-    def get_stats_client(self, scope: str) -> 'ScopedStatsClient':
+    def get_stats_client(self, scope: str) -> "ScopedStatsClient":
         """
         This method returns a new proxy to the same client
         which will prefix all calls to underlying methods with the scoped prefix
@@ -35,7 +35,7 @@ class ScopedStatsClient:
         return scoped_stats_client
 
     @staticmethod
-    def get_root_client() -> 'ScopedStatsClient':
+    def get_root_client() -> "ScopedStatsClient":
         client = ScopedStatsClient(prefix=None, root=None)  # type: ignore
         client._root = client
         return client

cartography/sync.py

@@ -1,7 +1,9 @@
 import argparse
 import logging
+import re
 import time
 from collections import OrderedDict
+from pkgutil import iter_modules
 from typing import Callable
 from typing import List
 from typing import Tuple
@@ -39,28 +41,30 @@ from cartography.util import STATUS_SUCCESS
 logger = logging.getLogger(__name__)
 
 
-TOP_LEVEL_MODULES = OrderedDict({  # preserve order so that the default sync always runs `analysis` at the very end
-    'create-indexes': cartography.intel.create_indexes.run,
-    'aws': cartography.intel.aws.start_aws_ingestion,
-    'azure': cartography.intel.azure.start_azure_ingestion,
-    'entra': cartography.intel.entra.start_entra_ingestion,
-    'crowdstrike': cartography.intel.crowdstrike.start_crowdstrike_ingestion,
-    'gcp': cartography.intel.gcp.start_gcp_ingestion,
-    'gsuite': cartography.intel.gsuite.start_gsuite_ingestion,
-    'cve': cartography.intel.cve.start_cve_ingestion,
-    'oci': cartography.intel.oci.start_oci_ingestion,
-    'okta': cartography.intel.okta.start_okta_ingestion,
-    'github': cartography.intel.github.start_github_ingestion,
-    'digitalocean': cartography.intel.digitalocean.start_digitalocean_ingestion,
-    'kandji': cartography.intel.kandji.start_kandji_ingestion,
-    'kubernetes': cartography.intel.kubernetes.start_k8s_ingestion,
-    'lastpass': cartography.intel.lastpass.start_lastpass_ingestion,
-    'bigfix': cartography.intel.bigfix.start_bigfix_ingestion,
-    'duo': cartography.intel.duo.start_duo_ingestion,
-    'semgrep': cartography.intel.semgrep.start_semgrep_ingestion,
-    'snipeit': cartography.intel.snipeit.start_snipeit_ingestion,
-    'analysis': cartography.intel.analysis.run,
-})
+TOP_LEVEL_MODULES = OrderedDict(
+    {  # preserve order so that the default sync always runs `analysis` at the very end
+        "create-indexes": cartography.intel.create_indexes.run,
+        "aws": cartography.intel.aws.start_aws_ingestion,
+        "azure": cartography.intel.azure.start_azure_ingestion,
+        "entra": cartography.intel.entra.start_entra_ingestion,
+        "crowdstrike": cartography.intel.crowdstrike.start_crowdstrike_ingestion,
+        "gcp": cartography.intel.gcp.start_gcp_ingestion,
+        "gsuite": cartography.intel.gsuite.start_gsuite_ingestion,
+        "cve": cartography.intel.cve.start_cve_ingestion,
+        "oci": cartography.intel.oci.start_oci_ingestion,
+        "okta": cartography.intel.okta.start_okta_ingestion,
+        "github": cartography.intel.github.start_github_ingestion,
+        "digitalocean": cartography.intel.digitalocean.start_digitalocean_ingestion,
+        "kandji": cartography.intel.kandji.start_kandji_ingestion,
+        "kubernetes": cartography.intel.kubernetes.start_k8s_ingestion,
+        "lastpass": cartography.intel.lastpass.start_lastpass_ingestion,
+        "bigfix": cartography.intel.bigfix.start_bigfix_ingestion,
+        "duo": cartography.intel.duo.start_duo_ingestion,
+        "semgrep": cartography.intel.semgrep.start_semgrep_ingestion,
+        "snipeit": cartography.intel.snipeit.start_snipeit_ingestion,
+        "analysis": cartography.intel.analysis.run,
+    }
+)
 
 
 class Sync:
@@ -98,7 +102,11 @@ class Sync:
         for name, func in stages:
             self.add_stage(name, func)
 
-    def run(self, neo4j_driver: neo4j.Driver, config: Union[Config, argparse.Namespace]) -> int:
+    def run(
+        self,
+        neo4j_driver: neo4j.Driver,
+        config: Union[Config, argparse.Namespace],
+    ) -> int:
         """
         Execute all stages in the sync task in sequence.
 
@@ -117,12 +125,77 @@ class Sync:
                     logger.warning("Sync interrupted during stage '%s'.", stage_name)
                     raise
                 except Exception:
-                    logger.exception("Unhandled exception during sync stage '%s'", stage_name)
+                    logger.exception(
+                        "Unhandled exception during sync stage '%s'",
+                        stage_name,
+                    )
                     raise  # TODO this should be configurable
                 logger.info("Finishing sync stage '%s'", stage_name)
         logger.info("Finishing sync with update tag '%d'", config.update_tag)
         return STATUS_SUCCESS
 
+    @classmethod
+    def list_intel_modules(cls) -> OrderedDict:
+        """
+        List all available intel modules.
+
+        This method will load all modules in the cartography.intel package and return a dictionary of their names and
+        their callable functions. The keys of the dictionary are the module names, and the values are the callable
+        functions (with `start_{module}_ingestion` pattern) that should be executed during the sync process.
+        analysis and create_indexes are loaded separately to ensure they are always available and run first
+        (for create-index) and last (for analysis).
+
+        :rtype: OrderedDict
+        :return: A dictionary of available intel modules.
+        """
+        available_modules = OrderedDict({})
+        available_modules["create-indexes"] = cartography.intel.create_indexes.run
+        callable_regex = re.compile(r"^start_(.+)_ingestion$")
+        # Load built-in modules
+        for intel_module_info in iter_modules(cartography.intel.__path__):
+            if intel_module_info.name in ("analysis", "create_indexes"):
+                continue
+            try:
+                logger.debug("Loading module: %s", intel_module_info.name)
+                intel_module = __import__(
+                    f"cartography.intel.{intel_module_info.name}",
+                    fromlist=[""],
+                )
+            except ImportError as e:
+                logger.error(
+                    "Failed to import module '%s'. Error: %s",
+                    intel_module_info.name,
+                    e,
+                )
+                continue
+            logger.debug("Loading module: %s", intel_module_info.name)
+            intel_module = __import__(
+                f"cartography.intel.{intel_module_info.name}",
+                fromlist=[""],
+            )
+            for k, v in intel_module.__dict__.items():
+                if not callable(v):
+                    continue
+                match_callable_name = callable_regex.match(k)
+                if not match_callable_name:
+                    continue
+                callable_module_name = (
+                    match_callable_name.group(1) if match_callable_name else None
+                )
+                if callable_module_name != intel_module_info.name:
+                    logger.debug(
+                        "Module name '%s' does not match intel module name '%s'.",
+                        callable_module_name,
+                        intel_module_info.name,
+                    )
+                available_modules[intel_module_info.name] = v
+        available_modules["analysis"] = cartography.intel.analysis.run
+        return available_modules
+
+
+# Used to avoid repeatedly calling Sync.list_intel_modules()
+TOP_LEVEL_MODULES = Sync.list_intel_modules()
+
 
 def run_with_config(sync: Sync, config: Union[Config, argparse.Namespace]) -> int:
     """
@@ -201,9 +274,12 @@ def build_default_sync() -> Sync:
     :return: The default cartography sync object.
     """
     sync = Sync()
-    sync.add_stages([
-        (stage_name, stage_func) for stage_name, stage_func in TOP_LEVEL_MODULES.items()
-    ])
+    sync.add_stages(
+        [
+            (stage_name, stage_func)
+            for stage_name, stage_func in TOP_LEVEL_MODULES.items()
+        ],
+    )
     return sync
 
 
@@ -214,18 +290,18 @@ def parse_and_validate_selected_modules(selected_modules: str) -> List[str]:
     :return: A validated list of module names that we will run
     """
     validated_modules: List[str] = []
-    for module in selected_modules.split(','):
+    for module in selected_modules.split(","):
         module = module.strip()
 
         if module in TOP_LEVEL_MODULES.keys():
             validated_modules.append(module)
         else:
-            valid_modules = ', '.join(TOP_LEVEL_MODULES.keys())
+            valid_modules = ", ".join(TOP_LEVEL_MODULES.keys())
             raise ValueError(
                 f'Error parsing `selected_modules`. You specified "{selected_modules}". '
-                f'Please check that your string is formatted properly. '
+                f"Please check that your string is formatted properly. "
                 f'Example valid input looks like "aws,gcp,analysis" or "azure, oci, crowdstrike". '
-                f'Our full list of valid values is: {valid_modules}.',
+                f"Our full list of valid values is: {valid_modules}.",
             )
     return validated_modules