cartography 0.104.0rc2__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/rules/runners.py

@@ -0,0 +1,216 @@
+"""
+Framework and Fact execution logic for Cartography rules.
+"""
+
+from neo4j import Driver
+from neo4j import GraphDatabase
+
+from cartography.client.core.tx import read_list_of_dicts_tx
+from cartography.rules.data.rules import RULES
+from cartography.rules.formatters import _format_and_output_results
+from cartography.rules.formatters import _generate_neo4j_browser_url
+from cartography.rules.spec.model import Fact
+from cartography.rules.spec.model import Maturity
+from cartography.rules.spec.model import Rule
+from cartography.rules.spec.result import CounterResult
+from cartography.rules.spec.result import FactResult
+from cartography.rules.spec.result import RuleResult
+
+
+def _run_fact(
+    fact: Fact,
+    rule: Rule,
+    driver: Driver,
+    database: str,
+    counter: CounterResult,
+    output_format: str,
+    neo4j_uri: str,
+) -> FactResult:
+    """Execute a single fact and return the result."""
+    if output_format == "text":
+        print(
+            f"\n\033[1mFact {counter.current_fact}/{counter.total_facts}: {fact.name}\033[0m"
+        )
+        # Display rule
+        print(f"  \033[36m{'Rule:':<12}\033[0m {rule.id} - {rule.name}")
+        # Display fact details
+        print(f"  \033[36m{'Fact ID:':<12}\033[0m {fact.id}")
+        print(f"  \033[36m{'Description:':<12}\033[0m {fact.description}")
+        print(f"  \033[36m{'Provider:':<12}\033[0m {fact.module.value}")
+
+        # Generate and display clickable Neo4j Browser URL
+        browser_url = _generate_neo4j_browser_url(neo4j_uri, fact.cypher_visual_query)
+        print(
+            f"  \033[36m{'Neo4j Query:':<12}\033[0m \033]8;;{browser_url}\033\\Click to run visual query\033]8;;\033\\"
+        )
+
+    with driver.session(database=database) as session:
+        raw_findings = session.execute_read(read_list_of_dicts_tx, fact.cypher_query)
+        findings = rule.parse_results(fact, raw_findings)
+        findings_count = len(findings)
+
+    if output_format == "text":
+        if findings_count > 0:
+            print(f"  \033[36m{'Results:':<12}\033[0m {findings_count} item(s) found")
+
+            # Show sample findings
+            print("    Sample results:")
+            for idx, finding in enumerate(findings[:3]):  # Show first 3
+                # Format rule output nicely
+                formatted_items = []
+                for key, value in finding.__class__.model_fields.items():
+                    if value is not None:
+                        # Truncate long values
+                        actual_value = getattr(finding, key)
+                        str_value = str(actual_value)
+                        if len(str_value) > 50:
+                            str_value = str_value[:47] + "..."
+                        formatted_items.append(f"{key}={str_value}")
+                if formatted_items:
+                    print(f"      {idx + 1}. {', '.join(formatted_items)}")
+
+            if findings_count > 3:
+                print(
+                    f"      ... and {findings_count - 3} more (use --output json to see all)"
+                )
+        else:
+            print(f"  \033[36m{'Results:':<12}\033[0m No items found")
+
+    # Create and return fact result
+    counter.total_findings += findings_count
+
+    return FactResult(
+        fact_id=fact.id,
+        fact_name=fact.name,
+        fact_description=fact.description,
+        fact_provider=fact.module.value,
+        findings=findings,
+    )
+
+
+def _run_single_rule(
+    rule_name: str,
+    driver: GraphDatabase.driver,
+    database: str,
+    output_format: str,
+    neo4j_uri: str,
+    fact_filter: str | None = None,
+    exclude_experimental: bool = False,
+) -> RuleResult:
+    """Execute a single rule and return results."""
+    rule = RULES[rule_name]
+    counter = CounterResult()
+
+    filtered_facts: list[Fact] = []
+    for fact in rule.facts:
+        if exclude_experimental and fact.maturity != Maturity.STABLE:
+            continue
+        if fact_filter:
+            if fact.id.lower() != fact_filter.lower():
+                continue
+        counter.total_facts += 1
+        filtered_facts.append(fact)
+
+    if output_format == "text":
+        print(f"Executing {rule.name} rule")
+        if fact_filter:
+            print(f"Filtered to fact: {fact_filter}")
+        print(f"Total facts: {counter.total_facts}")
+
+    # Execute requirements and collect results
+    rule_results = []
+
+    for fact in filtered_facts:
+        counter.current_fact += 1
+        fact_result = _run_fact(
+            fact,
+            rule,
+            driver,
+            database,
+            counter,
+            output_format,
+            neo4j_uri,
+        )
+        rule_results.append(fact_result)
+
+    # Create and return rule result
+    return RuleResult(
+        rule_id=rule.id,
+        rule_name=rule.name,
+        rule_description=rule.description,
+        facts=rule_results,
+        counter=counter,
+    )
+
+
+def run_rules(
+    rule_names: list[str],
+    uri: str,
+    neo4j_user: str,
+    neo4j_password: str,
+    neo4j_database: str,
+    output_format: str = "text",
+    fact_filter: str | None = None,
+    exclude_experimental: bool = False,
+):
+    """
+    Execute the specified rules and present results.
+
+    :param rule_names: The names of the rules to execute.
+    :param uri: The URI of the Neo4j database. E.g. "bolt://localhost:7687" or "neo4j+s://tenant123.databases.neo4j.io:7687"
+    :param neo4j_user: The username for the Neo4j database.
+    :param neo4j_password: The password for the Neo4j database.
+    :param neo4j_database: The name of the Neo4j database.
+    :param output_format: Either "text" or "json". Defaults to "text".
+    :param fact_filter: Optional fact ID to filter execution (case-insensitive).
+    :param exclude_experimental: Whether to exclude experimental facts from execution.
+    :return: The exit code.
+    """
+    # Validate all rules exist
+    for rule_name in rule_names:
+        if rule_name not in RULES:
+            if output_format == "text":
+                print(f"Unknown rule: {rule_name}")
+                print(f"Available rules: {', '.join(RULES.keys())}")
+            return 1
+
+    # Connect to Neo4j
+    if output_format == "text":
+        print(f"Connecting to Neo4j at {uri}...")
+    driver = GraphDatabase.driver(uri, auth=(neo4j_user, neo4j_password))
+
+    try:
+        driver.verify_connectivity()
+
+        # Execute rules
+        all_results = []
+        total_facts = 0
+        total_findings = 0
+
+        for i, rule_name in enumerate(rule_names):
+            if output_format == "text" and len(rule_names) > 1:
+                if i > 0:
+                    print("\n" + "=" * 60)
+                print(f"Executing rule {i + 1}/{len(rule_names)}: {rule_name}")
+
+            rule_result = _run_single_rule(
+                rule_name,
+                driver,
+                neo4j_database,
+                output_format,
+                uri,
+                fact_filter,
+                exclude_experimental,
+            )
+            all_results.append(rule_result)
+            total_facts += rule_result.counter.total_facts
+            total_findings += rule_result.counter.total_findings
+
+        # Output results
+        _format_and_output_results(
+            all_results, rule_names, output_format, total_facts, total_findings
+        )
+
+        return 0
+    finally:
+        driver.close()

cartography/rules/spec/model.py

@@ -0,0 +1,267 @@
+import json
+import logging
+from dataclasses import dataclass
+from dataclasses import field
+from enum import Enum
+from typing import Any
+from typing import no_type_check
+
+from pydantic import BaseModel
+from pydantic import ConfigDict
+from pydantic import model_validator
+
+logger = logging.getLogger(__name__)
+
+
+class Module(str, Enum):
+    """Services that can be monitored"""
+
+    AIRBYTE = "Airbyte"
+    """Airbyte data integration"""
+
+    ANTHROPIC = "Anthropic"
+    """Anthropic AI"""
+
+    AWS = "AWS"
+    """Amazon Web Services"""
+
+    AZURE = "Azure"
+    """Microsoft Azure"""
+
+    BIGFIX = "BigFix"
+    """BigFix patch management"""
+
+    CLOUDFLARE = "Cloudflare"
+    """Cloudflare services"""
+
+    CROWDSTRIKE = "CrowdStrike"
+    """CrowdStrike endpoint security"""
+
+    DIGITALOCEAN = "DigitalOcean"
+    """DigitalOcean cloud services"""
+
+    DUO = "Duo"
+    """Duo authentication"""
+
+    ENTRA = "Entra"
+    """Entra identity and access management"""
+
+    GCP = "GCP"
+    """Google Cloud Platform"""
+
+    GITHUB = "GitHub"
+    """GitHub source code management"""
+
+    GOOGLEWORKSPACE = "GoogleWorkspace"
+    """Google Workspace identity and access management"""
+
+    JAMF = "Jamf"
+    """Jamf endpoint security"""
+
+    KANDJI = "Kandji"
+    """Kandji endpoint security"""
+
+    KEYCLOAK = "Keycloak"
+    """Keycloak identity and access management"""
+
+    KUBERNETES = "Kubernetes"
+    """Kubernetes cluster security"""
+
+    LASTPASS = "LastPass"
+    """LastPass password manager"""
+
+    OCI = "OCI"
+    """Oracle Cloud Infrastructure"""
+
+    OKTA = "Okta"
+    """Okta identity and access management"""
+
+    OPENAI = "OpenAI"
+    """OpenAI"""
+
+    PAGERDUTY = "PagerDuty"
+    """PagerDuty incident response"""
+
+    SCALEWAY = "Scaleway"
+    """Scaleway cloud services"""
+
+    SEMGREP = "Semgrep"
+    """Semgrep code security"""
+
+    SENTINELONE = "SentinelOne"
+    """SentinelOne endpoint security"""
+
+    SNIPEIT = "Snipe-IT"
+    """Snipe-IT asset management"""
+
+    SPACELIFT = "SpaceLift"
+    """SpaceLift infrastructure as code"""
+
+    TAILSCALE = "TailScale"
+    """TailScale VPN"""
+
+    TRIVY = "Trivy"
+    """Trivy vulnerability scanner"""
+
+    CROSS_CLOUD = "Cross-Cloud"
+    """Multi-cloud or provider-agnostic rules"""
+
+
+class Maturity(str, Enum):
+    """Maturity levels for Facts."""
+
+    EXPERIMENTAL = "EXPERIMENTAL"
+    """Experimental: Initial version, may be unstable or incomplete."""
+
+    STABLE = "STABLE"
+    """Stable: Well-tested and reliable for production use."""
+
+
+MODULE_TO_CARTOGRAPHY_INTEL = {
+    Module.AIRBYTE: "airbyte",
+    Module.ANTHROPIC: "anthropic",
+    Module.AWS: "aws",
+    Module.AZURE: "azure",
+    Module.BIGFIX: "bigfix",
+    Module.CLOUDFLARE: "cloudflare",
+    Module.CROWDSTRIKE: "crowdstrike",
+    Module.DIGITALOCEAN: "digitalocean",
+    Module.DUO: "duo",
+    Module.ENTRA: "entra",
+    Module.GCP: "gcp",
+    Module.GITHUB: "github",
+    Module.GOOGLEWORKSPACE: "googleworkspace",
+    Module.JAMF: "jamf",
+    Module.KANDJI: "kandji",
+    Module.KEYCLOAK: "keycloak",
+    Module.KUBERNETES: "kubernetes",
+    Module.LASTPASS: "lastpass",
+    Module.OCI: "oci",
+    Module.OKTA: "okta",
+    Module.OPENAI: "openai",
+    Module.PAGERDUTY: "pagerduty",
+    Module.SCALEWAY: "scaleway",
+    Module.SEMGREP: "semgrep",
+    Module.SENTINELONE: "sentinelone",
+    Module.SNIPEIT: "snipeit",
+    Module.SPACELIFT: "spacelift",
+    Module.TAILSCALE: "tailscale",
+    Module.TRIVY: "trivy",
+}
+
+
+@dataclass(frozen=True)
+class RuleReference:
+    """A reference document for a Rule."""
+
+    text: str
+    url: str
+
+
+@dataclass(frozen=True)
+class Fact:
+    """A Fact gathers information about the environment using a Cypher query."""
+
+    id: str
+    """A descriptive identifier for the Fact. By convention, should be lowercase and use underscores like `rule-name-module`."""
+    name: str
+    """A descriptive name for the Fact."""
+    description: str
+    """More details about the Fact. Information on details that we're querying for."""
+    module: Module
+    """The Module that the Fact is associated with e.g. AWS, Azure, GCP, etc."""
+    maturity: Maturity
+    """The maturity level of the Fact query."""
+    # TODO can we lint the queries. full-on integ tests here are overkill though.
+    cypher_query: str
+    """The Cypher query to gather information about the environment. Returns data field by field e.g. `RETURN node.prop1, node.prop2`."""
+    cypher_visual_query: str
+    """
+    Same as `cypher_query`, returns it in a visual format for the web interface with `.. RETURN *`.
+    Often includes additional relationships to help give context.
+    """
+
+
+class Finding(BaseModel):
+    """Base class for Rule finding models."""
+
+    # TODO: make this property mandatory one all modules have been updated to new datamodel
+    source: str | None = None
+    """The source of the Fact data, e.g. the specific Cartography module that ingested the data. This field is useful especially for CROSS_CLOUD facts."""
+    extra: dict[str, Any] = {}
+    """A dictionary to hold any extra fields returned by the Fact query that are not explicitly defined in the output model."""
+
+    # Config to coerce numbers to strings during instantiation
+    model_config = ConfigDict(coerce_numbers_to_str=True)
+
+    # Coerce o strings
+    @no_type_check
+    @model_validator(mode="before")
+    @classmethod
+    def coerce_to_string(cls, data: Any) -> Any:
+        if not isinstance(data, dict):
+            return data
+
+        for name, f in cls.model_fields.items():
+            if f.annotation is not str:
+                continue
+            if name not in data:
+                continue
+            v = data[name]
+            if isinstance(v, (list, tuple, set)):
+                data[name] = ", ".join(v)
+            if isinstance(v, dict):
+                data[name] = json.dumps(v)
+
+        return data
+
+
+@dataclass(frozen=True)
+class Rule:
+    """A Rule represents a security issue or misconfiguration detected in the environment."""
+
+    id: str
+    """A unique identifier for the Rule. Should be globally unique within Cartography."""
+    name: str
+    """A brief name for the Rule."""
+    tags: tuple[str, ...]
+    """Tags associated with the Rule for categorization and filtering."""
+    description: str
+    """A brief description of the Rule. Can include details about the security issue or misconfiguration."""
+    version: str
+    """The version of the Rule definition."""
+    facts: tuple[Fact, ...]
+    """The Facts that contribute to this Rule."""
+    output_model: type[Finding]
+    """The output model class for the Rule."""
+    references: list[RuleReference] = field(default_factory=list)
+    """References or links to external resources related to the Rule."""
+
+    @property
+    def modules(self) -> set[Module]:
+        """Returns the set of modules associated with this rule."""
+        return {fact.module for fact in self.facts}
+
+    def get_fact_by_id(self, fact_id: str) -> Fact | None:
+        """Returns a fact by its ID, or None if not found."""
+        for fact in self.facts:
+            if fact.id.lower() == fact_id.lower():
+                return fact
+        return None
+
+    def parse_results(
+        self, fact: Fact, fact_results: list[dict[str, Any]]
+    ) -> list[Finding]:
+        # DOC
+        result: list[Finding] = []
+        for result_item in fact_results:
+            parsed_output: dict[str, Any] = {"extra": {}, "source": fact.module.value}
+            for key, value in result_item.items():
+                if value is None:
+                    continue
+                if key not in self.output_model.model_fields:
+                    parsed_output["extra"][key] = value
+                else:
+                    parsed_output[key] = value
+            result.append(self.output_model(**parsed_output))
+        return result

cartography/rules/spec/result.py

@@ -0,0 +1,38 @@
+# Execution result classes
+from dataclasses import dataclass
+from dataclasses import field
+
+from cartography.rules.spec.model import Finding
+
+
+@dataclass
+class CounterResult:
+    current_fact: int = 0
+    total_facts: int = 0
+    total_findings: int = 0
+
+
+@dataclass
+class FactResult:
+    """
+    Results for a single Fact.
+    """
+
+    fact_id: str
+    fact_name: str
+    fact_description: str
+    fact_provider: str
+    findings: list[Finding] = field(default_factory=list)
+
+
+@dataclass
+class RuleResult:
+    """
+    Results for a single Rule.
+    """
+
+    rule_id: str
+    rule_name: str
+    rule_description: str
+    counter: CounterResult
+    facts: list[FactResult] = field(default_factory=list)

cartography/sync.py

@@ -13,6 +13,7 @@ import neo4j.exceptions
 from neo4j import GraphDatabase
 from statsd import StatsClient
 
+import cartography.intel.airbyte
 import cartography.intel.analysis
 import cartography.intel.anthropic
 import cartography.intel.aws
@@ -27,16 +28,26 @@ import cartography.intel.duo
 import cartography.intel.entra
 import cartography.intel.gcp
 import cartography.intel.github
+import cartography.intel.googleworkspace
 import cartography.intel.gsuite
+import cartography.intel.jamf
 import cartography.intel.kandji
+import cartography.intel.keycloak
 import cartography.intel.kubernetes
 import cartography.intel.lastpass
 import cartography.intel.oci
 import cartography.intel.okta
+import cartography.intel.ontology
 import cartography.intel.openai
+import cartography.intel.pagerduty
+import cartography.intel.scaleway
 import cartography.intel.semgrep
+import cartography.intel.sentinelone
+import cartography.intel.slack
 import cartography.intel.snipeit
+import cartography.intel.spacelift
 import cartography.intel.tailscale
+import cartography.intel.trivy
 from cartography.config import Config
 from cartography.stats import set_stats_client
 from cartography.util import STATUS_FAILURE
@@ -45,9 +56,10 @@ from cartography.util import STATUS_SUCCESS
 logger = logging.getLogger(__name__)
 
 
-TOP_LEVEL_MODULES = OrderedDict(
+TOP_LEVEL_MODULES: OrderedDict[str, Callable[..., None]] = OrderedDict(
     {  # preserve order so that the default sync always runs `analysis` at the very end
         "create-indexes": cartography.intel.create_indexes.run,
+        "airbyte": cartography.intel.airbyte.start_airbyte_ingestion,
         "anthropic": cartography.intel.anthropic.start_anthropic_ingestion,
         "aws": cartography.intel.aws.start_aws_ingestion,
         "azure": cartography.intel.azure.start_azure_ingestion,
@@ -55,6 +67,7 @@ TOP_LEVEL_MODULES = OrderedDict(
         "cloudflare": cartography.intel.cloudflare.start_cloudflare_ingestion,
         "crowdstrike": cartography.intel.crowdstrike.start_crowdstrike_ingestion,
         "gcp": cartography.intel.gcp.start_gcp_ingestion,
+        "googleworkspace": cartography.intel.googleworkspace.start_googleworkspace_ingestion,
         "gsuite": cartography.intel.gsuite.start_gsuite_ingestion,
         "cve": cartography.intel.cve.start_cve_ingestion,
         "oci": cartography.intel.oci.start_oci_ingestion,
@@ -63,13 +76,23 @@ TOP_LEVEL_MODULES = OrderedDict(
         "github": cartography.intel.github.start_github_ingestion,
         "digitalocean": cartography.intel.digitalocean.start_digitalocean_ingestion,
         "kandji": cartography.intel.kandji.start_kandji_ingestion,
+        "keycloak": cartography.intel.keycloak.start_keycloak_ingestion,
         "kubernetes": cartography.intel.kubernetes.start_k8s_ingestion,
         "lastpass": cartography.intel.lastpass.start_lastpass_ingestion,
         "bigfix": cartography.intel.bigfix.start_bigfix_ingestion,
         "duo": cartography.intel.duo.start_duo_ingestion,
+        "scaleway": cartography.intel.scaleway.start_scaleway_ingestion,
         "semgrep": cartography.intel.semgrep.start_semgrep_ingestion,
         "snipeit": cartography.intel.snipeit.start_snipeit_ingestion,
         "tailscale": cartography.intel.tailscale.start_tailscale_ingestion,
+        "jamf": cartography.intel.jamf.start_jamf_ingestion,
+        "pagerduty": cartography.intel.pagerduty.start_pagerduty_ingestion,
+        "trivy": cartography.intel.trivy.start_trivy_ingestion,
+        "sentinelone": cartography.intel.sentinelone.start_sentinelone_ingestion,
+        "slack": cartography.intel.slack.start_slack_ingestion,
+        "spacelift": cartography.intel.spacelift.start_spacelift_ingestion,
+        "ontology": cartography.intel.ontology.run,
+        # Analysis should be the last stage
         "analysis": cartography.intel.analysis.run,
     }
 )
@@ -197,14 +220,11 @@ class Sync:
                         intel_module_info.name,
                     )
                 available_modules[intel_module_info.name] = v
+        available_modules["ontology"] = cartography.intel.ontology.run
         available_modules["analysis"] = cartography.intel.analysis.run
         return available_modules
 
 
-# Used to avoid repeatedly calling Sync.list_intel_modules()
-TOP_LEVEL_MODULES = Sync.list_intel_modules()
-
-
 def run_with_config(sync: Sync, config: Union[Config, argparse.Namespace]) -> int:
     """
     Execute the cartography.sync.Sync.run method with parameters built from the given configuration object.