sigma2sql 0.2.0__py3-none-any.whl

sigma2sql/profiles/evtx.yml

@@ -0,0 +1,71 @@
+description: Windows event logs exported to JSON by evtx_dump or python-evtx (Event.System / Event.EventData)
+nested: true
+timestamp: "Event.System.TimeCreated.#attributes.SystemTime"
+# Some providers (e.g. Defender) name fields with spaces: "New Value" for NewValue.
+default_field: ["Event.EventData.{field}", "Event.EventData.{field_words}"]
+fields:
+  # EventID carries Qualifiers on some logs and then becomes {"#attributes": ..., "#text": N}.
+  EventID: [Event.System.EventID.#text, Event.System.EventID]
+  Channel: Event.System.Channel
+  Computer: Event.System.Computer
+  Provider_Name: Event.System.Provider.#attributes.Name
+  Level: Event.System.Level
+  Keywords: Event.System.Keywords
+  # WMI-Activity events keep their data under UserData, one element per event type.
+  Operation:
+    - Event.EventData.Operation
+    - Event.UserData.Operation_ClientFailure.Operation
+    - Event.UserData.Operation_StartedOperational.Operation
+  ClientMachine: [Event.EventData.ClientMachine, Event.UserData.Operation_ClientFailure.ClientMachine]
+  PossibleCause: [Event.EventData.PossibleCause, Event.UserData.Operation_ClientFailure.PossibleCause]
+  Consumer: [Event.EventData.Consumer, Event.UserData.Operation_ESStoConsumerBinding.CONSUMER]
+  Query: [Event.EventData.Query, Event.UserData.Operation_TemporaryEssStarted.Query]
+logsources:
+  # Sysmon categories: the Sysmon channel plus the category's event ID.
+  - {match: {product: windows, category: process_creation}, where: {Event.System.Channel: Microsoft-Windows-Sysmon/Operational, Event.System.EventID: 1}}
+  - {match: {product: windows, category: file_change}, where: {Event.System.Channel: Microsoft-Windows-Sysmon/Operational, Event.System.EventID: 2}}
+  - {match: {product: windows, category: network_connection}, where: {Event.System.Channel: Microsoft-Windows-Sysmon/Operational, Event.System.EventID: 3}}
+  - {match: {product: windows, category: process_termination}, where: {Event.System.Channel: Microsoft-Windows-Sysmon/Operational, Event.System.EventID: 5}}
+  - {match: {product: windows, category: driver_load}, where: {Event.System.Channel: Microsoft-Windows-Sysmon/Operational, Event.System.EventID: 6}}
+  - {match: {product: windows, category: image_load}, where: {Event.System.Channel: Microsoft-Windows-Sysmon/Operational, Event.System.EventID: 7}}
+  - {match: {product: windows, category: create_remote_thread}, where: {Event.System.Channel: Microsoft-Windows-Sysmon/Operational, Event.System.EventID: 8}}
+  - {match: {product: windows, category: raw_access_thread}, where: {Event.System.Channel: Microsoft-Windows-Sysmon/Operational, Event.System.EventID: 9}}
+  - {match: {product: windows, category: process_access}, where: {Event.System.Channel: Microsoft-Windows-Sysmon/Operational, Event.System.EventID: 10}}
+  - {match: {product: windows, category: file_event}, where: {Event.System.Channel: Microsoft-Windows-Sysmon/Operational, Event.System.EventID: 11}}
+  - {match: {product: windows, category: registry_add}, where: {Event.System.Channel: Microsoft-Windows-Sysmon/Operational, Event.System.EventID: 12}}
+  - {match: {product: windows, category: registry_delete}, where: {Event.System.Channel: Microsoft-Windows-Sysmon/Operational, Event.System.EventID: 12}}
+  - {match: {product: windows, category: registry_set}, where: {Event.System.Channel: Microsoft-Windows-Sysmon/Operational, Event.System.EventID: 13}}
+  - {match: {product: windows, category: registry_rename}, where: {Event.System.Channel: Microsoft-Windows-Sysmon/Operational, Event.System.EventID: 14}}
+  - {match: {product: windows, category: registry_event}, where: {Event.System.Channel: Microsoft-Windows-Sysmon/Operational, Event.System.EventID: [12, 13, 14]}}
+  - {match: {product: windows, category: create_stream_hash}, where: {Event.System.Channel: Microsoft-Windows-Sysmon/Operational, Event.System.EventID: 15}}
+  - {match: {product: windows, category: pipe_created}, where: {Event.System.Channel: Microsoft-Windows-Sysmon/Operational, Event.System.EventID: [17, 18]}}
+  - {match: {product: windows, category: wmi_event}, where: {Event.System.Channel: Microsoft-Windows-Sysmon/Operational, Event.System.EventID: [19, 20, 21]}}
+  - {match: {product: windows, category: dns_query}, where: {Event.System.Channel: Microsoft-Windows-Sysmon/Operational, Event.System.EventID: 22}}
+  - {match: {product: windows, category: file_delete}, where: {Event.System.Channel: Microsoft-Windows-Sysmon/Operational, Event.System.EventID: [23, 26]}}
+  - {match: {product: windows, category: clipboard_capture}, where: {Event.System.Channel: Microsoft-Windows-Sysmon/Operational, Event.System.EventID: 24}}
+  - {match: {product: windows, category: process_tampering}, where: {Event.System.Channel: Microsoft-Windows-Sysmon/Operational, Event.System.EventID: 25}}
+  - {match: {product: windows, category: file_executable_detected}, where: {Event.System.Channel: Microsoft-Windows-Sysmon/Operational, Event.System.EventID: 29}}
+  # PowerShell categories.
+  - {match: {product: windows, category: ps_script}, where: {Event.System.Channel: Microsoft-Windows-PowerShell/Operational, Event.System.EventID: 4104}}
+  - {match: {product: windows, category: ps_module}, where: {Event.System.Channel: Microsoft-Windows-PowerShell/Operational, Event.System.EventID: 4103}}
+  - {match: {product: windows, category: ps_classic_start}, where: {Event.System.Channel: Windows PowerShell, Event.System.EventID: 400}}
+  - {match: {product: windows, category: ps_classic_provider_start}, where: {Event.System.Channel: Windows PowerShell, Event.System.EventID: 600}}
+  # Services map to their channel.
+  - {match: {product: windows, service: security}, where: {Event.System.Channel: Security}}
+  - {match: {product: windows, service: system}, where: {Event.System.Channel: System}}
+  - {match: {product: windows, service: application}, where: {Event.System.Channel: Application}}
+  - {match: {product: windows, service: sysmon}, where: {Event.System.Channel: Microsoft-Windows-Sysmon/Operational}}
+  - {match: {product: windows, service: powershell}, where: {Event.System.Channel: Microsoft-Windows-PowerShell/Operational}}
+  - {match: {product: windows, service: powershell-classic}, where: {Event.System.Channel: Windows PowerShell}}
+  - {match: {product: windows, service: taskscheduler}, where: {Event.System.Channel: Microsoft-Windows-TaskScheduler/Operational}}
+  - {match: {product: windows, service: wmi}, where: {Event.System.Channel: Microsoft-Windows-WMI-Activity/Operational}}
+  - {match: {product: windows, service: windefend}, where: {Event.System.Channel: Microsoft-Windows-Windows Defender/Operational}}
+  - {match: {product: windows, service: bits-client}, where: {Event.System.Channel: Microsoft-Windows-Bits-Client/Operational}}
+  - {match: {product: windows, service: firewall-as}, where: {Event.System.Channel: Microsoft-Windows-Windows Firewall With Advanced Security/Firewall}}
+  - {match: {product: windows, service: appxdeployment-server}, where: {Event.System.Channel: Microsoft-Windows-AppXDeploymentServer/Operational}}
+  - {match: {product: windows, service: codeintegrity-operational}, where: {Event.System.Channel: Microsoft-Windows-CodeIntegrity/Operational}}
+  - {match: {product: windows, service: dns-client}, where: {Event.System.Channel: Microsoft-Windows-DNS Client Events/Operational}}
+  - {match: {product: windows, service: driver-framework}, where: {Event.System.Channel: Microsoft-Windows-DriverFrameworks-UserMode/Operational}}
+  - {match: {product: windows, service: ntlm}, where: {Event.System.Channel: Microsoft-Windows-NTLM/Operational}}
+  - {match: {product: windows, service: terminalservices-localsessionmanager}, where: {Event.System.Channel: Microsoft-Windows-TerminalServices-LocalSessionManager/Operational}}
+  - {match: {product: windows, service: openssh}, where: {Event.System.Channel: OpenSSH/Operational}}

sigma2sql/profiles/ocsf.yml

@@ -0,0 +1,26 @@
+description: OCSF 1.x events (nested JSON, time in epoch milliseconds), routed by class_uid
+nested: true
+timestamp: time
+timestamp_format: epoch_ms
+fields:
+  Image: process.file.path
+  CommandLine: process.cmd_line
+  ProcessId: process.pid
+  CurrentDirectory: process.file.parent_folder
+  ParentImage: actor.process.file.path
+  ParentCommandLine: actor.process.cmd_line
+  ParentProcessId: actor.process.pid
+  User: actor.user.name
+  Computer: device.hostname
+  TargetFilename: file.path
+  DestinationIp: dst_endpoint.ip
+  DestinationPort: dst_endpoint.port
+  DestinationHostname: dst_endpoint.hostname
+  SourceIp: src_endpoint.ip
+  SourcePort: src_endpoint.port
+  QueryName: query.hostname
+logsources:
+  - {match: {category: process_creation}, where: {class_uid: 1007, activity_id: 1}}
+  - {match: {category: file_event}, where: {class_uid: 1001}}
+  - {match: {category: network_connection}, where: {class_uid: 4001}}
+  - {match: {category: dns_query}, where: {class_uid: 4003}}

sigma2sql/profiles/sysmon.yml

@@ -0,0 +1,26 @@
+description: Sysmon events with their original field names; each Sigma category is limited to its Sysmon event ID
+timestamp: UtcTime
+logsources:
+  - {match: {product: windows, category: process_creation}, where: {EventID: 1}}
+  - {match: {product: windows, category: file_change}, where: {EventID: 2}}
+  - {match: {product: windows, category: network_connection}, where: {EventID: 3}}
+  - {match: {product: windows, category: process_termination}, where: {EventID: 5}}
+  - {match: {product: windows, category: driver_load}, where: {EventID: 6}}
+  - {match: {product: windows, category: image_load}, where: {EventID: 7}}
+  - {match: {product: windows, category: create_remote_thread}, where: {EventID: 8}}
+  - {match: {product: windows, category: raw_access_thread}, where: {EventID: 9}}
+  - {match: {product: windows, category: process_access}, where: {EventID: 10}}
+  - {match: {product: windows, category: file_event}, where: {EventID: 11}}
+  - {match: {product: windows, category: registry_add}, where: {EventID: 12}}
+  - {match: {product: windows, category: registry_delete}, where: {EventID: 12}}
+  - {match: {product: windows, category: registry_set}, where: {EventID: 13}}
+  - {match: {product: windows, category: registry_rename}, where: {EventID: 14}}
+  - {match: {product: windows, category: registry_event}, where: {EventID: [12, 13, 14]}}
+  - {match: {product: windows, category: create_stream_hash}, where: {EventID: 15}}
+  - {match: {product: windows, category: pipe_created}, where: {EventID: [17, 18]}}
+  - {match: {product: windows, category: wmi_event}, where: {EventID: [19, 20, 21]}}
+  - {match: {product: windows, category: dns_query}, where: {EventID: 22}}
+  - {match: {product: windows, category: file_delete}, where: {EventID: [23, 26]}}
+  - {match: {product: windows, category: clipboard_capture}, where: {EventID: 24}}
+  - {match: {product: windows, category: process_tampering}, where: {EventID: 25}}
+  - {match: {product: windows, category: file_executable_detected}, where: {EventID: 29}}

sigma2sql/rule.py

@@ -0,0 +1,260 @@
+from __future__ import annotations
+
+import dataclasses
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass, replace
+from pathlib import Path
+from typing import Any, Literal
+
+import yaml
+
+from sigma2sql.condition import compile_condition
+from sigma2sql.detection import compile_search
+from sigma2sql.errors import RuleError, UnsupportedRule
+from sigma2sql.expr import Expr, any_of
+
+CorrelationType = Literal["event_count", "value_count", "value_sum", "value_avg", "temporal"]
+CORRELATION_TYPES: tuple[CorrelationType, ...] = (
+    "event_count",
+    "value_count",
+    "value_sum",
+    "value_avg",
+    "temporal",
+)
+OPS = {"gt": ">", "gte": ">=", "lt": "<", "lte": "<=", "eq": "="}
+LEGACY_OPS = {">": "gt", ">=": "gte", "<": "lt", "<=": "lte", "=": "eq", "==": "eq"}
+TIMESPAN = re.compile(r"^\s*(\d+)\s*([smhdw])\s*$")
+UNIT = {"s": 1, "m": 60, "h": 3600, "d": 86400, "w": 604800}
+# `selection | count(field) by a, b > 5`, the pre-correlation aggregation syntax.
+AGGREGATION = re.compile(
+    r"^count\(\s*(?P<field>[\w.-]*)\s*\)"
+    r"(?:\s+by\s+(?P<by>[\w.-]+(?:\s*,\s*[\w.-]+)*))?"
+    r"\s*(?P<op>>=|<=|==|=|>|<)\s*(?P<value>\d+)$"
+)
+
+Condition = tuple[tuple[str, float], ...]
+
+
+@dataclass(frozen=True)
+class Rule:
+    title: str
+    id: str | None
+    level: str | None
+    searches: dict[str, Any]
+    conditions: tuple[str, ...]
+    logsource: Mapping[str, str] = dataclasses.field(default_factory=dict)
+    name: str | None = None
+    # Set for rules written with the legacy `| count() ...` condition syntax.
+    aggregation: Correlation | None = None
+    path: Path | None = dataclasses.field(default=None, compare=False)
+
+    def compile(self) -> Expr:
+        """Build the match tree. Searches the condition never uses are not compiled."""
+        cache: dict[str, Expr] = {}
+
+        def resolve(name: str) -> Expr:
+            if name not in cache:
+                cache[name] = compile_search(name, self.searches[name])
+            return cache[name]
+
+        names = list(self.searches)
+        return any_of([compile_condition(c, names, resolve) for c in self.conditions])
+
+    @property
+    def ref(self) -> str:
+        """The name correlations use for this rule."""
+        return self.name or self.id or self.title
+
+
+@dataclass(frozen=True)
+class Correlation:
+    """A Sigma correlation rule: a count or sequence over the matches of other rules."""
+
+    title: str
+    id: str | None
+    level: str | None
+    type: CorrelationType
+    rules: tuple[str, ...]
+    group_by: tuple[str, ...] = ()
+    timespan: int | None = None  # seconds
+    condition: Condition = ()
+    field: str | None = None
+    aliases: Mapping[str, Mapping[str, str]] = dataclasses.field(default_factory=dict)
+    generate: bool = False
+    name: str | None = None
+    path: Path | None = dataclasses.field(default=None, compare=False)
+
+
+Document = Rule | Correlation
+
+
+def load_documents(text: str, path: Path | None = None) -> list[Document]:
+    try:
+        docs = [d for d in yaml.safe_load_all(text) if d is not None]
+    except yaml.YAMLError as exc:
+        raise RuleError(f"invalid YAML: {exc}") from exc
+    if not docs:
+        raise RuleError("empty rule file")
+    out: list[Document] = []
+    for doc in docs:
+        if not isinstance(doc, dict):
+            raise RuleError("a rule must be a YAML map")
+        if "action" in doc:
+            raise UnsupportedRule("legacy rule collections ('action: global') are not supported")
+        if "correlation" in doc:
+            out.append(_correlation(doc, path))
+        elif "filter" in doc:
+            raise UnsupportedRule("filter rules are not supported")
+        else:
+            out.append(_rule(doc, path))
+    return out
+
+
+def load_file(path: Path) -> list[Document]:
+    return load_documents(path.read_text(encoding="utf-8"), path)
+
+
+def load_rule(text: str, path: Path | None = None) -> Rule:
+    """Load a file that holds exactly one detection rule."""
+    docs = load_documents(text, path)
+    if len(docs) != 1 or not isinstance(docs[0], Rule):
+        raise UnsupportedRule("expected a single detection rule")
+    return docs[0]
+
+
+def load_rule_file(path: Path) -> Rule:
+    return load_rule(path.read_text(encoding="utf-8"), path)
+
+
+def _meta(doc: dict[str, Any], path: Path | None) -> dict[str, Any]:
+    def opt(key: str) -> str | None:
+        value = doc.get(key)
+        return str(value) if value is not None else None
+
+    return {
+        "title": str(doc.get("title") or (path.stem if path else "untitled")),
+        "id": opt("id"),
+        "level": opt("level"),
+        "name": opt("name"),
+        "path": path,
+    }
+
+
+def _rule(doc: dict[str, Any], path: Path | None) -> Rule:
+    detection = doc.get("detection")
+    if not isinstance(detection, dict):
+        raise RuleError("missing 'detection' section")
+
+    condition = detection.get("condition")
+    if isinstance(condition, str):
+        conditions: tuple[str, ...] = (condition,)
+    elif isinstance(condition, list) and condition and all(isinstance(c, str) for c in condition):
+        conditions = tuple(condition)
+    else:
+        raise RuleError("missing or invalid 'detection.condition'")
+
+    searches = {str(k): v for k, v in detection.items() if k not in ("condition", "timeframe")}
+    if not searches:
+        raise RuleError("detection has no searches")
+
+    logsource = doc.get("logsource") or {}
+    if not isinstance(logsource, dict):
+        raise RuleError("'logsource' must be a map")
+
+    rule = Rule(
+        searches=searches,
+        conditions=conditions,
+        logsource={str(k): str(v) for k, v in logsource.items() if v is not None},
+        **_meta(doc, path),
+    )
+    if not any("|" in c for c in conditions):
+        return rule
+    if len(conditions) != 1:
+        raise UnsupportedRule("aggregations need a single condition")
+    base, aggregation = (part.strip() for part in conditions[0].split("|", 1))
+    timeframe = detection.get("timeframe")
+    rule = replace(rule, conditions=(base,))
+    return replace(rule, aggregation=_aggregation(rule, aggregation, timeframe))
+
+
+def _aggregation(rule: Rule, text: str, timeframe: Any) -> Correlation:
+    m = AGGREGATION.match(text)
+    if not m:
+        raise UnsupportedRule(f"aggregation {text!r} is not supported (only count)")
+    by = tuple(g.strip() for g in m["by"].split(",")) if m["by"] else ()
+    return Correlation(
+        title=rule.title,
+        id=rule.id,
+        level=rule.level,
+        type="value_count" if m["field"] else "event_count",
+        rules=(rule.ref,),
+        group_by=by,
+        timespan=_timespan(timeframe) if timeframe is not None else None,
+        condition=((LEGACY_OPS[m["op"]], float(m["value"])),),
+        field=m["field"] or None,
+        path=rule.path,
+    )
+
+
+def _correlation(doc: dict[str, Any], path: Path | None) -> Correlation:
+    spec = doc["correlation"]
+    if not isinstance(spec, dict):
+        raise RuleError("'correlation' must be a map")
+    kind = spec.get("type")
+    if kind == "temporal_ordered":
+        raise UnsupportedRule("temporal_ordered correlations are not supported")
+    if kind not in CORRELATION_TYPES:
+        raise UnsupportedRule(f"correlation type {kind!r} is not supported")
+
+    rules = spec.get("rules")
+    if isinstance(rules, str):
+        rules = [rules]
+    if not isinstance(rules, list) or not rules:
+        raise RuleError("correlation needs a list of 'rules'")
+
+    group_by = spec.get("group-by") or []
+    if isinstance(group_by, str):
+        group_by = [group_by]
+
+    raw_condition = spec.get("condition") or {}
+    if not isinstance(raw_condition, dict):
+        raise RuleError("correlation 'condition' must be a map")
+    value_field = raw_condition.get("field", spec.get("field"))
+    condition: list[tuple[str, float]] = []
+    for op, value in raw_condition.items():
+        if op == "field":
+            continue
+        if op not in OPS or isinstance(value, bool) or not isinstance(value, int | float):
+            raise RuleError(f"invalid correlation condition {op}: {value!r}")
+        condition.append((op, float(value)))
+    if kind == "temporal" and not condition:
+        condition = [("gte", float(len(rules)))]
+    if not condition:
+        raise RuleError("correlation needs a condition")
+    if kind in ("value_count", "value_sum", "value_avg") and not value_field:
+        raise RuleError(f"{kind} correlations need a condition field")
+
+    aliases = spec.get("aliases") or {}
+    if not isinstance(aliases, dict) or not all(isinstance(v, dict) for v in aliases.values()):
+        raise RuleError("correlation 'aliases' must map names to {rule: field}")
+
+    timespan = spec.get("timespan")
+    return Correlation(
+        type=kind,
+        rules=tuple(str(r) for r in rules),
+        group_by=tuple(str(g) for g in group_by),
+        timespan=_timespan(timespan) if timespan is not None else None,
+        condition=tuple(condition),
+        field=str(value_field) if value_field else None,
+        aliases={str(k): {str(r): str(f) for r, f in v.items()} for k, v in aliases.items()},
+        generate=bool(spec.get("generate", False)),
+        **_meta(doc, path),
+    )
+
+
+def _timespan(value: Any) -> int:
+    m = TIMESPAN.match(str(value))
+    if not m:
+        raise RuleError(f"invalid timespan {value!r}; use e.g. 30s, 5m, 1h, 1d")
+    return int(m[1]) * UNIT[m[2]]

sigma2sql/runner.py

@@ -0,0 +1,82 @@
+"""Run rules against sample events in an in-memory SQLite database."""
+
+from __future__ import annotations
+
+import ipaddress
+import json
+import re
+import sqlite3
+from contextlib import closing
+from dataclasses import replace
+from typing import Any
+
+from sigma2sql.rule import Rule
+from sigma2sql.sql import Target, to_query
+
+TABLE = "events"
+COLUMN = "data"
+
+
+def connect() -> sqlite3.Connection:
+    """SQLite connection with the REGEXP and cidr_match functions the sqlite dialect uses."""
+    conn = sqlite3.connect(":memory:")
+    conn.create_function("regexp", 2, _regexp, deterministic=True)
+    conn.create_function("cidr_match", 2, _cidr_match, deterministic=True)
+    return conn
+
+
+def target(base: Target | None = None) -> Target:
+    """`base` (mapping, timestamp settings) pointed at the runner's events table."""
+    base = base or Target()
+    routes = tuple(replace(r, table=None) for r in base.routes)
+    return replace(base, dialect="sqlite", table=TABLE, json_column=COLUMN, routes=routes)
+
+
+def load(events: list[dict[str, Any]]) -> sqlite3.Connection:
+    conn = connect()
+    conn.execute(f"CREATE TABLE {TABLE} ({COLUMN} TEXT NOT NULL)")
+    conn.executemany(
+        f"INSERT INTO {TABLE} (rowid, {COLUMN}) VALUES (?, ?)",
+        [(i + 1, json.dumps(e)) for i, e in enumerate(events)],
+    )
+    return conn
+
+
+def matching_events(
+    rule: Rule, events: list[dict[str, Any]], base: Target | None = None
+) -> list[int]:
+    """Return the indexes of the events the rule matches."""
+    query = to_query(rule, target(base), columns="rowid")
+    with closing(load(events)) as conn:
+        return [row[0] - 1 for row in conn.execute(query + " ORDER BY rowid")]
+
+
+def rows(query: str, events: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Run a query (e.g. a correlation) over the events and return named rows."""
+    with closing(load(events)) as conn:
+        cur = conn.execute(query)
+        names = [d[0] for d in cur.description]
+        return [dict(zip(names, row, strict=True)) for row in cur.fetchall()]
+
+
+def read_jsonl(text: str) -> list[dict[str, Any]]:
+    events: list[dict[str, Any]] = []
+    for n, line in enumerate(text.splitlines(), 1):
+        if not line.strip():
+            continue
+        event = json.loads(line)
+        if not isinstance(event, dict):
+            raise ValueError(f"line {n}: each event must be a JSON object")
+        events.append(event)
+    return events
+
+
+def _regexp(pattern: str, value: Any) -> bool:
+    return value is not None and re.search(pattern, str(value)) is not None
+
+
+def _cidr_match(value: Any, network: str) -> bool:
+    try:
+        return ipaddress.ip_address(str(value)) in ipaddress.ip_network(network)
+    except ValueError:
+        return False