sigma2sql 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sigma2sql/__init__.py +21 -0
- sigma2sql/__main__.py +3 -0
- sigma2sql/cli.py +320 -0
- sigma2sql/collection.py +62 -0
- sigma2sql/condition.py +119 -0
- sigma2sql/correlation.py +82 -0
- sigma2sql/detection.py +230 -0
- sigma2sql/errors.py +6 -0
- sigma2sql/expr.py +140 -0
- sigma2sql/hunt.py +116 -0
- sigma2sql/profile.py +119 -0
- sigma2sql/profiles/__init__.py +0 -0
- sigma2sql/profiles/ecs-winlogbeat.yml +43 -0
- sigma2sql/profiles/evtx.yml +71 -0
- sigma2sql/profiles/ocsf.yml +26 -0
- sigma2sql/profiles/sysmon.yml +26 -0
- sigma2sql/py.typed +0 -0
- sigma2sql/rule.py +260 -0
- sigma2sql/runner.py +82 -0
- sigma2sql/sql.py +325 -0
- sigma2sql-0.2.0.dist-info/METADATA +277 -0
- sigma2sql-0.2.0.dist-info/RECORD +25 -0
- sigma2sql-0.2.0.dist-info/WHEEL +4 -0
- sigma2sql-0.2.0.dist-info/entry_points.txt +2 -0
- sigma2sql-0.2.0.dist-info/licenses/LICENSE +21 -0
sigma2sql/detection.py
ADDED
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
"""Compile Sigma search identifiers (the maps and lists under `detection`) into Expr trees."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import base64
|
|
6
|
+
import ipaddress
|
|
7
|
+
import itertools
|
|
8
|
+
import re
|
|
9
|
+
from typing import Any, Literal
|
|
10
|
+
|
|
11
|
+
from sigma2sql.errors import RuleError, UnsupportedRule
|
|
12
|
+
from sigma2sql.expr import (
|
|
13
|
+
Cidr,
|
|
14
|
+
Compare,
|
|
15
|
+
Exists,
|
|
16
|
+
Expr,
|
|
17
|
+
FieldRef,
|
|
18
|
+
NullMatch,
|
|
19
|
+
NumMatch,
|
|
20
|
+
Regex,
|
|
21
|
+
SigmaString,
|
|
22
|
+
StrMatch,
|
|
23
|
+
all_of,
|
|
24
|
+
any_of,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
POSITION = {"contains", "startswith", "endswith"}
|
|
28
|
+
COMPARE: dict[str, Literal["<", "<=", ">", ">="]] = {"lt": "<", "lte": "<=", "gt": ">", "gte": ">="}
|
|
29
|
+
# Text encodings applied before base64; `utf16` adds the little-endian byte order mark.
|
|
30
|
+
ENCODINGS = {"wide": "utf-16-le", "utf16le": "utf-16-le", "utf16be": "utf-16-be", "utf16": "utf-16"}
|
|
31
|
+
BASE64 = {"base64", "base64offset"}
|
|
32
|
+
KNOWN = (
|
|
33
|
+
POSITION
|
|
34
|
+
| set(COMPARE)
|
|
35
|
+
| set(ENCODINGS)
|
|
36
|
+
| BASE64
|
|
37
|
+
| {"all", "cased", "windash", "re", "i", "cidr", "exists", "fieldref"}
|
|
38
|
+
)
|
|
39
|
+
# Modifiers that make sense on keywords (searches over the whole event).
|
|
40
|
+
KEYWORD_MODS = POSITION | set(ENCODINGS) | BASE64 | {"all", "cased", "windash", "re", "i"}
|
|
41
|
+
|
|
42
|
+
# Characters Windows command-line tools accept in place of a leading dash.
|
|
43
|
+
WINDASH = ("-", "/", "–", "—", "―")
|
|
44
|
+
# A `-` or `/` that starts a word (same rule as pySigma).
|
|
45
|
+
WINDASH_FLAG = re.compile(r"\B[-/]\b")
|
|
46
|
+
MAX_WINDASH_FLAGS = 3
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def compile_search(name: str, value: Any) -> Expr:
|
|
50
|
+
if isinstance(value, dict):
|
|
51
|
+
return _compile_map(name, value)
|
|
52
|
+
if isinstance(value, list) and value:
|
|
53
|
+
if all(isinstance(v, dict) for v in value):
|
|
54
|
+
return any_of([_compile_map(name, v) for v in value])
|
|
55
|
+
if all(_is_scalar(v) for v in value):
|
|
56
|
+
return any_of([_keyword(name, v) for v in value])
|
|
57
|
+
if _is_scalar(value) and value is not None:
|
|
58
|
+
return _keyword(name, value)
|
|
59
|
+
raise RuleError(f"search {name!r} must be a map, a list of maps or a list of keywords")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _is_scalar(value: Any) -> bool:
|
|
63
|
+
return value is None or isinstance(value, str | int | float | bool)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _keyword(name: str, value: Any) -> Expr:
|
|
67
|
+
if value is None:
|
|
68
|
+
raise RuleError(f"search {name!r} has a null keyword")
|
|
69
|
+
# Keywords match anywhere in the event, so they behave like `contains`.
|
|
70
|
+
return StrMatch(None, SigmaString.parse(_as_text(value)).wrap(before=True, after=True))
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _compile_map(name: str, mapping: dict[Any, Any]) -> Expr:
|
|
74
|
+
if not mapping:
|
|
75
|
+
raise RuleError(f"search {name!r} is an empty map")
|
|
76
|
+
return all_of([_compile_field(str(key), value) for key, value in mapping.items()])
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _compile_field(key: str, value: Any) -> Expr:
|
|
80
|
+
name, *mods = key.split("|")
|
|
81
|
+
for mod in mods:
|
|
82
|
+
if mod not in KNOWN:
|
|
83
|
+
raise UnsupportedRule(f"modifier {mod!r} is not supported")
|
|
84
|
+
# `'|all': [a, b]` is a keyword search: no field, matched anywhere in the event.
|
|
85
|
+
field: str | None = name or None
|
|
86
|
+
label = name or "keywords"
|
|
87
|
+
if field is None:
|
|
88
|
+
extra = [m for m in mods if m not in KEYWORD_MODS]
|
|
89
|
+
if extra:
|
|
90
|
+
raise UnsupportedRule(f"modifier {extra[0]!r} is not supported on keywords")
|
|
91
|
+
|
|
92
|
+
values = value if isinstance(value, list) else [value]
|
|
93
|
+
if not values:
|
|
94
|
+
raise RuleError(f"field {label!r} has an empty value list")
|
|
95
|
+
for v in values:
|
|
96
|
+
if not _is_scalar(v):
|
|
97
|
+
raise RuleError(f"field {label!r} has a nested value")
|
|
98
|
+
|
|
99
|
+
items: list[Expr]
|
|
100
|
+
if field is None and "re" in mods:
|
|
101
|
+
items = [Regex(None, _as_text(v), ignore_case="i" in mods) for v in values]
|
|
102
|
+
elif field is None:
|
|
103
|
+
items = [
|
|
104
|
+
_strings(None, v, mods, "contains", "cased" in mods) for v in values if v is not None
|
|
105
|
+
]
|
|
106
|
+
if len(items) != len(values):
|
|
107
|
+
raise RuleError("keywords cannot be null")
|
|
108
|
+
elif "exists" in mods:
|
|
109
|
+
if len(values) != 1 or not isinstance(values[0], bool):
|
|
110
|
+
raise RuleError(f"{field}|exists expects true or false")
|
|
111
|
+
return Exists(field, values[0])
|
|
112
|
+
elif "fieldref" in mods:
|
|
113
|
+
items = [FieldRef(field, _as_text(v)) for v in values]
|
|
114
|
+
elif "re" in mods:
|
|
115
|
+
items = [Regex(field, _as_text(v), ignore_case="i" in mods) for v in values]
|
|
116
|
+
elif "cidr" in mods:
|
|
117
|
+
items = [Cidr(field, _network(field, v)) for v in values]
|
|
118
|
+
elif compare := [m for m in mods if m in COMPARE]:
|
|
119
|
+
if len(compare) > 1:
|
|
120
|
+
raise RuleError(f"field {field!r} has more than one comparison modifier")
|
|
121
|
+
items = [Compare(field, COMPARE[compare[0]], _number(field, v)) for v in values]
|
|
122
|
+
else:
|
|
123
|
+
position = [m for m in mods if m in POSITION]
|
|
124
|
+
if len(position) > 1:
|
|
125
|
+
raise RuleError(f"field {field!r} has more than one of contains/startswith/endswith")
|
|
126
|
+
items = [
|
|
127
|
+
_value(field, v, mods, position[0] if position else None, "cased" in mods)
|
|
128
|
+
for v in values
|
|
129
|
+
]
|
|
130
|
+
|
|
131
|
+
return all_of(items) if "all" in mods else any_of(items)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _value(field: str, value: Any, mods: list[str], position: str | None, cased: bool) -> Expr:
|
|
135
|
+
if value is None:
|
|
136
|
+
return NullMatch(field)
|
|
137
|
+
is_number = isinstance(value, int | float) and not isinstance(value, bool)
|
|
138
|
+
if is_number and not position and not (set(mods) & (BASE64 | {"windash"})):
|
|
139
|
+
return NumMatch(field, value)
|
|
140
|
+
return _strings(field, value, mods, position, cased)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _strings(
|
|
144
|
+
field: str | None, value: Any, mods: list[str], position: str | None, cased: bool
|
|
145
|
+
) -> Expr:
|
|
146
|
+
"""Match a string value after windash/encoding/base64 transformations."""
|
|
147
|
+
raw = _as_text(value)
|
|
148
|
+
encoding = [m for m in mods if m in ENCODINGS]
|
|
149
|
+
b64 = [m for m in mods if m in BASE64]
|
|
150
|
+
if len(encoding) > 1 or len(b64) > 1:
|
|
151
|
+
raise RuleError(f"{field or 'keywords'} has conflicting encoding modifiers")
|
|
152
|
+
if encoding and not b64:
|
|
153
|
+
raise UnsupportedRule(f"{encoding[0]} is only supported together with base64")
|
|
154
|
+
|
|
155
|
+
patterns: list[SigmaString]
|
|
156
|
+
if b64:
|
|
157
|
+
plain = SigmaString.parse(raw)
|
|
158
|
+
if plain.has_wildcards:
|
|
159
|
+
raise UnsupportedRule("base64 modifiers cannot encode wildcards")
|
|
160
|
+
data = _encode(plain.text, ENCODINGS[encoding[0]] if encoding else None)
|
|
161
|
+
encoded = [base64.b64encode(data).decode()] if b64[0] == "base64" else _offsets(data)
|
|
162
|
+
# Encoded text is literal and case-sensitive.
|
|
163
|
+
patterns = [SigmaString((e,)) for e in encoded]
|
|
164
|
+
cased = True
|
|
165
|
+
else:
|
|
166
|
+
variants = _windash(raw) if "windash" in mods else [raw]
|
|
167
|
+
patterns = [SigmaString.parse(v) for v in variants]
|
|
168
|
+
|
|
169
|
+
before = position in ("contains", "endswith")
|
|
170
|
+
after = position in ("contains", "startswith")
|
|
171
|
+
return any_of([StrMatch(field, p.wrap(before=before, after=after), cased) for p in patterns])
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _encode(text: str, encoding: str | None) -> bytes:
|
|
175
|
+
if encoding == "utf-16":
|
|
176
|
+
return b"\xff\xfe" + text.encode("utf-16-le")
|
|
177
|
+
return text.encode(encoding or "utf-8")
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _offsets(data: bytes) -> list[str]:
|
|
181
|
+
"""The three base64 forms of `data` at byte offsets 0, 1 and 2 inside a longer blob,
|
|
182
|
+
trimmed to the characters that do not depend on the surrounding bytes."""
|
|
183
|
+
start = (0, 2, 3)
|
|
184
|
+
end = (None, -3, -2)
|
|
185
|
+
out = []
|
|
186
|
+
for i in range(3):
|
|
187
|
+
encoded = base64.b64encode(i * b" " + data).decode()
|
|
188
|
+
out.append(encoded[start[i] : end[(len(data) + i) % 3]])
|
|
189
|
+
return out
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _windash(raw: str) -> list[str]:
|
|
193
|
+
"""Every way of writing the value's command-line flags with each WINDASH character.
|
|
194
|
+
|
|
195
|
+
A flag is a `-` or `/` that starts a word, so ` /accepteula` also matches ` -accepteula`.
|
|
196
|
+
"""
|
|
197
|
+
positions = [m.start() for m in WINDASH_FLAG.finditer(raw)]
|
|
198
|
+
if len(positions) > MAX_WINDASH_FLAGS:
|
|
199
|
+
# Too many combinations: use one dash style for all flags at once.
|
|
200
|
+
combos: list[tuple[str, ...]] = [(dash,) * len(positions) for dash in WINDASH]
|
|
201
|
+
else:
|
|
202
|
+
combos = list(itertools.product(WINDASH, repeat=len(positions)))
|
|
203
|
+
out: list[str] = []
|
|
204
|
+
for combo in combos:
|
|
205
|
+
chars = list(raw)
|
|
206
|
+
for pos, dash in zip(positions, combo, strict=True):
|
|
207
|
+
chars[pos] = dash
|
|
208
|
+
variant = "".join(chars)
|
|
209
|
+
if variant not in out:
|
|
210
|
+
out.append(variant)
|
|
211
|
+
return out
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _as_text(value: Any) -> str:
|
|
215
|
+
if isinstance(value, bool):
|
|
216
|
+
return "true" if value else "false"
|
|
217
|
+
return str(value)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _number(field: str, value: Any) -> int | float:
|
|
221
|
+
if isinstance(value, int | float) and not isinstance(value, bool):
|
|
222
|
+
return value
|
|
223
|
+
raise RuleError(f"field {field!r} needs a number for comparison, got {value!r}")
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _network(field: str, value: Any) -> str:
|
|
227
|
+
try:
|
|
228
|
+
return str(ipaddress.ip_network(_as_text(value), strict=False))
|
|
229
|
+
except ValueError as exc:
|
|
230
|
+
raise RuleError(f"field {field!r} has an invalid CIDR {value!r}") from exc
|
sigma2sql/errors.py
ADDED
sigma2sql/expr.py
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""Dialect-independent match tree that detections compile into."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from enum import Enum
|
|
7
|
+
from typing import Literal
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Wildcard(Enum):
|
|
11
|
+
MANY = "*"
|
|
12
|
+
ONE = "?"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
Part = str | Wildcard
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True)
|
|
19
|
+
class SigmaString:
|
|
20
|
+
parts: tuple[Part, ...]
|
|
21
|
+
|
|
22
|
+
@classmethod
|
|
23
|
+
def parse(cls, raw: str) -> SigmaString:
|
|
24
|
+
"""Split a Sigma value into literals and wildcards.
|
|
25
|
+
|
|
26
|
+
`\\*`, `\\?` and `\\\\` are escapes; any other backslash is literal.
|
|
27
|
+
"""
|
|
28
|
+
parts: list[Part] = []
|
|
29
|
+
buf: list[str] = []
|
|
30
|
+
i = 0
|
|
31
|
+
while i < len(raw):
|
|
32
|
+
ch = raw[i]
|
|
33
|
+
if ch == "\\" and i + 1 < len(raw) and raw[i + 1] in "*?\\":
|
|
34
|
+
buf.append(raw[i + 1])
|
|
35
|
+
i += 2
|
|
36
|
+
continue
|
|
37
|
+
if ch in "*?":
|
|
38
|
+
if buf:
|
|
39
|
+
parts.append("".join(buf))
|
|
40
|
+
buf = []
|
|
41
|
+
parts.append(Wildcard(ch))
|
|
42
|
+
else:
|
|
43
|
+
buf.append(ch)
|
|
44
|
+
i += 1
|
|
45
|
+
if buf:
|
|
46
|
+
parts.append("".join(buf))
|
|
47
|
+
return cls(tuple(parts))
|
|
48
|
+
|
|
49
|
+
def wrap(self, *, before: bool, after: bool) -> SigmaString:
|
|
50
|
+
parts = list(self.parts)
|
|
51
|
+
if before and parts[:1] != [Wildcard.MANY]:
|
|
52
|
+
parts.insert(0, Wildcard.MANY)
|
|
53
|
+
if after and parts[-1:] != [Wildcard.MANY]:
|
|
54
|
+
parts.append(Wildcard.MANY)
|
|
55
|
+
return SigmaString(tuple(parts))
|
|
56
|
+
|
|
57
|
+
@property
|
|
58
|
+
def has_wildcards(self) -> bool:
|
|
59
|
+
return any(isinstance(p, Wildcard) for p in self.parts)
|
|
60
|
+
|
|
61
|
+
@property
|
|
62
|
+
def text(self) -> str:
|
|
63
|
+
"""Literal text; only meaningful when there are no wildcards."""
|
|
64
|
+
return "".join(p for p in self.parts if isinstance(p, str))
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@dataclass(frozen=True)
|
|
68
|
+
class StrMatch:
|
|
69
|
+
field: str | None # None means a keyword search over the whole event
|
|
70
|
+
value: SigmaString
|
|
71
|
+
cased: bool = False
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@dataclass(frozen=True)
|
|
75
|
+
class NumMatch:
|
|
76
|
+
field: str
|
|
77
|
+
value: int | float
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@dataclass(frozen=True)
|
|
81
|
+
class NullMatch:
|
|
82
|
+
field: str
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@dataclass(frozen=True)
|
|
86
|
+
class Exists:
|
|
87
|
+
field: str
|
|
88
|
+
exists: bool
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@dataclass(frozen=True)
|
|
92
|
+
class Regex:
|
|
93
|
+
field: str | None # None means the whole event
|
|
94
|
+
pattern: str
|
|
95
|
+
ignore_case: bool = False
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@dataclass(frozen=True)
|
|
99
|
+
class Cidr:
|
|
100
|
+
field: str
|
|
101
|
+
network: str
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
@dataclass(frozen=True)
|
|
105
|
+
class Compare:
|
|
106
|
+
field: str
|
|
107
|
+
op: Literal["<", "<=", ">", ">="]
|
|
108
|
+
value: int | float
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
@dataclass(frozen=True)
|
|
112
|
+
class FieldRef:
|
|
113
|
+
field: str
|
|
114
|
+
other: str
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
@dataclass(frozen=True)
|
|
118
|
+
class And:
|
|
119
|
+
items: tuple[Expr, ...]
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@dataclass(frozen=True)
|
|
123
|
+
class Or:
|
|
124
|
+
items: tuple[Expr, ...]
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
@dataclass(frozen=True)
|
|
128
|
+
class Not:
|
|
129
|
+
item: Expr
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
Expr = StrMatch | NumMatch | NullMatch | Exists | Regex | Cidr | Compare | FieldRef | And | Or | Not
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def all_of(items: list[Expr]) -> Expr:
|
|
136
|
+
return items[0] if len(items) == 1 else And(tuple(items))
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def any_of(items: list[Expr]) -> Expr:
|
|
140
|
+
return items[0] if len(items) == 1 else Or(tuple(items))
|
sigma2sql/hunt.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""Run many rules over log files with DuckDB, no database server needed."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import ipaddress
|
|
6
|
+
from dataclasses import dataclass, field, replace
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from sigma2sql.collection import Collection
|
|
11
|
+
from sigma2sql.correlation import correlation_query
|
|
12
|
+
from sigma2sql.errors import RuleError, UnsupportedRule
|
|
13
|
+
from sigma2sql.rule import Correlation, Document
|
|
14
|
+
from sigma2sql.sql import Target, literal, to_query
|
|
15
|
+
|
|
16
|
+
LEVELS = ("critical", "high", "medium", "low", "informational")
|
|
17
|
+
TABLE = "events"
|
|
18
|
+
COLUMN = "data"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass
|
|
22
|
+
class Hit:
|
|
23
|
+
rule: Document
|
|
24
|
+
count: int
|
|
25
|
+
examples: list[str] = field(default_factory=list)
|
|
26
|
+
|
|
27
|
+
@property
|
|
28
|
+
def rank(self) -> int:
|
|
29
|
+
return LEVELS.index(self.rule.level) if self.rule.level in LEVELS else len(LEVELS)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class Report:
|
|
34
|
+
events: int
|
|
35
|
+
hits: list[Hit]
|
|
36
|
+
ran: int
|
|
37
|
+
skipped: list[tuple[Document, str]]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def connect() -> Any:
|
|
41
|
+
try:
|
|
42
|
+
import duckdb
|
|
43
|
+
except ImportError as exc: # pragma: no cover - depends on the install
|
|
44
|
+
raise SystemExit("sigma2sql hunt needs DuckDB: pip install 'sigma2sql[duckdb]'") from exc
|
|
45
|
+
con = duckdb.connect()
|
|
46
|
+
con.create_function("cidr_match", _cidr_match, ["VARCHAR", "VARCHAR"], "BOOLEAN")
|
|
47
|
+
con.execute(f"CREATE TABLE {TABLE} ({COLUMN} JSON)")
|
|
48
|
+
return con
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def load_logs(con: Any, paths: list[str]) -> int:
|
|
52
|
+
"""Load JSON/JSON Lines, Parquet or CSV files (globs allowed) as one JSON object per row."""
|
|
53
|
+
for path in paths:
|
|
54
|
+
source = literal(path)
|
|
55
|
+
suffix = Path(path).suffix.lower()
|
|
56
|
+
if suffix == ".parquet":
|
|
57
|
+
select = f"SELECT to_json(t) FROM read_parquet({source}) t"
|
|
58
|
+
elif suffix in (".csv", ".tsv"):
|
|
59
|
+
select = f"SELECT to_json(t) FROM read_csv_auto({source}) t"
|
|
60
|
+
elif suffix in (".json", ".jsonl", ".ndjson"):
|
|
61
|
+
select = f"SELECT json FROM read_json_objects({source}, format = 'auto')"
|
|
62
|
+
else:
|
|
63
|
+
raise ValueError(f"{path}: expected .json, .jsonl, .ndjson, .parquet or .csv")
|
|
64
|
+
con.execute(f"INSERT INTO {TABLE} {select}")
|
|
65
|
+
count: int = con.execute(f"SELECT COUNT(*) FROM {TABLE}").fetchone()[0]
|
|
66
|
+
return count
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def hunt(
|
|
70
|
+
rules: Collection, paths: list[str], base: Target | None = None, examples: int = 0
|
|
71
|
+
) -> Report:
|
|
72
|
+
"""Run every rule and correlation in `rules` over the logs in `paths`.
|
|
73
|
+
|
|
74
|
+
`base` carries the field mapping and timestamp settings; table and dialect are set here.
|
|
75
|
+
"""
|
|
76
|
+
con = connect()
|
|
77
|
+
events = load_logs(con, paths)
|
|
78
|
+
base = base or Target()
|
|
79
|
+
# All logs share one table here, so profile routes keep only their conditions.
|
|
80
|
+
routes = tuple(replace(r, table=None) for r in base.routes)
|
|
81
|
+
target = replace(base, dialect="duckdb", table=TABLE, json_column=COLUMN, routes=routes)
|
|
82
|
+
items: list[Document] = [*rules.standalone(), *rules.correlations]
|
|
83
|
+
hits: list[Hit] = []
|
|
84
|
+
skipped: list[tuple[Document, str]] = []
|
|
85
|
+
for item in items:
|
|
86
|
+
try:
|
|
87
|
+
query, as_rows = _query(item, rules, target)
|
|
88
|
+
except (RuleError, UnsupportedRule) as exc:
|
|
89
|
+
skipped.append((item, str(exc)))
|
|
90
|
+
continue
|
|
91
|
+
count = con.execute(f"SELECT COUNT(*) FROM ({query}) q").fetchone()[0]
|
|
92
|
+
if not count:
|
|
93
|
+
continue
|
|
94
|
+
sample: list[str] = []
|
|
95
|
+
if examples:
|
|
96
|
+
shown = f"SELECT CAST(to_json(q) AS VARCHAR) FROM ({query}) q" if as_rows else query
|
|
97
|
+
sample = [row[0] for row in con.execute(f"{shown} LIMIT {examples}").fetchall()]
|
|
98
|
+
hits.append(Hit(item, count, sample))
|
|
99
|
+
hits.sort(key=lambda h: (h.rank, -h.count, h.rule.title))
|
|
100
|
+
return Report(events, hits, len(items) - len(skipped), skipped)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _query(item: Document, rules: Collection, target: Target) -> tuple[str, bool]:
|
|
104
|
+
"""The SQL for one item and whether it returns result rows rather than events."""
|
|
105
|
+
if isinstance(item, Correlation):
|
|
106
|
+
return correlation_query(item, rules.resolve(item), target), True
|
|
107
|
+
if item.aggregation is not None:
|
|
108
|
+
return to_query(item, target), True
|
|
109
|
+
return to_query(item, target, columns=f"CAST({COLUMN} AS VARCHAR)"), False
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _cidr_match(value: str, network: str) -> bool:
|
|
113
|
+
try:
|
|
114
|
+
return ipaddress.ip_address(value) in ipaddress.ip_network(network)
|
|
115
|
+
except ValueError:
|
|
116
|
+
return False
|
sigma2sql/profile.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""Profiles: reusable field mappings, logsource routing and timestamp settings.
|
|
2
|
+
|
|
3
|
+
A profile is a YAML file:
|
|
4
|
+
|
|
5
|
+
description: one line shown by `sigma2sql profiles`
|
|
6
|
+
json_column: data # optional
|
|
7
|
+
nested: true # dotted names are paths into nested JSON
|
|
8
|
+
timestamp: "@timestamp"
|
|
9
|
+
timestamp_format: iso # iso, epoch_s or epoch_ms
|
|
10
|
+
default_field: "winlog.event_data.{field}"
|
|
11
|
+
fields:
|
|
12
|
+
Image: process.executable
|
|
13
|
+
EventID: [Event.System.EventID.#text, Event.System.EventID] # first present wins
|
|
14
|
+
logsources:
|
|
15
|
+
- match: {product: windows, category: process_creation}
|
|
16
|
+
table: process_events # optional
|
|
17
|
+
where: {event.code: "1"} # optional, in target field names
|
|
18
|
+
|
|
19
|
+
Built-in profiles ship in `sigma2sql/profiles/`; pass a name or a path.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
from dataclasses import replace
|
|
25
|
+
from importlib import resources
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
from typing import Any
|
|
28
|
+
|
|
29
|
+
import yaml
|
|
30
|
+
|
|
31
|
+
from sigma2sql.sql import Route, Target
|
|
32
|
+
|
|
33
|
+
KEYS = {
|
|
34
|
+
"description",
|
|
35
|
+
"json_column",
|
|
36
|
+
"nested",
|
|
37
|
+
"timestamp",
|
|
38
|
+
"timestamp_format",
|
|
39
|
+
"default_field",
|
|
40
|
+
"fields",
|
|
41
|
+
"logsources",
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def builtin_profiles() -> dict[str, str]:
|
|
46
|
+
"""Name -> description of the profiles that ship with sigma2sql."""
|
|
47
|
+
out: dict[str, str] = {}
|
|
48
|
+
for entry in sorted(resources.files("sigma2sql.profiles").iterdir(), key=lambda e: e.name):
|
|
49
|
+
if entry.name.endswith(".yml"):
|
|
50
|
+
data = yaml.safe_load(entry.read_text(encoding="utf-8")) or {}
|
|
51
|
+
out[entry.name.removesuffix(".yml")] = str(data.get("description", ""))
|
|
52
|
+
return out
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def load_profile(name_or_path: str) -> dict[str, Any]:
|
|
56
|
+
path = Path(name_or_path)
|
|
57
|
+
if path.suffix in (".yml", ".yaml") or path.exists():
|
|
58
|
+
text = path.read_text(encoding="utf-8")
|
|
59
|
+
else:
|
|
60
|
+
entry = resources.files("sigma2sql.profiles").joinpath(f"{name_or_path}.yml")
|
|
61
|
+
if not entry.is_file():
|
|
62
|
+
known = ", ".join(builtin_profiles())
|
|
63
|
+
raise ValueError(f"unknown profile {name_or_path!r} (built in: {known})")
|
|
64
|
+
text = entry.read_text(encoding="utf-8")
|
|
65
|
+
data = yaml.safe_load(text) or {}
|
|
66
|
+
if not isinstance(data, dict):
|
|
67
|
+
raise ValueError("a profile must be a YAML map")
|
|
68
|
+
unknown = set(data) - KEYS
|
|
69
|
+
if unknown:
|
|
70
|
+
raise ValueError(f"unknown profile keys: {', '.join(sorted(unknown))}")
|
|
71
|
+
return data
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def apply_profile(target: Target, profile: dict[str, Any]) -> Target:
|
|
75
|
+
"""Target with the profile's settings; fields already in `target.mapping` win."""
|
|
76
|
+
fields = profile.get("fields") or {}
|
|
77
|
+
if not isinstance(fields, dict):
|
|
78
|
+
raise ValueError("profile 'fields' must be a map")
|
|
79
|
+
routes: list[Route] = []
|
|
80
|
+
for item in profile.get("logsources") or []:
|
|
81
|
+
if not isinstance(item, dict) or not isinstance(item.get("match"), dict):
|
|
82
|
+
raise ValueError("each profile logsource needs a 'match' map")
|
|
83
|
+
where = item.get("where")
|
|
84
|
+
if where is not None and not isinstance(where, dict):
|
|
85
|
+
raise ValueError("profile logsource 'where' must be a map")
|
|
86
|
+
routes.append(
|
|
87
|
+
Route(
|
|
88
|
+
match={str(k): str(v) for k, v in item["match"].items()},
|
|
89
|
+
table=item.get("table"),
|
|
90
|
+
where=where,
|
|
91
|
+
)
|
|
92
|
+
)
|
|
93
|
+
template = profile.get("default_field")
|
|
94
|
+
templates = None
|
|
95
|
+
if template is not None:
|
|
96
|
+
templates = tuple(template) if isinstance(template, list) else (str(template),)
|
|
97
|
+
if not all("{field" in t for t in templates):
|
|
98
|
+
raise ValueError("profile 'default_field' must contain {field} or {field_words}")
|
|
99
|
+
fmt = profile.get("timestamp_format", target.timestamp_format)
|
|
100
|
+
if fmt not in ("iso", "epoch_s", "epoch_ms"):
|
|
101
|
+
raise ValueError(f"invalid timestamp_format {fmt!r}")
|
|
102
|
+
return replace(
|
|
103
|
+
target,
|
|
104
|
+
mapping={**{str(k): _target_field(v) for k, v in fields.items()}, **target.mapping},
|
|
105
|
+
json_column=profile.get("json_column", target.json_column),
|
|
106
|
+
nested=bool(profile.get("nested", target.nested)),
|
|
107
|
+
timestamp=str(profile.get("timestamp", target.timestamp)),
|
|
108
|
+
timestamp_format=fmt,
|
|
109
|
+
default_field=templates if templates is not None else target.default_field,
|
|
110
|
+
routes=(*target.routes, *routes),
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _target_field(value: Any) -> str | tuple[str, ...]:
|
|
115
|
+
if isinstance(value, list):
|
|
116
|
+
if not value or not all(isinstance(v, str) for v in value):
|
|
117
|
+
raise ValueError("a profile field list must hold field names")
|
|
118
|
+
return tuple(value)
|
|
119
|
+
return str(value)
|
|
File without changes
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
description: Windows events shipped by Winlogbeat in Elastic Common Schema (nested JSON); unmapped fields fall back to winlog.event_data
|
|
2
|
+
nested: true
|
|
3
|
+
timestamp: "@timestamp"
|
|
4
|
+
default_field: "winlog.event_data.{field}"
|
|
5
|
+
fields:
|
|
6
|
+
EventID: event.code
|
|
7
|
+
Channel: winlog.channel
|
|
8
|
+
Provider_Name: winlog.provider_name
|
|
9
|
+
Computer: host.name
|
|
10
|
+
Image: process.executable
|
|
11
|
+
CommandLine: process.command_line
|
|
12
|
+
ProcessId: process.pid
|
|
13
|
+
CurrentDirectory: process.working_directory
|
|
14
|
+
OriginalFileName: process.pe.original_file_name
|
|
15
|
+
Product: process.pe.product
|
|
16
|
+
Company: process.pe.company
|
|
17
|
+
Description: process.pe.description
|
|
18
|
+
ParentImage: process.parent.executable
|
|
19
|
+
ParentCommandLine: process.parent.command_line
|
|
20
|
+
ParentProcessId: process.parent.pid
|
|
21
|
+
User: user.name
|
|
22
|
+
TargetFilename: file.path
|
|
23
|
+
ImageLoaded: file.path
|
|
24
|
+
DestinationIp: destination.ip
|
|
25
|
+
DestinationPort: destination.port
|
|
26
|
+
DestinationHostname: destination.domain
|
|
27
|
+
SourceIp: source.ip
|
|
28
|
+
SourcePort: source.port
|
|
29
|
+
QueryName: dns.question.name
|
|
30
|
+
TargetObject: registry.path
|
|
31
|
+
logsources:
|
|
32
|
+
# Sysmon via the Winlogbeat sysmon module: event.code holds the event ID as a string.
|
|
33
|
+
- {match: {product: windows, category: process_creation}, where: {winlog.channel: Microsoft-Windows-Sysmon/Operational, event.code: "1"}}
|
|
34
|
+
- {match: {product: windows, category: network_connection}, where: {winlog.channel: Microsoft-Windows-Sysmon/Operational, event.code: "3"}}
|
|
35
|
+
- {match: {product: windows, category: image_load}, where: {winlog.channel: Microsoft-Windows-Sysmon/Operational, event.code: "7"}}
|
|
36
|
+
- {match: {product: windows, category: process_access}, where: {winlog.channel: Microsoft-Windows-Sysmon/Operational, event.code: "10"}}
|
|
37
|
+
- {match: {product: windows, category: file_event}, where: {winlog.channel: Microsoft-Windows-Sysmon/Operational, event.code: "11"}}
|
|
38
|
+
- {match: {product: windows, category: registry_set}, where: {winlog.channel: Microsoft-Windows-Sysmon/Operational, event.code: "13"}}
|
|
39
|
+
- {match: {product: windows, category: registry_event}, where: {winlog.channel: Microsoft-Windows-Sysmon/Operational, event.code: ["12", "13", "14"]}}
|
|
40
|
+
- {match: {product: windows, category: dns_query}, where: {winlog.channel: Microsoft-Windows-Sysmon/Operational, event.code: "22"}}
|
|
41
|
+
- {match: {product: windows, service: security}, where: {winlog.channel: Security}}
|
|
42
|
+
- {match: {product: windows, service: system}, where: {winlog.channel: System}}
|
|
43
|
+
- {match: {product: windows, service: powershell}, where: {winlog.channel: Microsoft-Windows-PowerShell/Operational}}
|