sigma2sql 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sigma2sql/__init__.py +21 -0
- sigma2sql/__main__.py +3 -0
- sigma2sql/cli.py +320 -0
- sigma2sql/collection.py +62 -0
- sigma2sql/condition.py +119 -0
- sigma2sql/correlation.py +82 -0
- sigma2sql/detection.py +230 -0
- sigma2sql/errors.py +6 -0
- sigma2sql/expr.py +140 -0
- sigma2sql/hunt.py +116 -0
- sigma2sql/profile.py +119 -0
- sigma2sql/profiles/__init__.py +0 -0
- sigma2sql/profiles/ecs-winlogbeat.yml +43 -0
- sigma2sql/profiles/evtx.yml +71 -0
- sigma2sql/profiles/ocsf.yml +26 -0
- sigma2sql/profiles/sysmon.yml +26 -0
- sigma2sql/py.typed +0 -0
- sigma2sql/rule.py +260 -0
- sigma2sql/runner.py +82 -0
- sigma2sql/sql.py +325 -0
- sigma2sql-0.2.0.dist-info/METADATA +277 -0
- sigma2sql-0.2.0.dist-info/RECORD +25 -0
- sigma2sql-0.2.0.dist-info/WHEEL +4 -0
- sigma2sql-0.2.0.dist-info/entry_points.txt +2 -0
- sigma2sql-0.2.0.dist-info/licenses/LICENSE +21 -0
sigma2sql/sql.py
ADDED
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
"""Render Expr trees as SQL for SQLite, PostgreSQL and DuckDB."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from collections.abc import Mapping
|
|
7
|
+
from dataclasses import dataclass, field, replace
|
|
8
|
+
from typing import Any, Literal
|
|
9
|
+
|
|
10
|
+
from sigma2sql.detection import compile_search
|
|
11
|
+
from sigma2sql.errors import UnsupportedRule
|
|
12
|
+
from sigma2sql.expr import (
|
|
13
|
+
And,
|
|
14
|
+
Cidr,
|
|
15
|
+
Compare,
|
|
16
|
+
Exists,
|
|
17
|
+
Expr,
|
|
18
|
+
FieldRef,
|
|
19
|
+
Not,
|
|
20
|
+
NullMatch,
|
|
21
|
+
NumMatch,
|
|
22
|
+
Or,
|
|
23
|
+
Regex,
|
|
24
|
+
SigmaString,
|
|
25
|
+
StrMatch,
|
|
26
|
+
Wildcard,
|
|
27
|
+
)
|
|
28
|
+
from sigma2sql.rule import Rule
|
|
29
|
+
|
|
30
|
+
Dialect = Literal["sqlite", "postgres", "duckdb"]
|
|
31
|
+
TimestampFormat = Literal["iso", "epoch_s", "epoch_ms"]
|
|
32
|
+
DIALECTS: tuple[Dialect, ...] = ("sqlite", "postgres", "duckdb")
|
|
33
|
+
|
|
34
|
+
# SQLite parses `a OR b OR ...` into a tree as deep as the list and stops at depth 1000,
|
|
35
|
+
# which rules with long hash lists exceed. Grouping keeps the depth logarithmic.
|
|
36
|
+
GROUP = 50
|
|
37
|
+
# Largest {m,n} repetition count each engine accepts (SQLite uses Python's re: no limit).
|
|
38
|
+
MAX_REPEAT: dict[Dialect, int | None] = {"sqlite": None, "postgres": 255, "duckdb": 1000}
|
|
39
|
+
REPEAT = re.compile(r"\{(\d+)(?:,(\d*))?\}")
|
|
40
|
+
# RE2 (DuckDB) has no lookaround or backreferences.
|
|
41
|
+
# Word boundaries inside CamelCase names: NewValue -> New Value.
|
|
42
|
+
CAMEL = re.compile(r"(?<=[a-z0-9])(?=[A-Z])")
|
|
43
|
+
RE2_UNSUPPORTED = re.compile(r"\(\?<?[=!]|\\[1-9]")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass(frozen=True)
|
|
47
|
+
class Route:
|
|
48
|
+
"""Logsource routing: rules whose `logsource` has every key/value in `match` read
|
|
49
|
+
from `table` (if set) and get `where` (a Sigma search map, in target field names) added.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
match: Mapping[str, str]
|
|
53
|
+
table: str | None = None
|
|
54
|
+
where: Mapping[str, Any] | None = None
|
|
55
|
+
|
|
56
|
+
def applies(self, logsource: Mapping[str, str]) -> bool:
|
|
57
|
+
return all(
|
|
58
|
+
str(logsource.get(k, "")).lower() == str(v).lower() for k, v in self.match.items()
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass(frozen=True)
|
|
63
|
+
class Target:
|
|
64
|
+
"""Where the events live and how Sigma field names reach them.
|
|
65
|
+
|
|
66
|
+
With `json_column`, every event is one JSON object in that column and fields are
|
|
67
|
+
read as top-level keys (or, with `nested`, dotted paths into nested objects).
|
|
68
|
+
Otherwise each field is its own column. A field missing from `mapping` is looked up
|
|
69
|
+
through the `default_field` templates (`{field}` is the Sigma name, `{field_words}` the
|
|
70
|
+
same name split into words, e.g. "New Value"), or under its own name.
|
|
71
|
+
"""
|
|
72
|
+
|
|
73
|
+
dialect: Dialect = "sqlite"
|
|
74
|
+
table: str = "events"
|
|
75
|
+
# Sigma field -> column/key, or a sequence of candidates tried in order.
|
|
76
|
+
mapping: Mapping[str, str | tuple[str, ...]] = field(default_factory=dict)
|
|
77
|
+
json_column: str | None = None
|
|
78
|
+
keyword_column: str | None = None
|
|
79
|
+
# Event time, used by correlations and aggregations with a timespan.
|
|
80
|
+
timestamp: str = "timestamp"
|
|
81
|
+
timestamp_format: TimestampFormat = "iso"
|
|
82
|
+
nested: bool = False
|
|
83
|
+
default_field: str | tuple[str, ...] | None = None
|
|
84
|
+
routes: tuple[Route, ...] = ()
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def to_query(rule: Rule, target: Target, columns: str = "*") -> str:
|
|
88
|
+
"""SQL for a detection rule. Rules with a legacy `| count()` aggregation return
|
|
89
|
+
one row per group (and window) that meets the threshold; `columns` is then ignored."""
|
|
90
|
+
if rule.aggregation is not None:
|
|
91
|
+
from sigma2sql.correlation import correlation_query
|
|
92
|
+
|
|
93
|
+
return correlation_query(rule.aggregation, [replace(rule, aggregation=None)], target)
|
|
94
|
+
table, where = source(rule, target)
|
|
95
|
+
return f"SELECT {columns} FROM {table} WHERE {where}"
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def source(rule: Rule, target: Target) -> tuple[str, str]:
|
|
99
|
+
"""The quoted table and WHERE clause for a rule, after logsource routing."""
|
|
100
|
+
where = to_where(rule.compile(), target)
|
|
101
|
+
route = next((r for r in target.routes if r.applies(rule.logsource)), None)
|
|
102
|
+
if route is None:
|
|
103
|
+
return quote_table(target.table), where
|
|
104
|
+
if route.where:
|
|
105
|
+
# Route conditions are written in target field names, so no mapping applies.
|
|
106
|
+
raw = replace(target, mapping={}, default_field=None)
|
|
107
|
+
extra = to_where(compile_search("logsource", dict(route.where)), raw)
|
|
108
|
+
where = f"({extra} AND {where})"
|
|
109
|
+
return quote_table(route.table or target.table), where
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def to_where(expr: Expr, target: Target) -> str:
|
|
113
|
+
return Renderer(target).render(expr)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def quote_ident(name: str) -> str:
|
|
117
|
+
return '"' + name.replace('"', '""') + '"'
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def quote_table(name: str) -> str:
|
|
121
|
+
return ".".join(quote_ident(part) for part in name.split("."))
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def literal(value: str) -> str:
|
|
125
|
+
return "'" + value.replace("'", "''") + "'"
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
class Renderer:
|
|
129
|
+
def __init__(self, target: Target) -> None:
|
|
130
|
+
self.t = target
|
|
131
|
+
self.d = target.dialect
|
|
132
|
+
self.json = target.json_column is not None
|
|
133
|
+
|
|
134
|
+
def render(self, expr: Expr) -> str:
|
|
135
|
+
match expr:
|
|
136
|
+
case And(items):
|
|
137
|
+
return _join([self.render(i) for i in items], "AND")
|
|
138
|
+
case Or(items):
|
|
139
|
+
return _join([self.render(i) for i in items], "OR")
|
|
140
|
+
case Not(item):
|
|
141
|
+
# A missing field makes the inner test NULL; Sigma treats that as "no match",
|
|
142
|
+
# so the negation has to come out true rather than NULL.
|
|
143
|
+
return f"NOT COALESCE({self.render(item)}, FALSE)"
|
|
144
|
+
case StrMatch(f, value, cased):
|
|
145
|
+
col = self.keyword() if f is None else self.column(f)
|
|
146
|
+
return self.string(col, value, cased)
|
|
147
|
+
case NumMatch(f, value):
|
|
148
|
+
col = self.column(f)
|
|
149
|
+
if not self.json:
|
|
150
|
+
return f"{col} = {value}"
|
|
151
|
+
# JSON may hold 4688 or "4688"; compare the text form so both match.
|
|
152
|
+
text = f"CAST({col} AS TEXT)" if self.d == "sqlite" else col
|
|
153
|
+
return f"{text} = {literal(str(value))}"
|
|
154
|
+
case NullMatch(f):
|
|
155
|
+
return f"{self.column(f)} IS NULL"
|
|
156
|
+
case Exists(f, exists):
|
|
157
|
+
return f"{self.column(f)} IS {'NOT ' if exists else ''}NULL"
|
|
158
|
+
case Regex(f, pattern, ignore_case):
|
|
159
|
+
col = self.keyword() if f is None else self.column(f)
|
|
160
|
+
return self.regex(col, ("(?i)" if ignore_case else "") + pattern)
|
|
161
|
+
case Cidr(f, network):
|
|
162
|
+
col = self.column(f)
|
|
163
|
+
if self.d == "postgres":
|
|
164
|
+
return f"({col})::inet <<= {literal(network)}::inet"
|
|
165
|
+
return f"cidr_match({col}, {literal(network)})"
|
|
166
|
+
case Compare(f, op, value):
|
|
167
|
+
return f"{self.number(self.column(f))} {op} {value}"
|
|
168
|
+
case FieldRef(f, other):
|
|
169
|
+
return f"{self.column(f)} = {self.column(other)}"
|
|
170
|
+
raise AssertionError(f"unhandled expression {expr!r}")
|
|
171
|
+
|
|
172
|
+
def column(self, sigma_field: str) -> str:
|
|
173
|
+
name = self.t.mapping.get(sigma_field)
|
|
174
|
+
if name is None:
|
|
175
|
+
templates = self.t.default_field
|
|
176
|
+
if isinstance(templates, str):
|
|
177
|
+
templates = (templates,)
|
|
178
|
+
name = _expand(templates, sigma_field) if templates else sigma_field
|
|
179
|
+
if not isinstance(name, str):
|
|
180
|
+
if len(name) == 1:
|
|
181
|
+
return self.path(name[0])
|
|
182
|
+
# Several candidate locations: the first one present wins.
|
|
183
|
+
return f"COALESCE({', '.join(self.path(n) for n in name)})"
|
|
184
|
+
return self.path(name)
|
|
185
|
+
|
|
186
|
+
def path(self, name: str) -> str:
|
|
187
|
+
if self.t.json_column is None:
|
|
188
|
+
return quote_ident(name)
|
|
189
|
+
col = quote_ident(self.t.json_column)
|
|
190
|
+
parts = [_dq(p) for p in (name.split(".") if self.t.nested else [name])]
|
|
191
|
+
if self.d == "postgres":
|
|
192
|
+
if len(parts) == 1:
|
|
193
|
+
return f"{col}->>{literal(name)}"
|
|
194
|
+
return f"{col}#>>{literal('{' + ','.join(parts) + '}')}"
|
|
195
|
+
path = literal("$" + "".join("." + p for p in parts))
|
|
196
|
+
if self.d == "duckdb":
|
|
197
|
+
return f"{col}->>{path}"
|
|
198
|
+
return f"json_extract({col}, {path})"
|
|
199
|
+
|
|
200
|
+
def number(self, col: str) -> str:
|
|
201
|
+
"""Numeric view of a column; JSON values arrive as text in PostgreSQL and DuckDB."""
|
|
202
|
+
if not self.json:
|
|
203
|
+
return col
|
|
204
|
+
if self.d == "postgres":
|
|
205
|
+
return f"({col})::numeric"
|
|
206
|
+
if self.d == "duckdb":
|
|
207
|
+
return f"TRY_CAST({col} AS DOUBLE)"
|
|
208
|
+
return f"CAST({col} AS REAL)"
|
|
209
|
+
|
|
210
|
+
def cast_number(self, expr: str) -> str:
|
|
211
|
+
if self.d == "postgres":
|
|
212
|
+
return f"CAST({expr} AS double precision)"
|
|
213
|
+
if self.d == "duckdb":
|
|
214
|
+
return f"TRY_CAST({expr} AS DOUBLE)"
|
|
215
|
+
return f"CAST({expr} AS REAL)"
|
|
216
|
+
|
|
217
|
+
def epoch(self) -> str:
|
|
218
|
+
"""Event time in seconds since 1970, from the configured timestamp field."""
|
|
219
|
+
col = self.column(self.t.timestamp)
|
|
220
|
+
fmt = self.t.timestamp_format
|
|
221
|
+
if fmt != "iso":
|
|
222
|
+
seconds = self.cast_number(col)
|
|
223
|
+
return seconds if fmt == "epoch_s" else f"({seconds} / 1000.0)"
|
|
224
|
+
if self.d == "postgres":
|
|
225
|
+
return f"EXTRACT(EPOCH FROM CAST({col} AS timestamptz))"
|
|
226
|
+
if self.d == "duckdb":
|
|
227
|
+
return f"epoch(TRY_CAST({col} AS TIMESTAMPTZ))"
|
|
228
|
+
return f"unixepoch({col}, 'subsec')"
|
|
229
|
+
|
|
230
|
+
def from_epoch(self, expr: str) -> str:
|
|
231
|
+
if self.d == "sqlite":
|
|
232
|
+
return f"datetime({expr}, 'unixepoch', 'subsec')"
|
|
233
|
+
if self.d == "postgres":
|
|
234
|
+
return f"to_timestamp({expr})"
|
|
235
|
+
# A plain (UTC) TIMESTAMP: TIMESTAMPTZ values need pytz on the Python side.
|
|
236
|
+
return f"make_timestamp(CAST(({expr}) * 1000000 AS BIGINT))"
|
|
237
|
+
|
|
238
|
+
def keyword(self) -> str:
|
|
239
|
+
if self.t.keyword_column:
|
|
240
|
+
return quote_ident(self.t.keyword_column)
|
|
241
|
+
if self.t.json_column:
|
|
242
|
+
col = quote_ident(self.t.json_column)
|
|
243
|
+
return col if self.d == "sqlite" else f"CAST({col} AS TEXT)"
|
|
244
|
+
raise UnsupportedRule("keyword searches need a keyword column or a JSON column")
|
|
245
|
+
|
|
246
|
+
def string(self, col: str, value: SigmaString, cased: bool) -> str:
|
|
247
|
+
if not value.has_wildcards:
|
|
248
|
+
text = value.text
|
|
249
|
+
if cased:
|
|
250
|
+
return f"{col} = {literal(text)}"
|
|
251
|
+
if self.d == "sqlite":
|
|
252
|
+
return f"{col} = {literal(text)} COLLATE NOCASE"
|
|
253
|
+
return f"LOWER({col}) = {literal(text.lower())}"
|
|
254
|
+
if self.d == "sqlite":
|
|
255
|
+
if cased:
|
|
256
|
+
return f"{col} GLOB {literal(_glob(value))}"
|
|
257
|
+
return f"{col} LIKE {literal(_like(value))} ESCAPE '\\'"
|
|
258
|
+
op = "LIKE" if cased else "ILIKE"
|
|
259
|
+
return f"{col} {op} {literal(_like(value))} ESCAPE '\\'"
|
|
260
|
+
|
|
261
|
+
def regex(self, col: str, pattern: str) -> str:
|
|
262
|
+
limit = MAX_REPEAT[self.d]
|
|
263
|
+
bounds = [int(n) for m in REPEAT.finditer(pattern) for n in m.groups() if n]
|
|
264
|
+
if limit is not None and any(n > limit for n in bounds):
|
|
265
|
+
raise UnsupportedRule(f"{self.d} regexes allow at most {limit} repetitions: {pattern}")
|
|
266
|
+
if self.d == "postgres":
|
|
267
|
+
return f"{col} ~ {literal(pattern)}"
|
|
268
|
+
if self.d == "duckdb":
|
|
269
|
+
if RE2_UNSUPPORTED.search(pattern):
|
|
270
|
+
raise UnsupportedRule(
|
|
271
|
+
f"duckdb (RE2) has no lookaround or backreferences: {pattern}"
|
|
272
|
+
)
|
|
273
|
+
return f"regexp_matches({col}, {literal(pattern)})"
|
|
274
|
+
return f"{col} REGEXP {literal(pattern)}"
|
|
275
|
+
|
|
276
|
+
def same(self, a: str, b: str) -> str:
|
|
277
|
+
"""Null-safe equality."""
|
|
278
|
+
return f"{a} IS {b}" if self.d == "sqlite" else f"{a} IS NOT DISTINCT FROM {b}"
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def _expand(templates: tuple[str, ...], sigma_field: str) -> tuple[str, ...]:
|
|
282
|
+
words = CAMEL.sub(" ", sigma_field)
|
|
283
|
+
names: list[str] = []
|
|
284
|
+
for template in templates:
|
|
285
|
+
name = template.replace("{field_words}", words).replace("{field}", sigma_field)
|
|
286
|
+
if name not in names:
|
|
287
|
+
names.append(name)
|
|
288
|
+
return tuple(names)
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def _dq(key: str) -> str:
|
|
292
|
+
"""A double-quoted JSON path key or PostgreSQL array element."""
|
|
293
|
+
return '"' + key.replace("\\", "\\\\").replace('"', '\\"') + '"'
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _join(parts: list[str], op: str) -> str:
|
|
297
|
+
while len(parts) > GROUP:
|
|
298
|
+
parts = [_join(parts[i : i + GROUP], op) for i in range(0, len(parts), GROUP)]
|
|
299
|
+
return "(" + f" {op} ".join(parts) + ")"
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def _like(value: SigmaString) -> str:
|
|
303
|
+
out: list[str] = []
|
|
304
|
+
for part in value.parts:
|
|
305
|
+
if part is Wildcard.MANY:
|
|
306
|
+
out.append("%")
|
|
307
|
+
elif part is Wildcard.ONE:
|
|
308
|
+
out.append("_")
|
|
309
|
+
else:
|
|
310
|
+
assert isinstance(part, str)
|
|
311
|
+
out.append(part.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_"))
|
|
312
|
+
return "".join(out)
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _glob(value: SigmaString) -> str:
|
|
316
|
+
out: list[str] = []
|
|
317
|
+
for part in value.parts:
|
|
318
|
+
if part is Wildcard.MANY:
|
|
319
|
+
out.append("*")
|
|
320
|
+
elif part is Wildcard.ONE:
|
|
321
|
+
out.append("?")
|
|
322
|
+
else:
|
|
323
|
+
assert isinstance(part, str)
|
|
324
|
+
out.append("".join(f"[{c}]" if c in "*?[" else c for c in part))
|
|
325
|
+
return "".join(out)
|
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: sigma2sql
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Sigma rules as SQL for SQLite, PostgreSQL and DuckDB: convert, hunt through log files and test rules
|
|
5
|
+
Project-URL: Homepage, https://github.com/brkozer/sigma2sql
|
|
6
|
+
Project-URL: Issues, https://github.com/brkozer/sigma2sql/issues
|
|
7
|
+
Project-URL: Changelog, https://github.com/brkozer/sigma2sql/blob/main/CHANGELOG.md
|
|
8
|
+
Author: Ahmet Burak Özer
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: detection-engineering,duckdb,postgresql,siem,sigma,sql,threat-hunting
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: Information Technology
|
|
15
|
+
Classifier: Intended Audience :: System Administrators
|
|
16
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Database
|
|
22
|
+
Classifier: Topic :: Security
|
|
23
|
+
Classifier: Typing :: Typed
|
|
24
|
+
Requires-Python: >=3.12
|
|
25
|
+
Requires-Dist: pyyaml>=6.0.2
|
|
26
|
+
Provides-Extra: duckdb
|
|
27
|
+
Requires-Dist: duckdb>=1.1; extra == 'duckdb'
|
|
28
|
+
Requires-Dist: numpy>=2.0; extra == 'duckdb'
|
|
29
|
+
Description-Content-Type: text/markdown
|
|
30
|
+
|
|
31
|
+
# sigma2sql
|
|
32
|
+
|
|
33
|
+
[](https://github.com/brkozer/sigma2sql/actions/workflows/ci.yml)
|
|
34
|
+

|
|
35
|
+

|
|
36
|
+
|
|
37
|
+
Run [Sigma](https://sigmahq.io) detection rules as SQL on **SQLite**, **PostgreSQL** and
|
|
38
|
+
**DuckDB**: hunt through log files without a SIEM, convert rules for the database your logs
|
|
39
|
+
already live in, and unit-test rules before they reach production.
|
|
40
|
+
|
|
41
|
+
- **Every SigmaHQ rule converts**: 3162 / 3162 on SQLite and DuckDB, 3160 / 3162 on
|
|
42
|
+
PostgreSQL, plus all 45 `count()` aggregation rules SigmaHQ keeps under `unsupported/`.
|
|
43
|
+
Every generated query is executed on all three databases in CI each week.
|
|
44
|
+
- **Checked against real events**: all 476 of SigmaHQ's regression samples match as expected,
|
|
45
|
+
and on those events sigma2sql agrees with pySigma on 3074 of 3076 rules; the two
|
|
46
|
+
differences are IPv6 addresses pySigma's text matching misses
|
|
47
|
+
([details](docs/correctness.md)).
|
|
48
|
+
- **Correlations**: event counts, distinct counts, sums and temporal sequences over a sliding
|
|
49
|
+
window, as plain SQL.
|
|
50
|
+
- **Your schema**: profiles map Sigma fields to your columns or nested JSON (Sysmon, EVTX
|
|
51
|
+
exports, Elastic/Winlogbeat ECS, OCSF built in) and limit each rule to the right log source.
|
|
52
|
+
|
|
53
|
+
<p align="center"><img src="https://raw.githubusercontent.com/brkozer/sigma2sql/main/docs/demo.svg" alt="sigma2sql hunting through sample logs, checking rule tests and converting a rule to PostgreSQL" width="100%"></p>
|
|
54
|
+
|
|
55
|
+
## Install
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
uv tool install "sigma2sql[duckdb] @ git+https://github.com/brkozer/sigma2sql"
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
The `duckdb` extra is needed for `hunt`; `convert`, `test` and `check` only need PyYAML.
|
|
62
|
+
|
|
63
|
+
## Hunt through log files
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
git clone --depth 1 https://github.com/SigmaHQ/sigma
|
|
67
|
+
sigma2sql hunt sigma/rules --logs 'logs/*.json' --profile evtx --examples 1
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
`hunt` loads JSON, JSON Lines, Parquet or CSV files into an in-memory DuckDB database, runs
|
|
71
|
+
every rule and correlation, and lists the ones that matched, most severe first. On the
|
|
72
|
+
examples in this repository:
|
|
73
|
+
|
|
74
|
+
```console
|
|
75
|
+
$ sigma2sql hunt examples/rules --logs examples/events.jsonl
|
|
76
|
+
LEVEL HITS RULE
|
|
77
|
+
high 2 PowerShell started with an encoded command (examples/rules/encoded_powershell.yml)
|
|
78
|
+
high 1 Brute force against one account (examples/rules/brute_force.yml)
|
|
79
|
+
high 1 Office application starting a shell (examples/rules/office_spawns_shell.yml)
|
|
80
|
+
medium 1 Successful RDP logon from a public address (examples/rules/rdp_from_outside.yml)
|
|
81
|
+
4 of 4 rules matched 12 events (0 rules skipped)
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
`--examples N` prints matching events (or correlation results such as
|
|
85
|
+
`{"TargetUserName":"admin","window_start":"2026-11-01 03:12:01","window_end":"2026-11-01 03:12:17","value":3}`),
|
|
86
|
+
`--format json` gives a machine-readable report and `--fail-on high` a non-zero exit code
|
|
87
|
+
when a rule at that level or above fires.
|
|
88
|
+
|
|
89
|
+
## Convert rules
|
|
90
|
+
|
|
91
|
+
```console
|
|
92
|
+
$ sigma2sql convert examples/rules/office_spawns_shell.yml -d postgres --json-column data
|
|
93
|
+
-- Office application starting a shell (8a7f3e91-2c44-4b8e-b0f6-3e5d9c1a7b02)
|
|
94
|
+
SELECT * FROM "events" WHERE (("data"->>'ParentImage' ILIKE '%\\winword.exe' ESCAPE '\' OR ...)
|
|
95
|
+
AND ("data"->>'Image' ILIKE '%\\cmd.exe' ESCAPE '\' OR ...));
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
A whole directory works the same way, e.g.
|
|
99
|
+
`sigma2sql convert sigma/rules -d postgres -t security.events --json-column event --profile ecs-winlogbeat`.
|
|
100
|
+
Without `--json-column` each Sigma field is a column. Unsupported rules are listed on stderr
|
|
101
|
+
with the reason, and the command exits non-zero.
|
|
102
|
+
|
|
103
|
+
| Option | Meaning |
|
|
104
|
+
| --- | --- |
|
|
105
|
+
| `-d, --dialect` | `sqlite` (default), `postgres` or `duckdb` |
|
|
106
|
+
| `-t, --table` | table name, `schema.table` allowed (default `events`) |
|
|
107
|
+
| `--json-column` | read fields from this JSON / JSONB column |
|
|
108
|
+
| `--keyword-column` | column searched by keyword rules (in JSON mode the whole document) |
|
|
109
|
+
| `-p, --profile` | built-in profile or a YAML file (see below) |
|
|
110
|
+
| `-m, --mapping` | YAML map from Sigma field to your column or key, applied over the profile |
|
|
111
|
+
| `--nested` | dotted names are paths into nested JSON |
|
|
112
|
+
| `--timestamp-field`, `--timestamp-format` | event time for correlations (`iso`, `epoch_s`, `epoch_ms`) |
|
|
113
|
+
|
|
114
|
+
## Correlations
|
|
115
|
+
|
|
116
|
+
Sigma correlation rules and the older `| count() by ... > N` syntax both become SQL. With a
|
|
117
|
+
`timespan`, each event closes a window over the events of the same group before it; the
|
|
118
|
+
window's events are found with a self-join, which works the same on all three databases and
|
|
119
|
+
allows distinct counts:
|
|
120
|
+
|
|
121
|
+
```yaml
|
|
122
|
+
title: Brute force against one account
|
|
123
|
+
correlation:
|
|
124
|
+
type: event_count # also value_count, value_sum, value_avg, temporal
|
|
125
|
+
rules: [failed_logon]
|
|
126
|
+
group-by: [TargetUserName]
|
|
127
|
+
timespan: 5m
|
|
128
|
+
condition: {gte: 3}
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
Each result row holds the group values, the window's start and end and the counted value.
|
|
132
|
+
Rules referenced by a correlation are not reported on their own unless it sets
|
|
133
|
+
`generate: true`. `temporal_ordered` is not supported yet.
|
|
134
|
+
|
|
135
|
+
## Profiles
|
|
136
|
+
|
|
137
|
+
A profile describes where your events keep Sigma's fields and which events each log source
|
|
138
|
+
covers. `sigma2sql profiles` lists the built-in ones:
|
|
139
|
+
|
|
140
|
+
| Profile | For |
|
|
141
|
+
| --- | --- |
|
|
142
|
+
| `sysmon` | Sysmon events with their own field names; each category limited to its event ID |
|
|
143
|
+
| `evtx` | EVTX exported to JSON by evtx_dump or python-evtx (`Event.System`, `Event.EventData`) |
|
|
144
|
+
| `ecs-winlogbeat` | Windows events shipped by Winlogbeat in Elastic Common Schema |
|
|
145
|
+
| `ocsf` | OCSF 1.x events, time in epoch milliseconds |
|
|
146
|
+
|
|
147
|
+
Your own profile is a small YAML file:
|
|
148
|
+
|
|
149
|
+
```yaml
|
|
150
|
+
nested: true
|
|
151
|
+
timestamp: "@timestamp"
|
|
152
|
+
default_field: "winlog.event_data.{field}" # fallback for unmapped fields
|
|
153
|
+
fields:
|
|
154
|
+
Image: process.executable
|
|
155
|
+
EventID: [event.code, winlog.event_id] # first one present wins
|
|
156
|
+
logsources:
|
|
157
|
+
- match: {product: windows, category: process_creation}
|
|
158
|
+
table: process_events # optional: route to a table
|
|
159
|
+
where: {event.code: "1"} # optional: extra condition
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
Without a profile, rules are not limited by log source: a network rule can then match a
|
|
163
|
+
process event that happens to have the same field values. The built-in profiles cover the
|
|
164
|
+
common Windows sources; for other data, add `logsources` entries.
|
|
165
|
+
|
|
166
|
+
## Test rules
|
|
167
|
+
|
|
168
|
+
Put example events next to a rule, one JSON object per line:
|
|
169
|
+
|
|
170
|
+
```
|
|
171
|
+
rules/office_spawns_shell.yml
|
|
172
|
+
rules/office_spawns_shell.match.jsonl # every event must match
|
|
173
|
+
rules/office_spawns_shell.nomatch.jsonl # no event may match
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
```console
|
|
177
|
+
$ sigma2sql check examples/rules --require-tests
|
|
178
|
+
ok examples/rules/brute_force.yml
|
|
179
|
+
ok examples/rules/encoded_powershell.yml
|
|
180
|
+
ok examples/rules/office_spawns_shell.yml
|
|
181
|
+
ok examples/rules/rdp_from_outside.yml
|
|
182
|
+
4 passed, 0 failed, 0 without tests
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
A failure names the rule, the file and the line, e.g. after adding an Outlook-spawned
|
|
186
|
+
`cmd.exe` to the match file of a rule that only watches Word, Excel and PowerPoint:
|
|
187
|
+
|
|
188
|
+
```console
|
|
189
|
+
FAIL rules/office_spawns_shell.yml: Office application starting a shell: missed office_spawns_shell.match.jsonl event(s) 1
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
The same check as a GitHub Action:
|
|
193
|
+
|
|
194
|
+
```yaml
|
|
195
|
+
- uses: brkozer/sigma2sql@v0.2.0
|
|
196
|
+
with:
|
|
197
|
+
path: rules
|
|
198
|
+
profile: sysmon # optional
|
|
199
|
+
require-tests: "true" # optional
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
`sigma2sql test RULE EVENTS.jsonl` prints the events (or correlation results) a single rule
|
|
203
|
+
produces.
|
|
204
|
+
|
|
205
|
+
## Supported Sigma features
|
|
206
|
+
|
|
207
|
+
| Feature | SQLite | PostgreSQL | DuckDB |
|
|
208
|
+
| --- | --- | --- | --- |
|
|
209
|
+
| maps, lists, keywords (with `all`, `re` and string modifiers) | ✓ | ✓ | ✓ |
|
|
210
|
+
| `and`, `or`, `not`, `1 of x*`, `all of x*`, `them`, condition lists | ✓ | ✓ | ✓ |
|
|
211
|
+
| wildcards and escapes, `contains`, `startswith`, `endswith`, `all`, `cased`, `windash` | ✓ | ✓ | ✓ |
|
|
212
|
+
| `base64`, `base64offset`, `utf16`, `utf16le`, `utf16be`, `wide` | ✓ | ✓ | ✓ |
|
|
213
|
+
| `re`, `re\|i` | `REGEXP` ¹ | `~` ² | `regexp_matches` ³ |
|
|
214
|
+
| `cidr` (IPv4 and IPv6, any notation) | `cidr_match()` ¹ | `inet <<=` | `cidr_match()` ¹ |
|
|
215
|
+
| `lt`, `lte`, `gt`, `gte`, `exists`, `fieldref`, `null` | ✓ | ✓ | ✓ |
|
|
216
|
+
| correlations, `count()` aggregations | ✓ | ✓ | ✓ |
|
|
217
|
+
| `expand` (placeholders), `temporal_ordered`, filter rules | – | – | – |
|
|
218
|
+
|
|
219
|
+
¹ Not built into the database. `hunt`, `test` and `check` register them; when you run the SQL
|
|
220
|
+
elsewhere, register `regexp(pattern, value)` and `cidr_match(value, network)` yourself
|
|
221
|
+
(`sigma2sql.runner.connect()` returns a SQLite connection with both).<br>
|
|
222
|
+
² PostgreSQL rejects repetition counts above 255; those two rules are reported as unsupported
|
|
223
|
+
rather than silently changed.<br>
|
|
224
|
+
³ RE2: no lookaround or backreferences; such rules are reported as unsupported on DuckDB.
|
|
225
|
+
|
|
226
|
+
Matching is case-insensitive by default (ASCII in SQLite, as its `LIKE` and `NOCASE` are);
|
|
227
|
+
values in a JSON column match whether they are stored as `4624` or `"4624"`, and `not` on a
|
|
228
|
+
missing field is true, as Sigma intends.
|
|
229
|
+
|
|
230
|
+
## How it works
|
|
231
|
+
|
|
232
|
+
```mermaid
|
|
233
|
+
flowchart LR
|
|
234
|
+
Y[rule files] --> L[loader<br/>rules, correlations,<br/>count aggregations]
|
|
235
|
+
L --> C[condition parser]
|
|
236
|
+
L --> D[search compiler<br/>modifiers, wildcards,<br/>base64, windash]
|
|
237
|
+
C --> E[match tree]
|
|
238
|
+
D --> E
|
|
239
|
+
P[profile<br/>fields, logsources,<br/>timestamp] --> R
|
|
240
|
+
E --> R[renderer<br/>SQLite · PostgreSQL · DuckDB]
|
|
241
|
+
R --> Q[SQL]
|
|
242
|
+
R --> K[correlation SQL<br/>self-join windows]
|
|
243
|
+
Q --> H[hunt / test / check]
|
|
244
|
+
K --> H
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
## Library use
|
|
248
|
+
|
|
249
|
+
```python
|
|
250
|
+
from sigma2sql import Target, load_rule_file, to_query
|
|
251
|
+
from sigma2sql.profile import apply_profile, load_profile
|
|
252
|
+
|
|
253
|
+
rule = load_rule_file("rule.yml")
|
|
254
|
+
target = apply_profile(
|
|
255
|
+
Target("postgres", "logs.windows", json_column="event"), load_profile("evtx")
|
|
256
|
+
)
|
|
257
|
+
print(to_query(rule, target))
|
|
258
|
+
```
|
|
259
|
+
|
|
260
|
+
## Development
|
|
261
|
+
|
|
262
|
+
```bash
|
|
263
|
+
uv sync
|
|
264
|
+
uv run pytest # SQLite and DuckDB
|
|
265
|
+
SIGMA2SQL_PG_DSN=postgresql://postgres:test@localhost/postgres uv run pytest # + PostgreSQL
|
|
266
|
+
uv run python scripts/check_sigmahq.py sigma/rules --dsn ... [--profile evtx]
|
|
267
|
+
uv run python scripts/check_regression.py sigma
|
|
268
|
+
uv run --with pysigma --with pysigma-backend-sqlite python scripts/diff_pysigma.py sigma
|
|
269
|
+
uv run python scripts/demo_svg.py docs/demo.svg
|
|
270
|
+
```
|
|
271
|
+
|
|
272
|
+
The behaviour tests in `tests/cases.py` and `tests/test_correlation.py` run unchanged on all
|
|
273
|
+
three databases. The example rules and events in `examples/` were written for this project.
|
|
274
|
+
|
|
275
|
+
## License
|
|
276
|
+
|
|
277
|
+
MIT
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
sigma2sql/__init__.py,sha256=cpLBs2Y68BYMLVhdEFVkD6tkNH4R6eJkKAC0Kgv8MLk,557
|
|
2
|
+
sigma2sql/__main__.py,sha256=4YSep7LVC8rdeGW9_2icqOpXY1VD5grwCkZQhIGOX68,57
|
|
3
|
+
sigma2sql/cli.py,sha256=BfgHahiMfy1Ln7iBITseev-1vBLkwlm3bude-dodch0,12929
|
|
4
|
+
sigma2sql/collection.py,sha256=TwTumiteM787v-MNUNOe-4Sk6mSfutgVTPKiUaKkF4k,2280
|
|
5
|
+
sigma2sql/condition.py,sha256=pyMqtZjMIElr0H7JJLP9ghJhku5z0jqIJwux_Hnc1-w,3944
|
|
6
|
+
sigma2sql/correlation.py,sha256=4r8KvDMMITbJ6DdiG0Z6BPhWpEP-84MdWy5Lgn_ckDA,3452
|
|
7
|
+
sigma2sql/detection.py,sha256=u3X4Ukq06KvHkNidnhyRZCnsPDrvazSWMUJHr-UB9C8,8832
|
|
8
|
+
sigma2sql/errors.py,sha256=EW7hyckgZU1OL0Rq8RZDOAa_fg0M6M2_eIHZ2Yf6ZKE,184
|
|
9
|
+
sigma2sql/expr.py,sha256=N9anXUvB1eaTg8Gj7YJVDhO9n9qUuHskVO4MdUZSzNw,3046
|
|
10
|
+
sigma2sql/hunt.py,sha256=NWTtZwohfQFY8PvDAndRG8ewXceAJooN5hQ8DK-B1Tc,4341
|
|
11
|
+
sigma2sql/profile.py,sha256=0TFABelU40CQlnkIBPKHG1v1rIeloLqOvbpGv5_IzA0,4597
|
|
12
|
+
sigma2sql/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
13
|
+
sigma2sql/rule.py,sha256=FcOpR5cT7pdNpzg3rpFpM7DFSNICpY5OZ6Csu76Fr5s,9359
|
|
14
|
+
sigma2sql/runner.py,sha256=VHVBVRSPzYbmCW9QA-iXVRErPfoGToGZV8RnMLkzkOg,2743
|
|
15
|
+
sigma2sql/sql.py,sha256=ghJDTI9O-POCsJo9vICyZqKHAIJmcs0QrHtsD9H9IcE,12908
|
|
16
|
+
sigma2sql/profiles/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
17
|
+
sigma2sql/profiles/ecs-winlogbeat.yml,sha256=xtjn6teXEMABcT7GaRAODFmYJWCEe2G1CRQSYwlyYlY,2536
|
|
18
|
+
sigma2sql/profiles/evtx.yml,sha256=wXNuvM4LpBdT7Xv0SAvokcF4nbVfxAOfaYbuOPpMBbs,7878
|
|
19
|
+
sigma2sql/profiles/ocsf.yml,sha256=JvaKI-NcNKJ6q0jivGKoO4I-hcnjmO-IQC2lKKp9qKk,979
|
|
20
|
+
sigma2sql/profiles/sysmon.yml,sha256=Cs63B2YAf1A3baCKKmUQR_p3z32-eE_w1x8gvQXa9lE,2018
|
|
21
|
+
sigma2sql-0.2.0.dist-info/METADATA,sha256=VlvZeBnDvSwz_ydvgT8M-yBVvbEVX21_JuAWv0R3BL4,11776
|
|
22
|
+
sigma2sql-0.2.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
23
|
+
sigma2sql-0.2.0.dist-info/entry_points.txt,sha256=CHAbeSFOiT1CGdNGjg6mjkOQ0EhV-OA5eLbhN7NqJ14,49
|
|
24
|
+
sigma2sql-0.2.0.dist-info/licenses/LICENSE,sha256=b3DflAVYTRz4ASbtAXIgB1cgVgkOOahMwfw7Zxi-g7A,1074
|
|
25
|
+
sigma2sql-0.2.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ahmet Burak Özer
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|