pySigma-backend-varpulis 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pysigma_backend_varpulis-0.1.0/.github/workflows/release.yml +76 -0
- pysigma_backend_varpulis-0.1.0/.github/workflows/test.yml +45 -0
- pysigma_backend_varpulis-0.1.0/.gitignore +6 -0
- pysigma_backend_varpulis-0.1.0/LICENSE +21 -0
- pysigma_backend_varpulis-0.1.0/PKG-INFO +221 -0
- pysigma_backend_varpulis-0.1.0/README.md +201 -0
- pysigma_backend_varpulis-0.1.0/pyproject.toml +33 -0
- pysigma_backend_varpulis-0.1.0/sigma/backends/varpulis/__init__.py +5 -0
- pysigma_backend_varpulis-0.1.0/sigma/backends/varpulis/varpulis.py +687 -0
- pysigma_backend_varpulis-0.1.0/tests/events/brute_force.jsonl +10 -0
- pysigma_backend_varpulis-0.1.0/tests/events/lateral_movement.jsonl +6 -0
- pysigma_backend_varpulis-0.1.0/tests/events/psexec_named.jsonl +4 -0
- pysigma_backend_varpulis-0.1.0/tests/rules/brute_force.yml +47 -0
- pysigma_backend_varpulis-0.1.0/tests/rules/lateral_movement.yml +39 -0
- pysigma_backend_varpulis-0.1.0/tests/rules/psexec.yml +27 -0
- pysigma_backend_varpulis-0.1.0/tests/test_backend.py +213 -0
- pysigma_backend_varpulis-0.1.0/tests/test_correlation.py +149 -0
- pysigma_backend_varpulis-0.1.0/tests/test_on_varpulis.py +82 -0
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
# A tag vX.Y.Z builds the wheel and the sdist, attaches them to a GitHub
|
|
4
|
+
# release, and publishes them to PyPI through trusted publishing (OIDC): no
|
|
5
|
+
# token lives in this repository. The PyPI job runs once the repository
|
|
6
|
+
# variable PYPI_PUBLISH is "true", which is set after the PyPI project (or its
|
|
7
|
+
# pending publisher) trusts this workflow. To publish a tag that was released
|
|
8
|
+
# before that, run this workflow by hand with the tag.
|
|
9
|
+
|
|
10
|
+
on:
|
|
11
|
+
push:
|
|
12
|
+
tags: ["v*"]
|
|
13
|
+
workflow_dispatch:
|
|
14
|
+
inputs:
|
|
15
|
+
tag:
|
|
16
|
+
description: "Existing tag to publish (vX.Y.Z)"
|
|
17
|
+
required: true
|
|
18
|
+
|
|
19
|
+
permissions:
|
|
20
|
+
contents: read
|
|
21
|
+
|
|
22
|
+
jobs:
|
|
23
|
+
build:
|
|
24
|
+
runs-on: ubuntu-latest
|
|
25
|
+
steps:
|
|
26
|
+
- uses: actions/checkout@v4
|
|
27
|
+
with:
|
|
28
|
+
ref: ${{ inputs.tag || github.ref }}
|
|
29
|
+
- uses: actions/setup-python@v5
|
|
30
|
+
with:
|
|
31
|
+
python-version: "3.12"
|
|
32
|
+
- name: The tag is the version
|
|
33
|
+
run: |
|
|
34
|
+
tag="${{ inputs.tag || github.ref_name }}"
|
|
35
|
+
version=$(python -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')
|
|
36
|
+
test "$tag" = "v$version" || { echo "tag $tag but pyproject says $version"; exit 1; }
|
|
37
|
+
- run: pip install build && python -m build
|
|
38
|
+
- uses: actions/upload-artifact@v4
|
|
39
|
+
with:
|
|
40
|
+
name: dist
|
|
41
|
+
path: dist/
|
|
42
|
+
|
|
43
|
+
github-release:
|
|
44
|
+
needs: build
|
|
45
|
+
if: github.event_name == 'push'
|
|
46
|
+
runs-on: ubuntu-latest
|
|
47
|
+
permissions:
|
|
48
|
+
contents: write
|
|
49
|
+
steps:
|
|
50
|
+
- uses: actions/download-artifact@v4
|
|
51
|
+
with:
|
|
52
|
+
name: dist
|
|
53
|
+
path: dist/
|
|
54
|
+
- name: Release with the wheel and the sdist
|
|
55
|
+
env:
|
|
56
|
+
GH_TOKEN: ${{ github.token }}
|
|
57
|
+
run: |
|
|
58
|
+
gh release create "$GITHUB_REF_NAME" dist/* --repo "$GITHUB_REPOSITORY" \
|
|
59
|
+
--title "$GITHUB_REF_NAME" \
|
|
60
|
+
--notes "pip install https://github.com/$GITHUB_REPOSITORY/releases/download/$GITHUB_REF_NAME/$(cd dist && ls *.whl)"
|
|
61
|
+
|
|
62
|
+
pypi:
|
|
63
|
+
needs: build
|
|
64
|
+
if: vars.PYPI_PUBLISH == 'true'
|
|
65
|
+
runs-on: ubuntu-latest
|
|
66
|
+
environment:
|
|
67
|
+
name: pypi
|
|
68
|
+
url: https://pypi.org/p/pysigma-backend-varpulis
|
|
69
|
+
permissions:
|
|
70
|
+
id-token: write
|
|
71
|
+
steps:
|
|
72
|
+
- uses: actions/download-artifact@v4
|
|
73
|
+
with:
|
|
74
|
+
name: dist
|
|
75
|
+
path: dist/
|
|
76
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
name: Test
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
schedule:
|
|
8
|
+
# The engine moves on its own main; catch a break within a day.
|
|
9
|
+
- cron: "17 5 * * *"
|
|
10
|
+
workflow_dispatch:
|
|
11
|
+
|
|
12
|
+
concurrency:
|
|
13
|
+
group: ${{ github.workflow }}-${{ github.ref }}
|
|
14
|
+
cancel-in-progress: true
|
|
15
|
+
|
|
16
|
+
jobs:
|
|
17
|
+
test:
|
|
18
|
+
runs-on: ubuntu-latest
|
|
19
|
+
strategy:
|
|
20
|
+
fail-fast: false
|
|
21
|
+
matrix:
|
|
22
|
+
python: ["3.10", "3.12", "3.13"]
|
|
23
|
+
steps:
|
|
24
|
+
- uses: actions/checkout@v4
|
|
25
|
+
- uses: actions/setup-python@v5
|
|
26
|
+
with:
|
|
27
|
+
python-version: ${{ matrix.python }}
|
|
28
|
+
- name: The engine the programs are written for
|
|
29
|
+
id: engine
|
|
30
|
+
run: |
|
|
31
|
+
rev=$(git ls-remote https://github.com/varpulis/varpulis refs/heads/main | cut -f1)
|
|
32
|
+
echo "rev=$rev" >> "$GITHUB_OUTPUT"
|
|
33
|
+
- uses: actions/cache@v4
|
|
34
|
+
id: cache
|
|
35
|
+
with:
|
|
36
|
+
path: ~/.cargo/bin/varpulis
|
|
37
|
+
key: varpulis-${{ steps.engine.outputs.rev }}
|
|
38
|
+
- name: Build varpulis
|
|
39
|
+
if: steps.cache.outputs.cache-hit != 'true'
|
|
40
|
+
run: cargo install --locked --git https://github.com/varpulis/varpulis --rev ${{ steps.engine.outputs.rev }} varpulis-cli
|
|
41
|
+
- run: pip install -e '.[test]'
|
|
42
|
+
- name: Tests, the engine ones included
|
|
43
|
+
run: VARPULIS_BIN=~/.cargo/bin/varpulis pytest -rs
|
|
44
|
+
- name: The engine tests really ran
|
|
45
|
+
run: VARPULIS_BIN=~/.cargo/bin/varpulis pytest -q tests/test_on_varpulis.py | tee out.txt && ! grep -q skipped out.txt
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Cyril Poder
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: pySigma-backend-varpulis
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: pySigma backend for Varpulis VPL: Sigma rules and correlation rules as detection streams
|
|
5
|
+
Project-URL: Homepage, https://www.varpulis-cep.com
|
|
6
|
+
Project-URL: Repository, https://github.com/varpulis/pySigma-backend-varpulis
|
|
7
|
+
Project-URL: Issues, https://github.com/varpulis/pySigma-backend-varpulis/issues
|
|
8
|
+
Author: Cyril Poder
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: correlation,detection,pysigma,siem,sigma,varpulis
|
|
12
|
+
Classifier: Intended Audience :: Information Technology
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Topic :: Security
|
|
15
|
+
Requires-Python: >=3.10
|
|
16
|
+
Requires-Dist: pysigma<2.0,>=1.0
|
|
17
|
+
Provides-Extra: test
|
|
18
|
+
Requires-Dist: pytest>=8; extra == 'test'
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
|
|
21
|
+
# pySigma-backend-varpulis
|
|
22
|
+
|
|
23
|
+
A [pySigma](https://github.com/SigmaHQ/pySigma) backend that turns Sigma rules,
|
|
24
|
+
correlation rules included, into [VPL](https://www.varpulis-cep.com/docs/language/overview),
|
|
25
|
+
the rule language of the Varpulis detection engine. The output is a program you
|
|
26
|
+
can run as it is: offline against a file of events with `varpulis simulate`, or
|
|
27
|
+
on a NATS bus as a detect unit of [Vejas](https://vejas.dev).
|
|
28
|
+
|
|
29
|
+
Correlation is the part worth looking at. Sigma's correlation rules
|
|
30
|
+
(`temporal_ordered`, `temporal`, `event_count`, `value_count`) are converted by
|
|
31
|
+
a handful of backends, each into a query over stored events that buckets
|
|
32
|
+
time. Here it becomes what a streaming engine does natively: a `temporal_ordered` correlation is a sequence matched
|
|
33
|
+
as events arrive, in the time the logs carry, and its state survives a restart
|
|
34
|
+
when it runs in Vejas.
|
|
35
|
+
|
|
36
|
+
## Install and convert
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
pip install sigma-cli
|
|
40
|
+
pip install https://github.com/varpulis/pySigma-backend-varpulis/releases/download/v0.1.0/pysigma_backend_varpulis-0.1.0-py3-none-any.whl
|
|
41
|
+
sigma convert -t varpulis rules/ # a VPL program on stdout
|
|
42
|
+
sigma convert -t varpulis -f vejas rules/ # the same, bound to a NATS bus
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
`pip install git+https://github.com/varpulis/pySigma-backend-varpulis` gives
|
|
46
|
+
the latest commit instead of the release. The package is not on PyPI yet.
|
|
47
|
+
|
|
48
|
+
The generated programs need a Varpulis engine with single-quoted raw strings,
|
|
49
|
+
`regex_match` and backticked field names, which is `main` from 2026-09-23 on
|
|
50
|
+
(`cargo install --git https://github.com/varpulis/varpulis varpulis-cli`).
|
|
51
|
+
|
|
52
|
+
On the 3 760 rules of the SigmaHQ repository (2026-09-22), all 3 760 convert
|
|
53
|
+
with `-O keyword_field=message` and 3 653 without it (the 107 others are
|
|
54
|
+
keyword searches, see below), and every generated program passes `varpulis
|
|
55
|
+
check`, the engine's parser and semantic validator. That says the programs
|
|
56
|
+
are well formed. What they catch is tested on the rules in `tests/rules`,
|
|
57
|
+
against events, not on the whole corpus.
|
|
58
|
+
|
|
59
|
+
## What a rule becomes
|
|
60
|
+
|
|
61
|
+
The PsExec rule from SigmaHQ, reduced to its selection:
|
|
62
|
+
|
|
63
|
+
```yaml
|
|
64
|
+
title: PsExec Execution
|
|
65
|
+
logsource:
|
|
66
|
+
category: process_creation
|
|
67
|
+
product: windows
|
|
68
|
+
detection:
|
|
69
|
+
selection:
|
|
70
|
+
- Image|endswith: ['\PsExec.exe', '\PsExec64.exe']
|
|
71
|
+
- OriginalFileName: 'psexec.c'
|
|
72
|
+
condition: selection
|
|
73
|
+
level: high
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
```vpl
|
|
77
|
+
# PsExec Execution
|
|
78
|
+
stream PsExecExecution = SysmonProcessCreate
|
|
79
|
+
.where(ends_with(lower(Image), '\psexec.exe') or ends_with(lower(Image), '\psexec64.exe') or lower(OriginalFileName) == 'psexec.c')
|
|
80
|
+
.emit(
|
|
81
|
+
rule: 'PsExec Execution',
|
|
82
|
+
sigma_id: '730fc21b-eaff-474b-ad23-90fd265d4988',
|
|
83
|
+
level: 'high',
|
|
84
|
+
mitre: 'T1569.002,T1021.002',
|
|
85
|
+
Image: Image,
|
|
86
|
+
OriginalFileName: OriginalFileName,
|
|
87
|
+
Computer: Computer,
|
|
88
|
+
Hostname: Hostname,
|
|
89
|
+
User: User,
|
|
90
|
+
CommandLine: CommandLine,
|
|
91
|
+
ParentImage: ParentImage,
|
|
92
|
+
ParentCommandLine: ParentCommandLine
|
|
93
|
+
)
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
The fields after the rule's own are the ones an analyst needs to act on a
|
|
97
|
+
process alert; one the event does not carry is left out of the alert.
|
|
98
|
+
|
|
99
|
+
Rename `PsExec.exe` to `svcupdate.exe` and that rule goes quiet. The behaviour
|
|
100
|
+
does not change though (an SMB connection, then a process that `services.exe`
|
|
101
|
+
starts on the target), and that is a two-rule correlation:
|
|
102
|
+
|
|
103
|
+
```yaml
|
|
104
|
+
correlation:
|
|
105
|
+
type: temporal_ordered
|
|
106
|
+
rules:
|
|
107
|
+
- smb_connection # DestinationPort: 445
|
|
108
|
+
- service_child # ParentImage|endswith: '\services.exe'
|
|
109
|
+
timespan: 2m
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
```vpl
|
|
113
|
+
stream LateralMovementOverSMBJudgedOnBehaviour = SmbConnection as a
|
|
114
|
+
-> ServiceChild as b
|
|
115
|
+
.within(2m)
|
|
116
|
+
.emit(
|
|
117
|
+
rule: 'Lateral movement over SMB, judged on behaviour',
|
|
118
|
+
level: 'critical',
|
|
119
|
+
SmbConnection_Hostname: a.Hostname,
|
|
120
|
+
ServiceChild_Hostname: b.Hostname,
|
|
121
|
+
ServiceChild_CommandLine: b.CommandLine
|
|
122
|
+
# ... and the other context fields of both events
|
|
123
|
+
)
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
Both files are in [`tests/rules`](tests/rules), and
|
|
127
|
+
[`tests/test_on_varpulis.py`](tests/test_on_varpulis.py) runs them through the
|
|
128
|
+
engine: the file name rule fires on PsExec and stays silent on the renamed
|
|
129
|
+
copy, the correlation catches the renamed copy across the two hosts.
|
|
130
|
+
|
|
131
|
+
## How things map
|
|
132
|
+
|
|
133
|
+
| Sigma | VPL |
|
|
134
|
+
|---|---|
|
|
135
|
+
| a value (case-insensitive, as Sigma specifies) | `lower(Field) == 'value'` |
|
|
136
|
+
| `startswith`, `endswith`, `contains` | `starts_with(lower(F), '...')` and friends |
|
|
137
|
+
| `cased` | the same without `lower()` |
|
|
138
|
+
| wildcards inside a value | `regex_match(F, '(?is)^...$')` |
|
|
139
|
+
| `re` (with `i`, `m`, `s`) | `regex_match(F, '(?i)...')` |
|
|
140
|
+
| numbers, `gt`/`gte`/`lt`/`lte` | `F == 4625`, `F >= 1000` |
|
|
141
|
+
| `null`, `exists` | `is_null(F)`, `not is_null(F)` |
|
|
142
|
+
| `cidr` | prefix matches (`starts_with(lower(F), '10.')`) |
|
|
143
|
+
| `fieldref` (and with `startswith`, `endswith`, `contains`) | `F == G`, `contains(F, G)` |
|
|
144
|
+
| a field name that is not an identifier (`cs-uri-query`) | `` `cs-uri-query` `` |
|
|
145
|
+
| keywords (a value with no field), with `-O keyword_field=message` | `contains(lower(message), 'value')` |
|
|
146
|
+
| `temporal_ordered` | a sequence `A as a -> B where g == a.g as b .within(T)` |
|
|
147
|
+
| `temporal` | that sequence in every order of its rules (up to three) |
|
|
148
|
+
| `event_count`, `value_count` | `.partition_by(g).window(T).aggregate(n: count())`, or `count_distinct(field)` |
|
|
149
|
+
| `value_sum`, `value_avg` | `sum(field)`, `avg(field)` |
|
|
150
|
+
|
|
151
|
+
Values are written as single-quoted VPL strings, which are raw: a backslash
|
|
152
|
+
is only a backslash, so `'\AppData\Local\Temp\'` goes through exactly as the
|
|
153
|
+
rule wrote it, and `''` stands for a quote.
|
|
154
|
+
|
|
155
|
+
A condition on a field the event does not carry is false, and `not` of it is
|
|
156
|
+
true, so `selection and not filter` keeps an event that lacks the filter's
|
|
157
|
+
field. That is how Splunk and Elasticsearch behave too, which matters when you
|
|
158
|
+
compare the alerts of a converted rule with the ones your SIEM raised.
|
|
159
|
+
|
|
160
|
+
The event type comes from the log source. Windows categories take the names
|
|
161
|
+
`varpulis simulate` gives Sysmon events (`SysmonProcessCreate`,
|
|
162
|
+
`SysmonNetworkConnect`, ...); any other log source is its product and
|
|
163
|
+
service or category in CamelCase (`WindowsSecurity`, `LinuxProcessCreation`,
|
|
164
|
+
`Proxy`). `-O event_type=MyEvents` forces one type for every rule.
|
|
165
|
+
|
|
166
|
+
## Options
|
|
167
|
+
|
|
168
|
+
| Option | Default | Meaning |
|
|
169
|
+
|---|---|---|
|
|
170
|
+
| `-O event_type=X` | from the log source | read every rule from event type `X` |
|
|
171
|
+
| `-O keyword_field=F` | none | the field that holds the log line, where keywords are searched |
|
|
172
|
+
| `-O dots=flat` | `nested` | read `id.orig_h` as one flat key instead of a path into nested objects |
|
|
173
|
+
| `-O subject_prefix=P` (`-f vejas`) | `logs` | event type `T` is read from the subject `P.T` |
|
|
174
|
+
| `-O alert_subject=S` (`-f vejas`) | `alerts.sigma` | where alerts are published |
|
|
175
|
+
|
|
176
|
+
## What does not convert, and why
|
|
177
|
+
|
|
178
|
+
- **Keyword detections** (a value with no field), unless you name the field
|
|
179
|
+
that holds the log line with `-O keyword_field=message`. An event has no
|
|
180
|
+
text of all its fields to search.
|
|
181
|
+
- **PCRE-only regular expressions.** The engine uses Rust's `regex`, which
|
|
182
|
+
matches in linear time whatever the input and so has no look-around and no
|
|
183
|
+
back-references. The conversion refuses such a rule and says which construct
|
|
184
|
+
it met, rather than emitting a pattern that would never compile.
|
|
185
|
+
- **A dotted field name is a path**: `process.parent.name` reads nested
|
|
186
|
+
objects. If your events carry flat keys with dots in them (Zeek's JSON
|
|
187
|
+
writes `id.orig_h` that way), pass `-O dots=flat`.
|
|
188
|
+
- **`temporal` over four rules or more**, which would take 24 orders and more.
|
|
189
|
+
Write it as `temporal_ordered` when the order is known.
|
|
190
|
+
- **`value_percentile`, `value_median`**, timestamp-part modifiers.
|
|
191
|
+
|
|
192
|
+
Two behaviours to know about. Count correlations use tumbling windows, the way
|
|
193
|
+
the Splunk and Elasticsearch backends bucket time, so a burst that straddles a
|
|
194
|
+
window boundary is counted in two halves. And a `temporal` correlation can
|
|
195
|
+
alert twice when its events come in both orders (A, B, A).
|
|
196
|
+
|
|
197
|
+
## Testing a conversion on your own logs
|
|
198
|
+
|
|
199
|
+
```bash
|
|
200
|
+
sigma convert -t varpulis my_rules/ > rules.vpl
|
|
201
|
+
varpulis check rules.vpl
|
|
202
|
+
varpulis simulate -p rules.vpl -e events.jsonl -w 1
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
Each JSON line is one event. Sysmon lines (`EventID` and `Channel`) are typed
|
|
206
|
+
automatically; anything else needs a `"type"` naming the event type the rule
|
|
207
|
+
reads, and an `@timestamp`, since sequences and windows are judged in the time
|
|
208
|
+
the events carry.
|
|
209
|
+
|
|
210
|
+
## Development
|
|
211
|
+
|
|
212
|
+
```bash
|
|
213
|
+
pip install -e '.[test]'
|
|
214
|
+
VARPULIS_BIN=/path/to/varpulis pytest
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
Without a `varpulis` binary the engine tests are skipped, and pytest says so.
|
|
218
|
+
|
|
219
|
+
## License
|
|
220
|
+
|
|
221
|
+
MIT.
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
# pySigma-backend-varpulis
|
|
2
|
+
|
|
3
|
+
A [pySigma](https://github.com/SigmaHQ/pySigma) backend that turns Sigma rules,
|
|
4
|
+
correlation rules included, into [VPL](https://www.varpulis-cep.com/docs/language/overview),
|
|
5
|
+
the rule language of the Varpulis detection engine. The output is a program you
|
|
6
|
+
can run as it is: offline against a file of events with `varpulis simulate`, or
|
|
7
|
+
on a NATS bus as a detect unit of [Vejas](https://vejas.dev).
|
|
8
|
+
|
|
9
|
+
Correlation is the part worth looking at. Sigma's correlation rules
|
|
10
|
+
(`temporal_ordered`, `temporal`, `event_count`, `value_count`) are converted by
|
|
11
|
+
a handful of backends, each into a query over stored events that buckets
|
|
12
|
+
time. Here it becomes what a streaming engine does natively: a `temporal_ordered` correlation is a sequence matched
|
|
13
|
+
as events arrive, in the time the logs carry, and its state survives a restart
|
|
14
|
+
when it runs in Vejas.
|
|
15
|
+
|
|
16
|
+
## Install and convert
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
pip install sigma-cli
|
|
20
|
+
pip install https://github.com/varpulis/pySigma-backend-varpulis/releases/download/v0.1.0/pysigma_backend_varpulis-0.1.0-py3-none-any.whl
|
|
21
|
+
sigma convert -t varpulis rules/ # a VPL program on stdout
|
|
22
|
+
sigma convert -t varpulis -f vejas rules/ # the same, bound to a NATS bus
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
`pip install git+https://github.com/varpulis/pySigma-backend-varpulis` gives
|
|
26
|
+
the latest commit instead of the release. The package is not on PyPI yet.
|
|
27
|
+
|
|
28
|
+
The generated programs need a Varpulis engine with single-quoted raw strings,
|
|
29
|
+
`regex_match` and backticked field names, which is `main` from 2026-09-23 on
|
|
30
|
+
(`cargo install --git https://github.com/varpulis/varpulis varpulis-cli`).
|
|
31
|
+
|
|
32
|
+
On the 3 760 rules of the SigmaHQ repository (2026-09-22), all 3 760 convert
|
|
33
|
+
with `-O keyword_field=message` and 3 653 without it (the 107 others are
|
|
34
|
+
keyword searches, see below), and every generated program passes `varpulis
|
|
35
|
+
check`, the engine's parser and semantic validator. That says the programs
|
|
36
|
+
are well formed. What they catch is tested on the rules in `tests/rules`,
|
|
37
|
+
against events, not on the whole corpus.
|
|
38
|
+
|
|
39
|
+
## What a rule becomes
|
|
40
|
+
|
|
41
|
+
The PsExec rule from SigmaHQ, reduced to its selection:
|
|
42
|
+
|
|
43
|
+
```yaml
|
|
44
|
+
title: PsExec Execution
|
|
45
|
+
logsource:
|
|
46
|
+
category: process_creation
|
|
47
|
+
product: windows
|
|
48
|
+
detection:
|
|
49
|
+
selection:
|
|
50
|
+
- Image|endswith: ['\PsExec.exe', '\PsExec64.exe']
|
|
51
|
+
- OriginalFileName: 'psexec.c'
|
|
52
|
+
condition: selection
|
|
53
|
+
level: high
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
```vpl
|
|
57
|
+
# PsExec Execution
|
|
58
|
+
stream PsExecExecution = SysmonProcessCreate
|
|
59
|
+
.where(ends_with(lower(Image), '\psexec.exe') or ends_with(lower(Image), '\psexec64.exe') or lower(OriginalFileName) == 'psexec.c')
|
|
60
|
+
.emit(
|
|
61
|
+
rule: 'PsExec Execution',
|
|
62
|
+
sigma_id: '730fc21b-eaff-474b-ad23-90fd265d4988',
|
|
63
|
+
level: 'high',
|
|
64
|
+
mitre: 'T1569.002,T1021.002',
|
|
65
|
+
Image: Image,
|
|
66
|
+
OriginalFileName: OriginalFileName,
|
|
67
|
+
Computer: Computer,
|
|
68
|
+
Hostname: Hostname,
|
|
69
|
+
User: User,
|
|
70
|
+
CommandLine: CommandLine,
|
|
71
|
+
ParentImage: ParentImage,
|
|
72
|
+
ParentCommandLine: ParentCommandLine
|
|
73
|
+
)
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
The fields after the rule's own are the ones an analyst needs to act on a
|
|
77
|
+
process alert; one the event does not carry is left out of the alert.
|
|
78
|
+
|
|
79
|
+
Rename `PsExec.exe` to `svcupdate.exe` and that rule goes quiet. The behaviour
|
|
80
|
+
does not change though (an SMB connection, then a process that `services.exe`
|
|
81
|
+
starts on the target), and that is a two-rule correlation:
|
|
82
|
+
|
|
83
|
+
```yaml
|
|
84
|
+
correlation:
|
|
85
|
+
type: temporal_ordered
|
|
86
|
+
rules:
|
|
87
|
+
- smb_connection # DestinationPort: 445
|
|
88
|
+
- service_child # ParentImage|endswith: '\services.exe'
|
|
89
|
+
timespan: 2m
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
```vpl
|
|
93
|
+
stream LateralMovementOverSMBJudgedOnBehaviour = SmbConnection as a
|
|
94
|
+
-> ServiceChild as b
|
|
95
|
+
.within(2m)
|
|
96
|
+
.emit(
|
|
97
|
+
rule: 'Lateral movement over SMB, judged on behaviour',
|
|
98
|
+
level: 'critical',
|
|
99
|
+
SmbConnection_Hostname: a.Hostname,
|
|
100
|
+
ServiceChild_Hostname: b.Hostname,
|
|
101
|
+
ServiceChild_CommandLine: b.CommandLine
|
|
102
|
+
# ... and the other context fields of both events
|
|
103
|
+
)
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
Both files are in [`tests/rules`](tests/rules), and
|
|
107
|
+
[`tests/test_on_varpulis.py`](tests/test_on_varpulis.py) runs them through the
|
|
108
|
+
engine: the file name rule fires on PsExec and stays silent on the renamed
|
|
109
|
+
copy, the correlation catches the renamed copy across the two hosts.
|
|
110
|
+
|
|
111
|
+
## How things map
|
|
112
|
+
|
|
113
|
+
| Sigma | VPL |
|
|
114
|
+
|---|---|
|
|
115
|
+
| a value (case-insensitive, as Sigma specifies) | `lower(Field) == 'value'` |
|
|
116
|
+
| `startswith`, `endswith`, `contains` | `starts_with(lower(F), '...')` and friends |
|
|
117
|
+
| `cased` | the same without `lower()` |
|
|
118
|
+
| wildcards inside a value | `regex_match(F, '(?is)^...$')` |
|
|
119
|
+
| `re` (with `i`, `m`, `s`) | `regex_match(F, '(?i)...')` |
|
|
120
|
+
| numbers, `gt`/`gte`/`lt`/`lte` | `F == 4625`, `F >= 1000` |
|
|
121
|
+
| `null`, `exists` | `is_null(F)`, `not is_null(F)` |
|
|
122
|
+
| `cidr` | prefix matches (`starts_with(lower(F), '10.')`) |
|
|
123
|
+
| `fieldref` (and with `startswith`, `endswith`, `contains`) | `F == G`, `contains(F, G)` |
|
|
124
|
+
| a field name that is not an identifier (`cs-uri-query`) | `` `cs-uri-query` `` |
|
|
125
|
+
| keywords (a value with no field), with `-O keyword_field=message` | `contains(lower(message), 'value')` |
|
|
126
|
+
| `temporal_ordered` | a sequence `A as a -> B where g == a.g as b .within(T)` |
|
|
127
|
+
| `temporal` | that sequence in every order of its rules (up to three) |
|
|
128
|
+
| `event_count`, `value_count` | `.partition_by(g).window(T).aggregate(n: count())`, or `count_distinct(field)` |
|
|
129
|
+
| `value_sum`, `value_avg` | `sum(field)`, `avg(field)` |
|
|
130
|
+
|
|
131
|
+
Values are written as single-quoted VPL strings, which are raw: a backslash
|
|
132
|
+
is only a backslash, so `'\AppData\Local\Temp\'` goes through exactly as the
|
|
133
|
+
rule wrote it, and `''` stands for a quote.
|
|
134
|
+
|
|
135
|
+
A condition on a field the event does not carry is false, and `not` of it is
|
|
136
|
+
true, so `selection and not filter` keeps an event that lacks the filter's
|
|
137
|
+
field. That is how Splunk and Elasticsearch behave too, which matters when you
|
|
138
|
+
compare the alerts of a converted rule with the ones your SIEM raised.
|
|
139
|
+
|
|
140
|
+
The event type comes from the log source. Windows categories take the names
|
|
141
|
+
`varpulis simulate` gives Sysmon events (`SysmonProcessCreate`,
|
|
142
|
+
`SysmonNetworkConnect`, ...); any other log source is its product and
|
|
143
|
+
service or category in CamelCase (`WindowsSecurity`, `LinuxProcessCreation`,
|
|
144
|
+
`Proxy`). `-O event_type=MyEvents` forces one type for every rule.
|
|
145
|
+
|
|
146
|
+
## Options
|
|
147
|
+
|
|
148
|
+
| Option | Default | Meaning |
|
|
149
|
+
|---|---|---|
|
|
150
|
+
| `-O event_type=X` | from the log source | read every rule from event type `X` |
|
|
151
|
+
| `-O keyword_field=F` | none | the field that holds the log line, where keywords are searched |
|
|
152
|
+
| `-O dots=flat` | `nested` | read `id.orig_h` as one flat key instead of a path into nested objects |
|
|
153
|
+
| `-O subject_prefix=P` (`-f vejas`) | `logs` | event type `T` is read from the subject `P.T` |
|
|
154
|
+
| `-O alert_subject=S` (`-f vejas`) | `alerts.sigma` | where alerts are published |
|
|
155
|
+
|
|
156
|
+
## What does not convert, and why
|
|
157
|
+
|
|
158
|
+
- **Keyword detections** (a value with no field), unless you name the field
|
|
159
|
+
that holds the log line with `-O keyword_field=message`. An event has no
|
|
160
|
+
text of all its fields to search.
|
|
161
|
+
- **PCRE-only regular expressions.** The engine uses Rust's `regex`, which
|
|
162
|
+
matches in linear time whatever the input and so has no look-around and no
|
|
163
|
+
back-references. The conversion refuses such a rule and says which construct
|
|
164
|
+
it met, rather than emitting a pattern that would never compile.
|
|
165
|
+
- **A dotted field name is a path**: `process.parent.name` reads nested
|
|
166
|
+
objects. If your events carry flat keys with dots in them (Zeek's JSON
|
|
167
|
+
writes `id.orig_h` that way), pass `-O dots=flat`.
|
|
168
|
+
- **`temporal` over four rules or more**, which would take 24 orders and more.
|
|
169
|
+
Write it as `temporal_ordered` when the order is known.
|
|
170
|
+
- **`value_percentile`, `value_median`**, timestamp-part modifiers.
|
|
171
|
+
|
|
172
|
+
Two behaviours to know about. Count correlations use tumbling windows, the way
|
|
173
|
+
the Splunk and Elasticsearch backends bucket time, so a burst that straddles a
|
|
174
|
+
window boundary is counted in two halves. And a `temporal` correlation can
|
|
175
|
+
alert twice when its events come in both orders (A, B, A).
|
|
176
|
+
|
|
177
|
+
## Testing a conversion on your own logs
|
|
178
|
+
|
|
179
|
+
```bash
|
|
180
|
+
sigma convert -t varpulis my_rules/ > rules.vpl
|
|
181
|
+
varpulis check rules.vpl
|
|
182
|
+
varpulis simulate -p rules.vpl -e events.jsonl -w 1
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
Each JSON line is one event. Sysmon lines (`EventID` and `Channel`) are typed
|
|
186
|
+
automatically; anything else needs a `"type"` naming the event type the rule
|
|
187
|
+
reads, and an `@timestamp`, since sequences and windows are judged in the time
|
|
188
|
+
the events carry.
|
|
189
|
+
|
|
190
|
+
## Development
|
|
191
|
+
|
|
192
|
+
```bash
|
|
193
|
+
pip install -e '.[test]'
|
|
194
|
+
VARPULIS_BIN=/path/to/varpulis pytest
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
Without a `varpulis` binary the engine tests are skipped, and pytest says so.
|
|
198
|
+
|
|
199
|
+
## License
|
|
200
|
+
|
|
201
|
+
MIT.
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "pySigma-backend-varpulis"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "pySigma backend for Varpulis VPL: Sigma rules and correlation rules as detection streams"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
authors = [{ name = "Cyril Poder" }]
|
|
13
|
+
keywords = ["sigma", "pysigma", "detection", "siem", "varpulis", "correlation"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Programming Language :: Python :: 3",
|
|
16
|
+
"Topic :: Security",
|
|
17
|
+
"Intended Audience :: Information Technology",
|
|
18
|
+
]
|
|
19
|
+
dependencies = ["pysigma>=1.0,<2.0"]
|
|
20
|
+
|
|
21
|
+
[project.optional-dependencies]
|
|
22
|
+
test = ["pytest>=8"]
|
|
23
|
+
|
|
24
|
+
[project.urls]
|
|
25
|
+
Homepage = "https://www.varpulis-cep.com"
|
|
26
|
+
Repository = "https://github.com/varpulis/pySigma-backend-varpulis"
|
|
27
|
+
Issues = "https://github.com/varpulis/pySigma-backend-varpulis/issues"
|
|
28
|
+
|
|
29
|
+
[tool.hatch.build.targets.wheel]
|
|
30
|
+
packages = ["sigma"]
|
|
31
|
+
|
|
32
|
+
[tool.pytest.ini_options]
|
|
33
|
+
testpaths = ["tests"]
|