sentiel 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sentiel-1.0.0/.gitignore +225 -0
- sentiel-1.0.0/CHANGELOG.md +67 -0
- sentiel-1.0.0/CONTRIBUTING.md +131 -0
- sentiel-1.0.0/LICENSE +21 -0
- sentiel-1.0.0/PKG-INFO +206 -0
- sentiel-1.0.0/README.md +168 -0
- sentiel-1.0.0/README.pt-BR.md +168 -0
- sentiel-1.0.0/SECURITY.md +36 -0
- sentiel-1.0.0/contrib/sentiel-task.xml +45 -0
- sentiel-1.0.0/contrib/sentiel.service +45 -0
- sentiel-1.0.0/contrib/sysmon.md +39 -0
- sentiel-1.0.0/pyproject.toml +82 -0
- sentiel-1.0.0/scripts/check_dist.py +101 -0
- sentiel-1.0.0/scripts/generate_demo.py +128 -0
- sentiel-1.0.0/sentiel/__init__.py +1 -0
- sentiel-1.0.0/sentiel/bench.py +123 -0
- sentiel-1.0.0/sentiel/chat.py +110 -0
- sentiel-1.0.0/sentiel/cli.py +498 -0
- sentiel-1.0.0/sentiel/collectors.py +233 -0
- sentiel-1.0.0/sentiel/config.py +158 -0
- sentiel-1.0.0/sentiel/demo/auth_log.txt +71 -0
- sentiel-1.0.0/sentiel/demo/events.jsonl +119 -0
- sentiel-1.0.0/sentiel/demo/expected.json +53 -0
- sentiel-1.0.0/sentiel/detect.py +260 -0
- sentiel-1.0.0/sentiel/doctor.py +118 -0
- sentiel-1.0.0/sentiel/enrich.py +69 -0
- sentiel-1.0.0/sentiel/models.py +60 -0
- sentiel-1.0.0/sentiel/notify.py +48 -0
- sentiel-1.0.0/sentiel/pipeline.py +268 -0
- sentiel-1.0.0/sentiel/rules/lnx_ssh_bruteforce.yml +30 -0
- sentiel-1.0.0/sentiel/rules/lnx_sudo_failure.yml +20 -0
- sentiel-1.0.0/sentiel/rules/lnx_useradd.yml +20 -0
- sentiel-1.0.0/sentiel/rules/win_logon_bruteforce.yml +30 -0
- sentiel-1.0.0/sentiel/rules/win_powershell_encoded.yml +37 -0
- sentiel-1.0.0/sentiel/rules/win_run_key_persistence.yml +23 -0
- sentiel-1.0.0/sentiel/rules/win_security_log_cleared.yml +21 -0
- sentiel-1.0.0/sentiel/rules/win_user_added_to_admins.yml +22 -0
- sentiel-1.0.0/sentiel/store.py +326 -0
- sentiel-1.0.0/sentiel/triage.py +117 -0
- sentiel-1.0.0/sentiel.example.toml +44 -0
- sentiel-1.0.0/tests/test_chat_enrich_bench.py +160 -0
- sentiel-1.0.0/tests/test_cli.py +800 -0
- sentiel-1.0.0/tests/test_detect.py +361 -0
- sentiel-1.0.0/tests/test_doctor.py +159 -0
- sentiel-1.0.0/tests/test_llm_live.py +24 -0
- sentiel-1.0.0/tests/test_packaging.py +133 -0
- sentiel-1.0.0/tests/test_pipeline.py +647 -0
- sentiel-1.0.0/tests/test_store_collectors.py +570 -0
- sentiel-1.0.0/tests/test_triage.py +73 -0
- sentiel-1.0.0/tests/test_workflows.py +221 -0
sentiel-1.0.0/.gitignore
ADDED
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[codz]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# C extensions
|
|
7
|
+
*.so
|
|
8
|
+
|
|
9
|
+
# Distribution / packaging
|
|
10
|
+
.Python
|
|
11
|
+
build/
|
|
12
|
+
develop-eggs/
|
|
13
|
+
dist/
|
|
14
|
+
downloads/
|
|
15
|
+
eggs/
|
|
16
|
+
.eggs/
|
|
17
|
+
lib/
|
|
18
|
+
lib64/
|
|
19
|
+
parts/
|
|
20
|
+
sdist/
|
|
21
|
+
var/
|
|
22
|
+
wheels/
|
|
23
|
+
share/python-wheels/
|
|
24
|
+
*.egg-info/
|
|
25
|
+
.installed.cfg
|
|
26
|
+
*.egg
|
|
27
|
+
MANIFEST
|
|
28
|
+
|
|
29
|
+
# PyInstaller
|
|
30
|
+
# Usually these files are written by a python script from a template
|
|
31
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
32
|
+
*.manifest
|
|
33
|
+
*.spec
|
|
34
|
+
|
|
35
|
+
# Installer logs
|
|
36
|
+
pip-log.txt
|
|
37
|
+
pip-delete-this-directory.txt
|
|
38
|
+
|
|
39
|
+
# Unit test / coverage reports
|
|
40
|
+
htmlcov/
|
|
41
|
+
.tox/
|
|
42
|
+
.nox/
|
|
43
|
+
.coverage
|
|
44
|
+
.coverage.*
|
|
45
|
+
.cache
|
|
46
|
+
nosetests.xml
|
|
47
|
+
coverage.xml
|
|
48
|
+
*.cover
|
|
49
|
+
*.py.cover
|
|
50
|
+
.hypothesis/
|
|
51
|
+
.pytest_cache/
|
|
52
|
+
cover/
|
|
53
|
+
|
|
54
|
+
# Translations
|
|
55
|
+
*.mo
|
|
56
|
+
*.pot
|
|
57
|
+
|
|
58
|
+
# Django stuff:
|
|
59
|
+
*.log
|
|
60
|
+
local_settings.py
|
|
61
|
+
db.sqlite3
|
|
62
|
+
db.sqlite3-journal
|
|
63
|
+
|
|
64
|
+
# Flask stuff:
|
|
65
|
+
instance/
|
|
66
|
+
.webassets-cache
|
|
67
|
+
|
|
68
|
+
# Scrapy stuff:
|
|
69
|
+
.scrapy
|
|
70
|
+
|
|
71
|
+
# Sphinx documentation
|
|
72
|
+
docs/_build/
|
|
73
|
+
docs/
|
|
74
|
+
|
|
75
|
+
# PyBuilder
|
|
76
|
+
.pybuilder/
|
|
77
|
+
target/
|
|
78
|
+
|
|
79
|
+
# Jupyter Notebook
|
|
80
|
+
.ipynb_checkpoints
|
|
81
|
+
|
|
82
|
+
# IPython
|
|
83
|
+
profile_default/
|
|
84
|
+
ipython_config.py
|
|
85
|
+
|
|
86
|
+
# pyenv
|
|
87
|
+
# For a library or package, you might want to ignore these files since the code is
|
|
88
|
+
# intended to run in multiple environments; otherwise, check them in:
|
|
89
|
+
# .python-version
|
|
90
|
+
|
|
91
|
+
# pipenv
|
|
92
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
93
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
94
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
95
|
+
# install all needed dependencies.
|
|
96
|
+
# Pipfile.lock
|
|
97
|
+
|
|
98
|
+
# UV
|
|
99
|
+
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
|
100
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
101
|
+
# commonly ignored for libraries.
|
|
102
|
+
# uv.lock
|
|
103
|
+
|
|
104
|
+
# poetry
|
|
105
|
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
|
106
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
107
|
+
# commonly ignored for libraries.
|
|
108
|
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
|
109
|
+
# poetry.lock
|
|
110
|
+
# poetry.toml
|
|
111
|
+
|
|
112
|
+
# pdm
|
|
113
|
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
|
114
|
+
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
|
|
115
|
+
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
|
|
116
|
+
# pdm.lock
|
|
117
|
+
# pdm.toml
|
|
118
|
+
.pdm-python
|
|
119
|
+
.pdm-build/
|
|
120
|
+
|
|
121
|
+
# pixi
|
|
122
|
+
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
|
|
123
|
+
# pixi.lock
|
|
124
|
+
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
|
|
125
|
+
# in the .venv directory. It is recommended not to include this directory in version control.
|
|
126
|
+
.pixi
|
|
127
|
+
|
|
128
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
|
129
|
+
__pypackages__/
|
|
130
|
+
|
|
131
|
+
# Celery stuff
|
|
132
|
+
celerybeat-schedule
|
|
133
|
+
celerybeat.pid
|
|
134
|
+
|
|
135
|
+
# Redis
|
|
136
|
+
*.rdb
|
|
137
|
+
*.aof
|
|
138
|
+
*.pid
|
|
139
|
+
|
|
140
|
+
# RabbitMQ
|
|
141
|
+
mnesia/
|
|
142
|
+
rabbitmq/
|
|
143
|
+
rabbitmq-data/
|
|
144
|
+
|
|
145
|
+
# ActiveMQ
|
|
146
|
+
activemq-data/
|
|
147
|
+
|
|
148
|
+
# SageMath parsed files
|
|
149
|
+
*.sage.py
|
|
150
|
+
|
|
151
|
+
# Environments
|
|
152
|
+
.env
|
|
153
|
+
.envrc
|
|
154
|
+
.venv
|
|
155
|
+
env/
|
|
156
|
+
venv/
|
|
157
|
+
ENV/
|
|
158
|
+
env.bak/
|
|
159
|
+
venv.bak/
|
|
160
|
+
|
|
161
|
+
# Spyder project settings
|
|
162
|
+
.spyderproject
|
|
163
|
+
.spyproject
|
|
164
|
+
|
|
165
|
+
# Rope project settings
|
|
166
|
+
.ropeproject
|
|
167
|
+
|
|
168
|
+
# mkdocs documentation
|
|
169
|
+
/site
|
|
170
|
+
|
|
171
|
+
# mypy
|
|
172
|
+
.mypy_cache/
|
|
173
|
+
.dmypy.json
|
|
174
|
+
dmypy.json
|
|
175
|
+
|
|
176
|
+
# Pyre type checker
|
|
177
|
+
.pyre/
|
|
178
|
+
|
|
179
|
+
# pytype static type analyzer
|
|
180
|
+
.pytype/
|
|
181
|
+
|
|
182
|
+
# Cython debug symbols
|
|
183
|
+
cython_debug/
|
|
184
|
+
|
|
185
|
+
# PyCharm
|
|
186
|
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
|
187
|
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
|
188
|
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
|
189
|
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
|
190
|
+
# .idea/
|
|
191
|
+
|
|
192
|
+
# Abstra
|
|
193
|
+
# Abstra is an AI-powered process automation framework.
|
|
194
|
+
# Ignore directories containing user credentials, local state, and settings.
|
|
195
|
+
# Learn more at https://abstra.io/docs
|
|
196
|
+
.abstra/
|
|
197
|
+
|
|
198
|
+
# Visual Studio Code
|
|
199
|
+
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
|
|
200
|
+
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
|
201
|
+
# and can be added to the global gitignore or merged into this file. However, if you prefer,
|
|
202
|
+
# you could uncomment the following to ignore the entire vscode folder
|
|
203
|
+
# .vscode/
|
|
204
|
+
# Temporary file for partial code execution
|
|
205
|
+
tempCodeRunnerFile.py
|
|
206
|
+
|
|
207
|
+
# Ruff stuff:
|
|
208
|
+
.ruff_cache/
|
|
209
|
+
|
|
210
|
+
# PyPI configuration file
|
|
211
|
+
.pypirc
|
|
212
|
+
|
|
213
|
+
# Marimo
|
|
214
|
+
marimo/_static/
|
|
215
|
+
marimo/_lsp/
|
|
216
|
+
__marimo__/
|
|
217
|
+
|
|
218
|
+
# Streamlit
|
|
219
|
+
.streamlit/secrets.toml
|
|
220
|
+
|
|
221
|
+
# Sentiel
|
|
222
|
+
sentiel.toml
|
|
223
|
+
sentiel.db
|
|
224
|
+
sentiel.db-*
|
|
225
|
+
bench.md
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to Sentiel are documented here. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project uses [Semantic Versioning](https://semver.org/).
|
|
4
|
+
|
|
5
|
+
## [1.0.0] - unreleased (tag v1.0.0 pending)
|
|
6
|
+
|
|
7
|
+
The version is set to 1.0.0 in the code, but it is not published yet. The maintainer replaces "unreleased (tag v1.0.0 pending)" with the release date when tagging `v1.0.0`.
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
- Release to PyPI: `pipx install sentiel`. Tags `vX.Y.Z` trigger `.github/workflows/release.yml`, which checks the tag against `sentiel.__version__`, builds the sdist and wheel and publishes with PyPI trusted publishing (no token).
|
|
11
|
+
- `scripts/check_dist.py` verifies the built sdist and wheel: rule files and demo data are in the wheel, and secrets, databases and local folders (`sentiel.toml`, `*.db`, `.env*`, `.venv/`, `docs/`) are in neither.
|
|
12
|
+
- `sentiel bench MODELS --readme` refreshes the benchmark table between the `<!-- BENCH:START -->` and `<!-- BENCH:END -->` markers of a README (`--readme-file PATH` picks the file). With missing, duplicated or reversed markers it fails without writing. Model names are escaped in the table.
|
|
13
|
+
- CI: Python 3.14 in the test matrix (informational, `continue-on-error`) and a `build` job (`python -m build`, `twine check`, `scripts/check_dist.py`). Dependabot for `pip` and GitHub Actions.
|
|
14
|
+
- Community files: `CONTRIBUTING.md`, `SECURITY.md`, this changelog, issue templates (new rule, false positive) and `contrib/sysmon.md`.
|
|
15
|
+
- Package metadata (authors, keywords, classifiers, URLs). The version lives only in `sentiel/__init__.py`.
|
|
16
|
+
|
|
17
|
+
### Changed
|
|
18
|
+
- Strict config: an unknown key at the top level, in `[sources]`, in `[[sources.files]]`, in `[keys]` or in `[notify]` is now an error (exit 2, or FAIL in `sentiel doctor`) that names the key and never echoes its value. Valid keys are unchanged, including the `json` alias in `[sources]`.
|
|
19
|
+
- `sentiel scan` shows only the exception class name in the `triage failed` warning.
|
|
20
|
+
|
|
21
|
+
### Fixed
|
|
22
|
+
- Rule titles, MITRE tags, notes and AI summaries from third parties are stripped of terminal control and bidi characters before display in `alerts`, `show`, `report`, `rules list` and `rules test`.
|
|
23
|
+
- `rules test` with a non-UTF-8 rule or events file prints a short error instead of a traceback.
|
|
24
|
+
- `demo` and `bench` close the database connection on errors, and `bench --out` is written even if a later model fails.
|
|
25
|
+
|
|
26
|
+
## [0.4.0]
|
|
27
|
+
|
|
28
|
+
### Added
|
|
29
|
+
- Database migrations (`PRAGMA user_version`), applied automatically and atomically when the database is opened. A database newer than the installed Sentiel is refused and left untouched.
|
|
30
|
+
- Alert status and notes: `sentiel set <id> open|ack|closed|false_positive [--note TEXT]`, and `sentiel alerts --all`. Only a human changes status. Closed, false-positive and suppressed alerts are never triaged or notified.
|
|
31
|
+
- `[[suppress]]` entries in the config (`rule_id`, `field`, `equals`, `reason`), and `sentiel set <id> false_positive --suppress FIELD` to print one. Only the first event of an alert is checked.
|
|
32
|
+
- Alert grouping with `group_window` (default 3600 seconds, `0` disables). A grown alert is not notified again.
|
|
33
|
+
- Daily digest: `digest_hour` for `sentiel watch`, and `sentiel digest` to send it now.
|
|
34
|
+
|
|
35
|
+
### Changed
|
|
36
|
+
- Invalid config makes every command exit with code 2 and a short message, without a traceback or the offending value.
|
|
37
|
+
|
|
38
|
+
## [0.3.0]
|
|
39
|
+
|
|
40
|
+
### Added
|
|
41
|
+
- Your own rules with `rules_dirs` and `disabled_rules`. Rules that do not convert, or that target a product Sentiel does not collect, are skipped with one summary warning per cycle.
|
|
42
|
+
- Sigma correlation rules (`event_count`, `gte`, one `group-by`). The two brute-force detections are now rule files (`win_logon_bruteforce.yml`, `lnx_ssh_bruteforce.yml`); alert ids are unchanged.
|
|
43
|
+
- Regex text-log sources with `[[sources.files]]`.
|
|
44
|
+
- `sentiel rules list [--skipped]` and `sentiel rules test RULE.yml EVENTS.jsonl`.
|
|
45
|
+
|
|
46
|
+
### Fixed
|
|
47
|
+
- Events with an out-of-range timestamp no longer break the scan cycle: the line is skipped.
|
|
48
|
+
- A rule that fails at run time becomes a warning instead of stopping detection.
|
|
49
|
+
- Text logs are read at most 32 MiB per cycle and split only on `\n`, so a log line cannot forge events with other separators.
|
|
50
|
+
|
|
51
|
+
## [0.2.0]
|
|
52
|
+
|
|
53
|
+
### Added
|
|
54
|
+
- LLM backoff (30 s up to 1800 s) when the model is down, with a hint to force a retry using `sentiel triage`.
|
|
55
|
+
- Concurrent triage with `triage_concurrency`.
|
|
56
|
+
- Notification per channel: one failing channel does not block the others.
|
|
57
|
+
- `sentiel doctor` (database, rules, sources, model, notifications, heartbeat) and a heartbeat on every scan.
|
|
58
|
+
- Services: `contrib/sentiel.service` (systemd) and `contrib/sentiel-task.xml` (Windows Task Scheduler).
|
|
59
|
+
|
|
60
|
+
### Changed
|
|
61
|
+
- Rules are cached by path and modification time.
|
|
62
|
+
|
|
63
|
+
## [0.1.1]
|
|
64
|
+
|
|
65
|
+
### Fixed
|
|
66
|
+
- Restored the `demo`, `bench`, `chat` and `version` commands.
|
|
67
|
+
- `auth.log` dates across the new year, including 29 February. An invalid date skips the line instead of failing the whole scan.
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
# Contributing to Sentiel
|
|
2
|
+
|
|
3
|
+
Thanks for helping. Sentiel is small on purpose, so the best contributions are usually a new detection rule, a false-positive fix or a clear bug report.
|
|
4
|
+
|
|
5
|
+
## Setup
|
|
6
|
+
|
|
7
|
+
Python 3.11 or newer.
|
|
8
|
+
|
|
9
|
+
Linux and macOS:
|
|
10
|
+
|
|
11
|
+
```
|
|
12
|
+
git clone https://github.com/shinitihm/Sentiel && cd Sentiel
|
|
13
|
+
python -m venv .venv && source .venv/bin/activate
|
|
14
|
+
pip install -e ".[dev]"
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
Windows:
|
|
18
|
+
|
|
19
|
+
```
|
|
20
|
+
git clone https://github.com/shinitihm/Sentiel; cd Sentiel
|
|
21
|
+
python -m venv .venv; .venv\Scripts\activate
|
|
22
|
+
pip install -e ".[dev]"
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Before opening a pull request:
|
|
26
|
+
|
|
27
|
+
```
|
|
28
|
+
pytest -q
|
|
29
|
+
ruff check .
|
|
30
|
+
sentiel demo --no-ai # must still show 7 alerts
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Ruff uses 120 columns. Tests that need a real LLM (`tests/test_llm_live.py`) are skipped unless you opt in; the rest run offline.
|
|
34
|
+
|
|
35
|
+
## Adding a rule
|
|
36
|
+
|
|
37
|
+
Sentiel reads Sigma rules. You do not need to write Python.
|
|
38
|
+
|
|
39
|
+
### A rule of your own, or a rule to propose
|
|
40
|
+
|
|
41
|
+
1. Write the Sigma rule as a `.yml` file. The `id` must be a UUID. In a Sigma rule, `logsource.product` is `windows`, `linux` or omitted. A rule for any other product is skipped with a warning ("product not collected"). A rule **without** `logsource.product` matches generic events (source `json`). `sentiel/rules/win_run_key_persistence.yml` is a Windows example. This is a generic one:
|
|
42
|
+
|
|
43
|
+
```yaml
|
|
44
|
+
title: Example Backup Tool Run
|
|
45
|
+
id: 6f1c1d52-3a0e-4b7e-9d0f-0a1b2c3d4e5f
|
|
46
|
+
logsource:
|
|
47
|
+
category: process_creation
|
|
48
|
+
detection:
|
|
49
|
+
selection:
|
|
50
|
+
Image|endswith: '\backup-tool.exe'
|
|
51
|
+
condition: selection
|
|
52
|
+
level: medium
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
2. Write a `.jsonl` file with a few events, one JSON object per line. For a rule without a product, plain objects work: `ts` (ISO 8601 with offset), `host` and the event fields at the top level. Include at least one event that should match and, ideally, one that should not:
|
|
56
|
+
|
|
57
|
+
```
|
|
58
|
+
{"ts": "2026-10-09T08:00:00+00:00", "host": "PC1", "Image": "C:\\Tools\\backup-tool.exe"}
|
|
59
|
+
{"ts": "2026-10-09T08:05:00+00:00", "host": "PC1", "Image": "C:\\Windows\\notepad.exe"}
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
For a rule with `product: windows` or `product: linux`, each line needs the full shape `{"ts": ..., "source": "windows", "host": ..., "fields": {...}}`, with `source` equal to the rule's product. `sentiel/demo/events.jsonl` has real examples.
|
|
63
|
+
3. Check it:
|
|
64
|
+
|
|
65
|
+
```
|
|
66
|
+
sentiel rules test my_rule.yml events.jsonl
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
The rule file must be self-contained. The command prints what the rule would alert on (here, one match) and exits 1 when nothing matches.
|
|
70
|
+
4. To use it locally, put the `.yml` in a folder listed in `rules_dirs` in `sentiel.toml`, then run `sentiel rules list` to confirm it loaded without warnings (`sentiel rules list --skipped` shows load warnings).
|
|
71
|
+
|
|
72
|
+
Correlation rules support `event_count` with `gte` and exactly one `group-by`. See `sentiel/rules/lnx_ssh_bruteforce.yml`.
|
|
73
|
+
|
|
74
|
+
### A built-in rule (pull request)
|
|
75
|
+
|
|
76
|
+
1. Add the `.yml` to `sentiel/rules/`.
|
|
77
|
+
2. Add events that trigger it to the demo data (`sentiel/demo/events.jsonl`, or `sentiel/demo/auth_log.txt` for Linux `auth.log`).
|
|
78
|
+
3. Add the rule `id` to `sentiel/demo/expected.json` with the attack name, expected severity and MITRE techniques. `tests/test_detect.py` checks that the demo detects exactly the rules listed there, and the demo must keep working with `--no-ai`.
|
|
79
|
+
4. Run `pytest -q` and `ruff check .`.
|
|
80
|
+
|
|
81
|
+
## Style
|
|
82
|
+
|
|
83
|
+
- YAGNI. Prefer the standard library, a few clear lines and no new runtime dependency. A new dependency needs a strong reason.
|
|
84
|
+
- Mark deliberate shortcuts with a `# ponytail:` comment saying what was skipped and why, so they can be found later.
|
|
85
|
+
- Match the style of the file you are editing. Keep each file's line endings (`sentiel/cli.py` uses CRLF).
|
|
86
|
+
- Add a test with every behavior change. Tests are offline and must not call a real model.
|
|
87
|
+
- Do not print secrets or config values in errors. Name the offending key, never its value.
|
|
88
|
+
|
|
89
|
+
## Principles that do not change
|
|
90
|
+
|
|
91
|
+
A pull request that breaks one of these will not be merged.
|
|
92
|
+
|
|
93
|
+
- The model never decides which alerts exist, and it cannot lower an alert's effective severity (`max(rule level, AI severity)`).
|
|
94
|
+
- Logs are hostile input. They reach the model only inside an `<untrusted_log_data>` block, and the model has no tools. Text from logs and third-party rules is sanitised before it reaches the terminal.
|
|
95
|
+
- No alert is lost. Each stage writes to SQLite before the next, and a pending alert is retried.
|
|
96
|
+
- A broken source or rule becomes a warning, not a crash.
|
|
97
|
+
- Status changes and suppressions come only from a human (`sentiel set`, `[[suppress]]` in the config).
|
|
98
|
+
- Secrets are never logged or echoed. Data leaves the machine only with a non-local model.
|
|
99
|
+
|
|
100
|
+
## Reporting bugs and rule issues
|
|
101
|
+
|
|
102
|
+
Use the issue templates: "New rule" and "False positive". Do not paste sensitive logs. For vulnerabilities, follow [SECURITY.md](SECURITY.md) and do not open a public issue.
|
|
103
|
+
|
|
104
|
+
## Releasing (maintainer only)
|
|
105
|
+
|
|
106
|
+
Releases are published to PyPI by `.github/workflows/release.yml` when a `vX.Y.Z` tag is pushed. It uses PyPI trusted publishing, so there is no PyPI token anywhere. The workflow checks that the tag equals `sentiel.__version__`, builds the sdist and wheel, runs `scripts/check_dist.py`, and publishes from the `pypi` environment.
|
|
107
|
+
|
|
108
|
+
One-time setup:
|
|
109
|
+
|
|
110
|
+
1. On PyPI, create a *pending publisher* (Account settings, Publishing): project `sentiel`, repository `shinitihm/Sentiel`, workflow `release.yml`, environment `pypi`. Keep the workflow and environment fields filled in, so the publisher cannot be used from another workflow.
|
|
111
|
+
2. On GitHub, create the environment `pypi` (Settings, Environments). Add required reviewers, and restrict deployment to tags matching `v*` (deployment branches and tags rule).
|
|
112
|
+
3. Add a tag ruleset (or tag protection) for `v*` so that only maintainers can create those tags. Anyone who can push a `v*` tag can start a release.
|
|
113
|
+
|
|
114
|
+
Each release:
|
|
115
|
+
|
|
116
|
+
1. Make sure CI is green on the exact commit you will tag. The release workflow does **not** run `pytest`; it only builds, checks the distribution and publishes.
|
|
117
|
+
2. Set the version in `sentiel/__init__.py` (the only place; `pyproject.toml` reads it).
|
|
118
|
+
3. Update `CHANGELOG.md`: put the release date on the new version heading (for 1.0.0, replace "unreleased (tag v1.0.0 pending)" with the date).
|
|
119
|
+
4. Check that the README (the page PyPI shows) describes the state of this release: roadmap, install steps and links. A PyPI page cannot be changed for a published version. Then tag and push only that tag:
|
|
120
|
+
|
|
121
|
+
```
|
|
122
|
+
git tag vX.Y.Z
|
|
123
|
+
git push origin vX.Y.Z
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
5. Approve the `pypi` deployment on GitHub, then check the project page on PyPI and try `pipx install sentiel` in a clean environment.
|
|
127
|
+
|
|
128
|
+
Notes:
|
|
129
|
+
|
|
130
|
+
- CI also runs on Python 3.14 as informational only (`continue-on-error`), because some native dependencies may lack 3.14 wheels. The README does not promise 3.14 until that job is green.
|
|
131
|
+
- The PyPI badge in the README is added only after the first publication.
|
sentiel-1.0.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 shinitihm
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
sentiel-1.0.0/PKG-INFO
ADDED
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: sentiel
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Personal AI-powered SOC: Sigma detection + LLM triage with any model
|
|
5
|
+
Project-URL: Homepage, https://github.com/shinitihm/Sentiel
|
|
6
|
+
Project-URL: Issues, https://github.com/shinitihm/Sentiel/issues
|
|
7
|
+
Project-URL: Changelog, https://github.com/shinitihm/Sentiel/blob/main/CHANGELOG.md
|
|
8
|
+
Author: shinitihm
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: detection,llm,security,siem,sigma,soc,triage
|
|
12
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: Information Technology
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
17
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
23
|
+
Classifier: Topic :: Security
|
|
24
|
+
Requires-Python: >=3.11
|
|
25
|
+
Requires-Dist: httpx>=0.27
|
|
26
|
+
Requires-Dist: litellm>=1.50
|
|
27
|
+
Requires-Dist: pydantic>=2.7
|
|
28
|
+
Requires-Dist: pysigma-backend-sqlite>=1.0
|
|
29
|
+
Requires-Dist: pysigma>=0.11
|
|
30
|
+
Requires-Dist: pywin32>=306; sys_platform == 'win32'
|
|
31
|
+
Requires-Dist: rich>=13
|
|
32
|
+
Requires-Dist: tenacity>=8
|
|
33
|
+
Requires-Dist: typer>=0.12
|
|
34
|
+
Provides-Extra: dev
|
|
35
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
36
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
37
|
+
Description-Content-Type: text/markdown
|
|
38
|
+
|
|
39
|
+
# Sentiel
|
|
40
|
+
|
|
41
|
+

|
|
42
|
+

|
|
43
|
+

|
|
44
|
+

|
|
45
|
+

|
|
46
|
+

|
|
47
|
+

|
|
48
|
+
|
|
49
|
+
[Português](https://github.com/shinitihm/Sentiel/blob/main/README.pt-BR.md)
|
|
50
|
+
|
|
51
|
+
> A personal SOC with AI triage that works with any LLM. Sigma rules detect attacks in your own logs, and a language model explains and prioritises each alert. The model never decides which alerts exist, and it cannot be talked into downgrading one.
|
|
52
|
+
|
|
53
|
+
### Improvements and roadmap
|
|
54
|
+
|
|
55
|
+
The project is still under development. The next releases follow this roadmap:
|
|
56
|
+
|
|
57
|
+
- [x] v0.1.1: CLI repair (`demo`, `bench`, `chat`, `version`) and `auth.log` dates across the new year
|
|
58
|
+
- [x] v0.2: orchestration (LLM backoff, concurrent triage, `sentiel doctor`, heartbeat, systemd and Task Scheduler services)
|
|
59
|
+
- [x] v0.3: data-driven extension (your own rules with `rules_dirs`/`disabled_rules`, Sigma correlations, regex sources, `sentiel rules list/test`)
|
|
60
|
+
- [x] v0.4: database migrations, alert status and notes (`sentiel set`), suppressions, grouping and a daily digest
|
|
61
|
+
- v1.0:
|
|
62
|
+
- [x] strict config, [CONTRIBUTING](https://github.com/shinitihm/Sentiel/blob/main/CONTRIBUTING.md), [SECURITY](https://github.com/shinitihm/Sentiel/blob/main/SECURITY.md), [CHANGELOG](https://github.com/shinitihm/Sentiel/blob/main/CHANGELOG.md), `sentiel bench --readme`, release workflow and CI with an informational Python 3.14 job
|
|
63
|
+
- [x] v1.0 release (PyPI)
|
|
64
|
+
|
|
65
|
+
## 💻 Prerequisites
|
|
66
|
+
|
|
67
|
+
Before you start, make sure you meet these requirements:
|
|
68
|
+
|
|
69
|
+
- Python **3.11 or newer**.
|
|
70
|
+
- Windows, Linux or macOS. On Windows, reading the **Security** log requires an administrator shell. Without it, Sentiel says so and continues with the other sources.
|
|
71
|
+
- Optional, for AI triage: [Ollama](https://ollama.com) with a local model, or an API key for any provider supported by [LiteLLM](https://github.com/BerriAI/litellm) (Anthropic, OpenAI, Gemini and 100+ more).
|
|
72
|
+
|
|
73
|
+
## 🚀 Installing Sentiel
|
|
74
|
+
|
|
75
|
+
Install it with [pipx](https://pipx.pypa.io):
|
|
76
|
+
|
|
77
|
+
```
|
|
78
|
+
pipx install sentiel
|
|
79
|
+
sentiel doctor
|
|
80
|
+
sentiel demo --no-ai
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
To work on the code, install from source:
|
|
84
|
+
|
|
85
|
+
Linux and macOS:
|
|
86
|
+
|
|
87
|
+
```
|
|
88
|
+
git clone https://github.com/shinitihm/Sentiel && cd Sentiel
|
|
89
|
+
python -m venv .venv && source .venv/bin/activate
|
|
90
|
+
pip install .
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Windows:
|
|
94
|
+
|
|
95
|
+
```
|
|
96
|
+
git clone https://github.com/shinitihm/Sentiel; cd Sentiel
|
|
97
|
+
python -m venv .venv; .venv\Scripts\activate
|
|
98
|
+
pip install .
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
## ☕ Using Sentiel
|
|
102
|
+
|
|
103
|
+
To use Sentiel, follow these steps:
|
|
104
|
+
|
|
105
|
+
```
|
|
106
|
+
sentiel demo --no-ai # 7 simulated attacks, detection only, offline
|
|
107
|
+
sentiel demo -m ollama/llama3.1:8b # the same, with local AI triage
|
|
108
|
+
|
|
109
|
+
sentiel doctor # checks config, database, rules, sources, model and notifications
|
|
110
|
+
sentiel scan # one pass over your logs
|
|
111
|
+
sentiel watch # continuous monitoring
|
|
112
|
+
sentiel alerts / show 3 / report # browse alerts (open and ack; `alerts --all` shows the rest)
|
|
113
|
+
sentiel set 3 ack --note "looking" # open | ack | closed | false_positive
|
|
114
|
+
sentiel digest # last-24h summary, sent now
|
|
115
|
+
sentiel chat "which IPs failed to log in most?"
|
|
116
|
+
sentiel rules list --skipped # loaded rules and warnings
|
|
117
|
+
sentiel rules test my_rule.yml events.jsonl
|
|
118
|
+
sentiel bench ollama/llama3.1:8b,anthropic/claude-sonnet-5-5
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
To configure it, get the example config (the pipx package does not include it), then edit the model, sources and notifications. If you cloned the repository, just copy it with `cp sentiel.example.toml sentiel.toml`.
|
|
122
|
+
|
|
123
|
+
Windows (PowerShell):
|
|
124
|
+
|
|
125
|
+
```
|
|
126
|
+
Invoke-WebRequest https://raw.githubusercontent.com/shinitihm/Sentiel/main/sentiel.example.toml -OutFile sentiel.toml
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
Linux and macOS:
|
|
130
|
+
|
|
131
|
+
```
|
|
132
|
+
curl -o sentiel.toml https://raw.githubusercontent.com/shinitihm/Sentiel/main/sentiel.example.toml
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
To run it as a service, use `contrib/sentiel.service` on Linux (systemd) or `contrib/sentiel-task.xml` on Windows (Task Scheduler). The steps are in each file's header. For Sysmon events on Windows, see [contrib/sysmon.md](https://github.com/shinitihm/Sentiel/blob/main/contrib/sysmon.md). LLM provider keys come only from the environment: use `/etc/sentiel/env` on Linux and machine-level variables on Windows.
|
|
136
|
+
|
|
137
|
+
### Model benchmark
|
|
138
|
+
|
|
139
|
+
`sentiel bench MODELS --readme` runs the demo attacks through each model and refreshes the table below (`--readme-file PATH` picks another file). It needs the two markers and fails without touching the file if they are missing.
|
|
140
|
+
|
|
141
|
+
<!-- BENCH:START -->
|
|
142
|
+
| Model | Severity | MITRE recall | Valid JSON 1st try | Prompt injection | Avg latency | Cost |
|
|
143
|
+
|---|---|---|---|---|---|---|
|
|
144
|
+
| not run yet | - | - | - | - | - | - |
|
|
145
|
+
<!-- BENCH:END -->
|
|
146
|
+
|
|
147
|
+
### Extending Sentiel
|
|
148
|
+
|
|
149
|
+
- **Your own rules:** `rules_dirs = ["~/.sentiel/rules"]` loads extra Sigma rules. For a SigmaHQ clone, point it at its `rules/` folder. `disabled_rules` turns rules off. Rules that do not convert, or that target a product Sentiel does not collect, are skipped with one summary warning per cycle (`sentiel rules list --skipped`).
|
|
150
|
+
- **Correlation:** Sigma correlation rules with `event_count`, `gte` and a single `group-by`. See `sentiel/rules/lnx_ssh_bruteforce.yml`.
|
|
151
|
+
- **Any text log:** `[[sources.files]]` with a regex. Named groups become Sigma fields, and `ts` and `host` are special groups. Keep the regex linear: log lines are attacker-controlled and Python's `re` has no timeout.
|
|
152
|
+
|
|
153
|
+
### Alert lifecycle
|
|
154
|
+
|
|
155
|
+
- **Status and notes:** `sentiel set <id> <status> [--note TEXT]` with `open`, `ack`, `closed` or `false_positive` (note up to 1000 characters). `suppressed` exists but only comes from the config file. `sentiel alerts` lists `open` and `ack`; `--all` also shows `closed`, `false_positive` and `suppressed`. Only humans change status: the model never does. Closed, false-positive and suppressed alerts are never triaged or notified.
|
|
156
|
+
- **Suppressions:** `sentiel set <id> false_positive --suppress FIELD` prints a `[[suppress]]` block (`rule_id`, `field`, `equals`, `reason`) for you to paste into the config. Nothing is edited for you, and the model never creates suppressions. Only the **first** event of an alert is checked, so a correlated burst that opens with a suppressed event is suppressed as a whole. See `sentiel.example.toml`.
|
|
157
|
+
- **Grouping:** `group_window` (default 3600 seconds, `0` turns it off) merges repeats of the same rule, host and group into one alert whose `count` grows. An alert that was already notified is **not** notified again when it grows: the `count` shows in `sentiel show` and in the digest.
|
|
158
|
+
- **Daily digest:** `digest_hour = 9` (local hour, 0-23) makes `sentiel watch` send a 24-hour summary once a day to your notification channels. `sentiel digest` sends it now, or prints it when no channel is set. The digest has counts and rule titles, never raw logs. The digest uses the event time (`last_ts`), so old logs ingested now may not show up in it.
|
|
159
|
+
- **Database upgrades:** the schema is migrated automatically (`PRAGMA user_version`) when the database is opened, and your alerts and logs are kept. Copy `sentiel.db` before upgrading Sentiel. A database newer than the installed Sentiel is refused with a clear message and left untouched.
|
|
160
|
+
- **Invalid config:** every command exits with code 2 and a short message, without a traceback and without echoing the offending value. Exception: `sentiel doctor` exits 1 and shows FAIL in its table.
|
|
161
|
+
|
|
162
|
+
### Security by design
|
|
163
|
+
|
|
164
|
+
- **Prompt injection cannot hide an alert:** effective severity is `max(rule level, AI severity)`. If the AI calls a `high` rule "benign", the alert shows as **CONTESTED** and is still notified.
|
|
165
|
+
- **Logs are data, not instructions:** events reach the model in an `<untrusted_log_data>` block, and the model has no tools.
|
|
166
|
+
- **Read-only chat:** the database is opened with `mode=ro`, a SQLite authorizer allows only `SELECT` on `logs`/`alerts`, and queries have time and row limits.
|
|
167
|
+
- **No alert is lost:** every stage writes to SQLite before the next. If the model is down, alerts stay pending and are retried.
|
|
168
|
+
|
|
169
|
+
## 📫 Contributing to Sentiel
|
|
170
|
+
|
|
171
|
+
To contribute to Sentiel, follow these steps:
|
|
172
|
+
|
|
173
|
+
1. Fork this repository.
|
|
174
|
+
2. Create a branch: `git checkout -b <branch_name>`.
|
|
175
|
+
3. Make your changes, run `pip install -e ".[dev]"`, `pytest -q` and `ruff check .`, then commit them: `git commit -m '<commit_message>'`.
|
|
176
|
+
4. Push to your fork: `git push origin <branch_name>`.
|
|
177
|
+
5. Open a pull request.
|
|
178
|
+
|
|
179
|
+
The full guide is in [CONTRIBUTING.md](https://github.com/shinitihm/Sentiel/blob/main/CONTRIBUTING.md): how to add a rule with only a YAML file and a `.jsonl` file, style and principles. To report a vulnerability, see [SECURITY.md](https://github.com/shinitihm/Sentiel/blob/main/SECURITY.md). Changes per version are in [CHANGELOG.md](https://github.com/shinitihm/Sentiel/blob/main/CHANGELOG.md).
|
|
180
|
+
|
|
181
|
+
See also GitHub's documentation on [creating a pull request](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request).
|
|
182
|
+
|
|
183
|
+
## 🤝 Contributors
|
|
184
|
+
|
|
185
|
+
Thanks to the following people who have contributed to this project:
|
|
186
|
+
|
|
187
|
+
<table>
|
|
188
|
+
<tr>
|
|
189
|
+
<td align="center">
|
|
190
|
+
<a href="https://github.com/shinitihm" title="shinitihm on GitHub">
|
|
191
|
+
<img src="https://github.com/shinitihm.png" width="100px;" alt="shinitihm's photo on GitHub"/><br>
|
|
192
|
+
<sub>
|
|
193
|
+
<b>shinitihm</b>
|
|
194
|
+
</sub>
|
|
195
|
+
</a>
|
|
196
|
+
</td>
|
|
197
|
+
</tr>
|
|
198
|
+
</table>
|
|
199
|
+
|
|
200
|
+
## 😄 Become a contributor
|
|
201
|
+
|
|
202
|
+
Want to be part of this project? Open an issue (there are templates for new rules and false positives) or a pull request, and read [CONTRIBUTING.md](https://github.com/shinitihm/Sentiel/blob/main/CONTRIBUTING.md) first.
|
|
203
|
+
|
|
204
|
+
## 📝 License
|
|
205
|
+
|
|
206
|
+
This project is under the MIT license. See the [LICENSE](https://github.com/shinitihm/Sentiel/blob/main/LICENSE) file for details.
|