syslog2cef 0.1.4__tar.gz → 0.1.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {syslog2cef-0.1.4/syslog2cef.egg-info → syslog2cef-0.1.5}/PKG-INFO +119 -1
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/README.md +116 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/pyproject.toml +12 -1
- {syslog2cef-0.1.4 → syslog2cef-0.1.5/syslog2cef.egg-info}/PKG-INFO +119 -1
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslog2cef.egg-info/SOURCES.txt +4 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslog2cef.egg-info/requires.txt +3 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/__init__.py +4 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/api.py +3 -3
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/cef.py +1 -2
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/cli.py +155 -14
- syslog2cef-0.1.5/syslogcef/custom.py +305 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/normalizer.py +2 -2
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/parsers.py +25 -3
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/utils.py +1 -2
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/tests/test_adaptive.py +0 -1
- syslog2cef-0.1.5/tests/test_cli.py +178 -0
- syslog2cef-0.1.5/tests/test_custom_parsers.py +290 -0
- syslog2cef-0.1.5/tests/test_fuzz_properties.py +154 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/tests/test_validation.py +1 -1
- syslog2cef-0.1.4/tests/test_cli.py +0 -45
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/LICENSE +0 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/setup.cfg +0 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslog2cef.egg-info/dependency_links.txt +0 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslog2cef.egg-info/entry_points.txt +0 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslog2cef.egg-info/top_level.txt +0 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/__main__.py +0 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/adaptive.py +0 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/mappings/__init__.py +0 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/mappings/cisco_asa.json +0 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/mappings/cisco_ios.json +0 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/mappings/f5.json +0 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/mappings/fortinet.json +0 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/mappings/linux.json +0 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/mappings/sophos.json +0 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/mappings/vmware.json +0 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/net.py +0 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/validation.py +0 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/tests/test_cef.py +0 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/tests/test_net.py +0 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/tests/test_normalizer.py +0 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/tests/test_parsers.py +0 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/tests/test_parsers_extra.py +0 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/tests/test_pipeline.py +0 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/tests/test_real_world.py +0 -0
- {syslog2cef-0.1.4 → syslog2cef-0.1.5}/tests/test_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: syslog2cef
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.5
|
|
4
4
|
Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
|
|
5
5
|
Author-email: Tamir Suliman <allamiro@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/allamiro/syslogcef
|
|
@@ -17,6 +17,8 @@ Provides-Extra: test
|
|
|
17
17
|
Requires-Dist: pytest>=7; extra == "test"
|
|
18
18
|
Provides-Extra: kafka
|
|
19
19
|
Requires-Dist: kafka-python>=2.0; extra == "kafka"
|
|
20
|
+
Provides-Extra: fuzz
|
|
21
|
+
Requires-Dist: hypothesis>=6; extra == "fuzz"
|
|
20
22
|
Dynamic: license-file
|
|
21
23
|
|
|
22
24
|
<p align="center">
|
|
@@ -199,6 +201,55 @@ Field templates that reference missing keys resolve to an empty string
|
|
|
199
201
|
rather than failing the event. See [docs/cef_fields.md](docs/cef_fields.md)
|
|
200
202
|
for the full CEF extension dictionary.
|
|
201
203
|
|
|
204
|
+
## Custom Parsers
|
|
205
|
+
|
|
206
|
+
When a device emits a format no built-in parser handles, add your own
|
|
207
|
+
detection regexes the same way you add mappings — no code required:
|
|
208
|
+
|
|
209
|
+
```bash
|
|
210
|
+
syslogcef acme.log --patterns /etc/syslogcef/patterns.json
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
```json
|
|
214
|
+
{
|
|
215
|
+
"patterns": [
|
|
216
|
+
{
|
|
217
|
+
"name": "acme_fw",
|
|
218
|
+
"regex": "^ACME (?P<ts>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}) (?P<host>\\S+) (?P<app>\\w+)\\[(?P<pid>\\d+)\\]: (?P<msg>.*)$",
|
|
219
|
+
"timestamp_format": "%Y-%m-%d %H:%M:%S",
|
|
220
|
+
"priority": "after"
|
|
221
|
+
}
|
|
222
|
+
]
|
|
223
|
+
}
|
|
224
|
+
```
|
|
225
|
+
|
|
226
|
+
- Named groups map to event fields: `pri`, `host`, `app`, `pid`,
|
|
227
|
+
`msgid`, `msg`, and `ts`.
|
|
228
|
+
- A `ts` group requires `timestamp_format`: a strptime format,
|
|
229
|
+
`iso8601`, or `epoch`. Yearless formats get the same year-rollover
|
|
230
|
+
inference as the built-in parsers; an unparseable timestamp never
|
|
231
|
+
drops the event.
|
|
232
|
+
- `priority` is `"after"` (default: tried after the built-ins, before
|
|
233
|
+
the adaptive fallback) or `"before"` (tried first, overriding
|
|
234
|
+
built-ins).
|
|
235
|
+
- Pattern names work as `--mode` values, and files are validated at
|
|
236
|
+
startup — bad regexes, unknown groups, or duplicate names fail with
|
|
237
|
+
a clear message instead of mid-stream.
|
|
238
|
+
- Per-service-instance patterns: set
|
|
239
|
+
`EXTRA_ARGS=--patterns /etc/syslogcef/patterns.json` in the
|
|
240
|
+
instance's conf.d file.
|
|
241
|
+
- Your regexes run against untrusted log content — keep them anchored
|
|
242
|
+
and avoid nested quantifiers (ReDoS).
|
|
243
|
+
|
|
244
|
+
From Python, register full parser functions instead:
|
|
245
|
+
|
|
246
|
+
```python
|
|
247
|
+
from syslogcef import register_parser, load_patterns
|
|
248
|
+
|
|
249
|
+
load_patterns("patterns.json") # same file format
|
|
250
|
+
register_parser("marker", my_parse_fn) # fn(line) -> ParsedEvent | None
|
|
251
|
+
```
|
|
252
|
+
|
|
202
253
|
## Running as a Service
|
|
203
254
|
|
|
204
255
|
The RPM and Debian packages install a systemd unit and an environment
|
|
@@ -209,6 +260,9 @@ configuration in `/etc/conf.d/syslogcef`):
|
|
|
209
260
|
arguments for the converter.
|
|
210
261
|
- `syslogcef.service` — runs `syslogcef --tail` against the configured
|
|
211
262
|
input and appends CEF to the configured output.
|
|
263
|
+
- `syslogcef@.service` — template unit for running several independent
|
|
264
|
+
pipelines from `/etc/syslogcef/conf.d/` (see below).
|
|
265
|
+
- `/etc/logrotate.d/syslogcef` — daily rotation for flat `.cef` archives.
|
|
212
266
|
|
|
213
267
|
```bash
|
|
214
268
|
sudo dnf install syslogcef-*.rpm
|
|
@@ -216,6 +270,70 @@ sudo vi /etc/syslogcef/syslogcef.conf
|
|
|
216
270
|
sudo systemctl enable --now syslogcef
|
|
217
271
|
```
|
|
218
272
|
|
|
273
|
+
The environment file has three variables; everything else goes through
|
|
274
|
+
`EXTRA_ARGS`, which accepts any command line flag (`--mode`, `--mapping`,
|
|
275
|
+
`--listen`, `--send`, `--eps`, `--validate`, `--strict`,
|
|
276
|
+
`--multiprocess`, `--log-level`, ...):
|
|
277
|
+
|
|
278
|
+
```bash
|
|
279
|
+
# File(s) to follow for new syslog lines, separated by spaces.
|
|
280
|
+
INPUT_FILE=/var/log/messages
|
|
281
|
+
|
|
282
|
+
# File that converted CEF events are appended to.
|
|
283
|
+
OUTPUT_FILE=/var/log/syslogcef/events.cef
|
|
284
|
+
|
|
285
|
+
# Extra arguments; leave INPUT_FILE empty and use --listen for a
|
|
286
|
+
# network daemon: EXTRA_ARGS=--listen udp:514 --send tcp://siem:514
|
|
287
|
+
EXTRA_ARGS=
|
|
288
|
+
```
|
|
289
|
+
|
|
290
|
+
### Timed output files
|
|
291
|
+
|
|
292
|
+
`OUTPUT_FILE` (and `--output` generally) accepts strftime codes; the
|
|
293
|
+
file is reopened whenever the rendered path changes and parent
|
|
294
|
+
directories are created automatically:
|
|
295
|
+
|
|
296
|
+
```bash
|
|
297
|
+
# New file each hour, grouped in a directory per day:
|
|
298
|
+
OUTPUT_FILE=/var/log/syslogcef/%Y-%m-%d/events-%H.cef
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
Unsupported `%` codes are rejected at startup (use `%%` for a literal
|
|
302
|
+
percent). Keep templated outputs in a dated subdirectory as shown above:
|
|
303
|
+
the installed logrotate snippet rotates every flat `.cef` file directly
|
|
304
|
+
under `/var/log/syslogcef/`, and a templated file rendering flat there
|
|
305
|
+
would be rotated twice. Prefer a stable filename if a downstream
|
|
306
|
+
collector reads the archive — flat files are rotated daily by logrotate
|
|
307
|
+
instead.
|
|
308
|
+
|
|
309
|
+
### Multiple pipelines (one input per output)
|
|
310
|
+
|
|
311
|
+
To map specific inputs to specific outputs, run one instance of the
|
|
312
|
+
template unit per pipeline. Each instance reads its own file in
|
|
313
|
+
`/etc/syslogcef/conf.d/` (a commented `example.conf.sample` is
|
|
314
|
+
installed there) and has independent restart, logs, and options:
|
|
315
|
+
|
|
316
|
+
```bash
|
|
317
|
+
sudo cp /etc/syslogcef/conf.d/example.conf.sample /etc/syslogcef/conf.d/secure.conf
|
|
318
|
+
sudo cp /etc/syslogcef/conf.d/example.conf.sample /etc/syslogcef/conf.d/firewall.conf
|
|
319
|
+
sudo vi /etc/syslogcef/conf.d/secure.conf # /var/log/secure -> secure.cef
|
|
320
|
+
sudo vi /etc/syslogcef/conf.d/firewall.conf # --listen udp:514 --mode cisco_seq
|
|
321
|
+
sudo systemctl enable --now syslogcef@secure syslogcef@firewall
|
|
322
|
+
```
|
|
323
|
+
|
|
324
|
+
On Alpine, OpenRC gets the same result with symlinked services:
|
|
325
|
+
|
|
326
|
+
```bash
|
|
327
|
+
sudo ln -s syslogcef /etc/init.d/syslogcef.firewall
|
|
328
|
+
sudo cp /etc/conf.d/syslogcef /etc/conf.d/syslogcef.firewall
|
|
329
|
+
sudo vi /etc/conf.d/syslogcef.firewall # set a distinct INPUT_FILE and OUTPUT_FILE
|
|
330
|
+
sudo rc-update add syslogcef.firewall && sudo rc-service syslogcef.firewall start
|
|
331
|
+
```
|
|
332
|
+
|
|
333
|
+
Give every instance its own `INPUT_FILE` and `OUTPUT_FILE` — two
|
|
334
|
+
instances sharing them would process the same events twice and append
|
|
335
|
+
to the same file concurrently.
|
|
336
|
+
|
|
219
337
|
See [packaging/rpm/](packaging/rpm/) for the spec file and build
|
|
220
338
|
instructions, including GPG signing of the RPM.
|
|
221
339
|
|
|
@@ -178,6 +178,55 @@ Field templates that reference missing keys resolve to an empty string
|
|
|
178
178
|
rather than failing the event. See [docs/cef_fields.md](docs/cef_fields.md)
|
|
179
179
|
for the full CEF extension dictionary.
|
|
180
180
|
|
|
181
|
+
## Custom Parsers
|
|
182
|
+
|
|
183
|
+
When a device emits a format no built-in parser handles, add your own
|
|
184
|
+
detection regexes the same way you add mappings — no code required:
|
|
185
|
+
|
|
186
|
+
```bash
|
|
187
|
+
syslogcef acme.log --patterns /etc/syslogcef/patterns.json
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
```json
|
|
191
|
+
{
|
|
192
|
+
"patterns": [
|
|
193
|
+
{
|
|
194
|
+
"name": "acme_fw",
|
|
195
|
+
"regex": "^ACME (?P<ts>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}) (?P<host>\\S+) (?P<app>\\w+)\\[(?P<pid>\\d+)\\]: (?P<msg>.*)$",
|
|
196
|
+
"timestamp_format": "%Y-%m-%d %H:%M:%S",
|
|
197
|
+
"priority": "after"
|
|
198
|
+
}
|
|
199
|
+
]
|
|
200
|
+
}
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
- Named groups map to event fields: `pri`, `host`, `app`, `pid`,
|
|
204
|
+
`msgid`, `msg`, and `ts`.
|
|
205
|
+
- A `ts` group requires `timestamp_format`: a strptime format,
|
|
206
|
+
`iso8601`, or `epoch`. Yearless formats get the same year-rollover
|
|
207
|
+
inference as the built-in parsers; an unparseable timestamp never
|
|
208
|
+
drops the event.
|
|
209
|
+
- `priority` is `"after"` (default: tried after the built-ins, before
|
|
210
|
+
the adaptive fallback) or `"before"` (tried first, overriding
|
|
211
|
+
built-ins).
|
|
212
|
+
- Pattern names work as `--mode` values, and files are validated at
|
|
213
|
+
startup — bad regexes, unknown groups, or duplicate names fail with
|
|
214
|
+
a clear message instead of mid-stream.
|
|
215
|
+
- Per-service-instance patterns: set
|
|
216
|
+
`EXTRA_ARGS=--patterns /etc/syslogcef/patterns.json` in the
|
|
217
|
+
instance's conf.d file.
|
|
218
|
+
- Your regexes run against untrusted log content — keep them anchored
|
|
219
|
+
and avoid nested quantifiers (ReDoS).
|
|
220
|
+
|
|
221
|
+
From Python, register full parser functions instead:
|
|
222
|
+
|
|
223
|
+
```python
|
|
224
|
+
from syslogcef import register_parser, load_patterns
|
|
225
|
+
|
|
226
|
+
load_patterns("patterns.json") # same file format
|
|
227
|
+
register_parser("marker", my_parse_fn) # fn(line) -> ParsedEvent | None
|
|
228
|
+
```
|
|
229
|
+
|
|
181
230
|
## Running as a Service
|
|
182
231
|
|
|
183
232
|
The RPM and Debian packages install a systemd unit and an environment
|
|
@@ -188,6 +237,9 @@ configuration in `/etc/conf.d/syslogcef`):
|
|
|
188
237
|
arguments for the converter.
|
|
189
238
|
- `syslogcef.service` — runs `syslogcef --tail` against the configured
|
|
190
239
|
input and appends CEF to the configured output.
|
|
240
|
+
- `syslogcef@.service` — template unit for running several independent
|
|
241
|
+
pipelines from `/etc/syslogcef/conf.d/` (see below).
|
|
242
|
+
- `/etc/logrotate.d/syslogcef` — daily rotation for flat `.cef` archives.
|
|
191
243
|
|
|
192
244
|
```bash
|
|
193
245
|
sudo dnf install syslogcef-*.rpm
|
|
@@ -195,6 +247,70 @@ sudo vi /etc/syslogcef/syslogcef.conf
|
|
|
195
247
|
sudo systemctl enable --now syslogcef
|
|
196
248
|
```
|
|
197
249
|
|
|
250
|
+
The environment file has three variables; everything else goes through
|
|
251
|
+
`EXTRA_ARGS`, which accepts any command line flag (`--mode`, `--mapping`,
|
|
252
|
+
`--listen`, `--send`, `--eps`, `--validate`, `--strict`,
|
|
253
|
+
`--multiprocess`, `--log-level`, ...):
|
|
254
|
+
|
|
255
|
+
```bash
|
|
256
|
+
# File(s) to follow for new syslog lines, separated by spaces.
|
|
257
|
+
INPUT_FILE=/var/log/messages
|
|
258
|
+
|
|
259
|
+
# File that converted CEF events are appended to.
|
|
260
|
+
OUTPUT_FILE=/var/log/syslogcef/events.cef
|
|
261
|
+
|
|
262
|
+
# Extra arguments; leave INPUT_FILE empty and use --listen for a
|
|
263
|
+
# network daemon: EXTRA_ARGS=--listen udp:514 --send tcp://siem:514
|
|
264
|
+
EXTRA_ARGS=
|
|
265
|
+
```
|
|
266
|
+
|
|
267
|
+
### Timed output files
|
|
268
|
+
|
|
269
|
+
`OUTPUT_FILE` (and `--output` generally) accepts strftime codes; the
|
|
270
|
+
file is reopened whenever the rendered path changes and parent
|
|
271
|
+
directories are created automatically:
|
|
272
|
+
|
|
273
|
+
```bash
|
|
274
|
+
# New file each hour, grouped in a directory per day:
|
|
275
|
+
OUTPUT_FILE=/var/log/syslogcef/%Y-%m-%d/events-%H.cef
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
Unsupported `%` codes are rejected at startup (use `%%` for a literal
|
|
279
|
+
percent). Keep templated outputs in a dated subdirectory as shown above:
|
|
280
|
+
the installed logrotate snippet rotates every flat `.cef` file directly
|
|
281
|
+
under `/var/log/syslogcef/`, and a templated file rendering flat there
|
|
282
|
+
would be rotated twice. Prefer a stable filename if a downstream
|
|
283
|
+
collector reads the archive — flat files are rotated daily by logrotate
|
|
284
|
+
instead.
|
|
285
|
+
|
|
286
|
+
### Multiple pipelines (one input per output)
|
|
287
|
+
|
|
288
|
+
To map specific inputs to specific outputs, run one instance of the
|
|
289
|
+
template unit per pipeline. Each instance reads its own file in
|
|
290
|
+
`/etc/syslogcef/conf.d/` (a commented `example.conf.sample` is
|
|
291
|
+
installed there) and has independent restart, logs, and options:
|
|
292
|
+
|
|
293
|
+
```bash
|
|
294
|
+
sudo cp /etc/syslogcef/conf.d/example.conf.sample /etc/syslogcef/conf.d/secure.conf
|
|
295
|
+
sudo cp /etc/syslogcef/conf.d/example.conf.sample /etc/syslogcef/conf.d/firewall.conf
|
|
296
|
+
sudo vi /etc/syslogcef/conf.d/secure.conf # /var/log/secure -> secure.cef
|
|
297
|
+
sudo vi /etc/syslogcef/conf.d/firewall.conf # --listen udp:514 --mode cisco_seq
|
|
298
|
+
sudo systemctl enable --now syslogcef@secure syslogcef@firewall
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
On Alpine, OpenRC gets the same result with symlinked services:
|
|
302
|
+
|
|
303
|
+
```bash
|
|
304
|
+
sudo ln -s syslogcef /etc/init.d/syslogcef.firewall
|
|
305
|
+
sudo cp /etc/conf.d/syslogcef /etc/conf.d/syslogcef.firewall
|
|
306
|
+
sudo vi /etc/conf.d/syslogcef.firewall # set a distinct INPUT_FILE and OUTPUT_FILE
|
|
307
|
+
sudo rc-update add syslogcef.firewall && sudo rc-service syslogcef.firewall start
|
|
308
|
+
```
|
|
309
|
+
|
|
310
|
+
Give every instance its own `INPUT_FILE` and `OUTPUT_FILE` — two
|
|
311
|
+
instances sharing them would process the same events twice and append
|
|
312
|
+
to the same file concurrently.
|
|
313
|
+
|
|
198
314
|
See [packaging/rpm/](packaging/rpm/) for the spec file and build
|
|
199
315
|
instructions, including GPG signing of the RPM.
|
|
200
316
|
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "syslog2cef"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.5"
|
|
8
8
|
description = "Convert RFC3164/RFC5424 syslog lines into ArcSight CEF"
|
|
9
9
|
authors = [{name = "Tamir Suliman", email = "allamiro@gmail.com"}]
|
|
10
10
|
readme = "README.md"
|
|
@@ -29,6 +29,9 @@ test = [
|
|
|
29
29
|
kafka = [
|
|
30
30
|
"kafka-python>=2.0",
|
|
31
31
|
]
|
|
32
|
+
fuzz = [
|
|
33
|
+
"hypothesis>=6",
|
|
34
|
+
]
|
|
32
35
|
|
|
33
36
|
[project.scripts]
|
|
34
37
|
syslogcef = "syslogcef.cli:main"
|
|
@@ -43,3 +46,11 @@ syslogcef = ["mappings/*.json"]
|
|
|
43
46
|
[tool.pytest.ini_options]
|
|
44
47
|
addopts = "-ra"
|
|
45
48
|
pythonpath = ["."]
|
|
49
|
+
|
|
50
|
+
[tool.ruff]
|
|
51
|
+
target-version = "py39"
|
|
52
|
+
|
|
53
|
+
[tool.ruff.lint]
|
|
54
|
+
# Start conservative: syntax errors and pyflakes (undefined names,
|
|
55
|
+
# unused imports). Widen the selection as the codebase is cleaned up.
|
|
56
|
+
select = ["E9", "F"]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: syslog2cef
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.5
|
|
4
4
|
Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
|
|
5
5
|
Author-email: Tamir Suliman <allamiro@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/allamiro/syslogcef
|
|
@@ -17,6 +17,8 @@ Provides-Extra: test
|
|
|
17
17
|
Requires-Dist: pytest>=7; extra == "test"
|
|
18
18
|
Provides-Extra: kafka
|
|
19
19
|
Requires-Dist: kafka-python>=2.0; extra == "kafka"
|
|
20
|
+
Provides-Extra: fuzz
|
|
21
|
+
Requires-Dist: hypothesis>=6; extra == "fuzz"
|
|
20
22
|
Dynamic: license-file
|
|
21
23
|
|
|
22
24
|
<p align="center">
|
|
@@ -199,6 +201,55 @@ Field templates that reference missing keys resolve to an empty string
|
|
|
199
201
|
rather than failing the event. See [docs/cef_fields.md](docs/cef_fields.md)
|
|
200
202
|
for the full CEF extension dictionary.
|
|
201
203
|
|
|
204
|
+
## Custom Parsers
|
|
205
|
+
|
|
206
|
+
When a device emits a format no built-in parser handles, add your own
|
|
207
|
+
detection regexes the same way you add mappings — no code required:
|
|
208
|
+
|
|
209
|
+
```bash
|
|
210
|
+
syslogcef acme.log --patterns /etc/syslogcef/patterns.json
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
```json
|
|
214
|
+
{
|
|
215
|
+
"patterns": [
|
|
216
|
+
{
|
|
217
|
+
"name": "acme_fw",
|
|
218
|
+
"regex": "^ACME (?P<ts>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}) (?P<host>\\S+) (?P<app>\\w+)\\[(?P<pid>\\d+)\\]: (?P<msg>.*)$",
|
|
219
|
+
"timestamp_format": "%Y-%m-%d %H:%M:%S",
|
|
220
|
+
"priority": "after"
|
|
221
|
+
}
|
|
222
|
+
]
|
|
223
|
+
}
|
|
224
|
+
```
|
|
225
|
+
|
|
226
|
+
- Named groups map to event fields: `pri`, `host`, `app`, `pid`,
|
|
227
|
+
`msgid`, `msg`, and `ts`.
|
|
228
|
+
- A `ts` group requires `timestamp_format`: a strptime format,
|
|
229
|
+
`iso8601`, or `epoch`. Yearless formats get the same year-rollover
|
|
230
|
+
inference as the built-in parsers; an unparseable timestamp never
|
|
231
|
+
drops the event.
|
|
232
|
+
- `priority` is `"after"` (default: tried after the built-ins, before
|
|
233
|
+
the adaptive fallback) or `"before"` (tried first, overriding
|
|
234
|
+
built-ins).
|
|
235
|
+
- Pattern names work as `--mode` values, and files are validated at
|
|
236
|
+
startup — bad regexes, unknown groups, or duplicate names fail with
|
|
237
|
+
a clear message instead of mid-stream.
|
|
238
|
+
- Per-service-instance patterns: set
|
|
239
|
+
`EXTRA_ARGS=--patterns /etc/syslogcef/patterns.json` in the
|
|
240
|
+
instance's conf.d file.
|
|
241
|
+
- Your regexes run against untrusted log content — keep them anchored
|
|
242
|
+
and avoid nested quantifiers (ReDoS).
|
|
243
|
+
|
|
244
|
+
From Python, register full parser functions instead:
|
|
245
|
+
|
|
246
|
+
```python
|
|
247
|
+
from syslogcef import register_parser, load_patterns
|
|
248
|
+
|
|
249
|
+
load_patterns("patterns.json") # same file format
|
|
250
|
+
register_parser("marker", my_parse_fn) # fn(line) -> ParsedEvent | None
|
|
251
|
+
```
|
|
252
|
+
|
|
202
253
|
## Running as a Service
|
|
203
254
|
|
|
204
255
|
The RPM and Debian packages install a systemd unit and an environment
|
|
@@ -209,6 +260,9 @@ configuration in `/etc/conf.d/syslogcef`):
|
|
|
209
260
|
arguments for the converter.
|
|
210
261
|
- `syslogcef.service` — runs `syslogcef --tail` against the configured
|
|
211
262
|
input and appends CEF to the configured output.
|
|
263
|
+
- `syslogcef@.service` — template unit for running several independent
|
|
264
|
+
pipelines from `/etc/syslogcef/conf.d/` (see below).
|
|
265
|
+
- `/etc/logrotate.d/syslogcef` — daily rotation for flat `.cef` archives.
|
|
212
266
|
|
|
213
267
|
```bash
|
|
214
268
|
sudo dnf install syslogcef-*.rpm
|
|
@@ -216,6 +270,70 @@ sudo vi /etc/syslogcef/syslogcef.conf
|
|
|
216
270
|
sudo systemctl enable --now syslogcef
|
|
217
271
|
```
|
|
218
272
|
|
|
273
|
+
The environment file has three variables; everything else goes through
|
|
274
|
+
`EXTRA_ARGS`, which accepts any command line flag (`--mode`, `--mapping`,
|
|
275
|
+
`--listen`, `--send`, `--eps`, `--validate`, `--strict`,
|
|
276
|
+
`--multiprocess`, `--log-level`, ...):
|
|
277
|
+
|
|
278
|
+
```bash
|
|
279
|
+
# File(s) to follow for new syslog lines, separated by spaces.
|
|
280
|
+
INPUT_FILE=/var/log/messages
|
|
281
|
+
|
|
282
|
+
# File that converted CEF events are appended to.
|
|
283
|
+
OUTPUT_FILE=/var/log/syslogcef/events.cef
|
|
284
|
+
|
|
285
|
+
# Extra arguments; leave INPUT_FILE empty and use --listen for a
|
|
286
|
+
# network daemon: EXTRA_ARGS=--listen udp:514 --send tcp://siem:514
|
|
287
|
+
EXTRA_ARGS=
|
|
288
|
+
```
|
|
289
|
+
|
|
290
|
+
### Timed output files
|
|
291
|
+
|
|
292
|
+
`OUTPUT_FILE` (and `--output` generally) accepts strftime codes; the
|
|
293
|
+
file is reopened whenever the rendered path changes and parent
|
|
294
|
+
directories are created automatically:
|
|
295
|
+
|
|
296
|
+
```bash
|
|
297
|
+
# New file each hour, grouped in a directory per day:
|
|
298
|
+
OUTPUT_FILE=/var/log/syslogcef/%Y-%m-%d/events-%H.cef
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
Unsupported `%` codes are rejected at startup (use `%%` for a literal
|
|
302
|
+
percent). Keep templated outputs in a dated subdirectory as shown above:
|
|
303
|
+
the installed logrotate snippet rotates every flat `.cef` file directly
|
|
304
|
+
under `/var/log/syslogcef/`, and a templated file rendering flat there
|
|
305
|
+
would be rotated twice. Prefer a stable filename if a downstream
|
|
306
|
+
collector reads the archive — flat files are rotated daily by logrotate
|
|
307
|
+
instead.
|
|
308
|
+
|
|
309
|
+
### Multiple pipelines (one input per output)
|
|
310
|
+
|
|
311
|
+
To map specific inputs to specific outputs, run one instance of the
|
|
312
|
+
template unit per pipeline. Each instance reads its own file in
|
|
313
|
+
`/etc/syslogcef/conf.d/` (a commented `example.conf.sample` is
|
|
314
|
+
installed there) and has independent restart, logs, and options:
|
|
315
|
+
|
|
316
|
+
```bash
|
|
317
|
+
sudo cp /etc/syslogcef/conf.d/example.conf.sample /etc/syslogcef/conf.d/secure.conf
|
|
318
|
+
sudo cp /etc/syslogcef/conf.d/example.conf.sample /etc/syslogcef/conf.d/firewall.conf
|
|
319
|
+
sudo vi /etc/syslogcef/conf.d/secure.conf # /var/log/secure -> secure.cef
|
|
320
|
+
sudo vi /etc/syslogcef/conf.d/firewall.conf # --listen udp:514 --mode cisco_seq
|
|
321
|
+
sudo systemctl enable --now syslogcef@secure syslogcef@firewall
|
|
322
|
+
```
|
|
323
|
+
|
|
324
|
+
On Alpine, OpenRC gets the same result with symlinked services:
|
|
325
|
+
|
|
326
|
+
```bash
|
|
327
|
+
sudo ln -s syslogcef /etc/init.d/syslogcef.firewall
|
|
328
|
+
sudo cp /etc/conf.d/syslogcef /etc/conf.d/syslogcef.firewall
|
|
329
|
+
sudo vi /etc/conf.d/syslogcef.firewall # set a distinct INPUT_FILE and OUTPUT_FILE
|
|
330
|
+
sudo rc-update add syslogcef.firewall && sudo rc-service syslogcef.firewall start
|
|
331
|
+
```
|
|
332
|
+
|
|
333
|
+
Give every instance its own `INPUT_FILE` and `OUTPUT_FILE` — two
|
|
334
|
+
instances sharing them would process the same events twice and append
|
|
335
|
+
to the same file concurrently.
|
|
336
|
+
|
|
219
337
|
See [packaging/rpm/](packaging/rpm/) for the spec file and build
|
|
220
338
|
instructions, including GPG signing of the RPM.
|
|
221
339
|
|
|
@@ -7,6 +7,7 @@ pyproject.toml
|
|
|
7
7
|
./syslogcef/api.py
|
|
8
8
|
./syslogcef/cef.py
|
|
9
9
|
./syslogcef/cli.py
|
|
10
|
+
./syslogcef/custom.py
|
|
10
11
|
./syslogcef/net.py
|
|
11
12
|
./syslogcef/normalizer.py
|
|
12
13
|
./syslogcef/parsers.py
|
|
@@ -32,6 +33,7 @@ syslogcef/adaptive.py
|
|
|
32
33
|
syslogcef/api.py
|
|
33
34
|
syslogcef/cef.py
|
|
34
35
|
syslogcef/cli.py
|
|
36
|
+
syslogcef/custom.py
|
|
35
37
|
syslogcef/net.py
|
|
36
38
|
syslogcef/normalizer.py
|
|
37
39
|
syslogcef/parsers.py
|
|
@@ -48,6 +50,8 @@ syslogcef/mappings/vmware.json
|
|
|
48
50
|
tests/test_adaptive.py
|
|
49
51
|
tests/test_cef.py
|
|
50
52
|
tests/test_cli.py
|
|
53
|
+
tests/test_custom_parsers.py
|
|
54
|
+
tests/test_fuzz_properties.py
|
|
51
55
|
tests/test_net.py
|
|
52
56
|
tests/test_normalizer.py
|
|
53
57
|
tests/test_parsers.py
|
|
@@ -10,10 +10,14 @@ conversion workflow.
|
|
|
10
10
|
from __future__ import annotations
|
|
11
11
|
|
|
12
12
|
from .api import convert_line, normalize_event, parse_syslog, to_cef
|
|
13
|
+
from .custom import PatternFileError, load_patterns, register_parser
|
|
13
14
|
|
|
14
15
|
__all__ = [
|
|
15
16
|
"convert_line",
|
|
16
17
|
"normalize_event",
|
|
17
18
|
"parse_syslog",
|
|
18
19
|
"to_cef",
|
|
20
|
+
"load_patterns",
|
|
21
|
+
"register_parser",
|
|
22
|
+
"PatternFileError",
|
|
19
23
|
]
|
|
@@ -2,10 +2,10 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
4
|
import logging
|
|
5
|
-
from dataclasses import dataclass
|
|
6
|
-
from datetime import datetime
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from datetime import datetime
|
|
7
7
|
from pathlib import Path
|
|
8
|
-
from typing import Any,
|
|
8
|
+
from typing import Any, Mapping, Optional
|
|
9
9
|
|
|
10
10
|
from .parsers import ParsedEvent, autodetect_and_parse
|
|
11
11
|
from .normalizer import NormalizedEvent, normalize
|
|
@@ -2,8 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
4
|
from dataclasses import dataclass
|
|
5
|
-
from
|
|
6
|
-
from typing import Any, Dict, Mapping, Optional
|
|
5
|
+
from typing import Any, Dict, Mapping
|
|
7
6
|
|
|
8
7
|
from .normalizer import NormalizedEvent
|
|
9
8
|
from .utils import cef_escape, cef_escape_header
|