syslog2cef 0.1.4__tar.gz → 0.1.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. {syslog2cef-0.1.4/syslog2cef.egg-info → syslog2cef-0.1.5}/PKG-INFO +119 -1
  2. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/README.md +116 -0
  3. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/pyproject.toml +12 -1
  4. {syslog2cef-0.1.4 → syslog2cef-0.1.5/syslog2cef.egg-info}/PKG-INFO +119 -1
  5. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslog2cef.egg-info/SOURCES.txt +4 -0
  6. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslog2cef.egg-info/requires.txt +3 -0
  7. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/__init__.py +4 -0
  8. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/api.py +3 -3
  9. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/cef.py +1 -2
  10. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/cli.py +155 -14
  11. syslog2cef-0.1.5/syslogcef/custom.py +305 -0
  12. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/normalizer.py +2 -2
  13. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/parsers.py +25 -3
  14. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/utils.py +1 -2
  15. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/tests/test_adaptive.py +0 -1
  16. syslog2cef-0.1.5/tests/test_cli.py +178 -0
  17. syslog2cef-0.1.5/tests/test_custom_parsers.py +290 -0
  18. syslog2cef-0.1.5/tests/test_fuzz_properties.py +154 -0
  19. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/tests/test_validation.py +1 -1
  20. syslog2cef-0.1.4/tests/test_cli.py +0 -45
  21. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/LICENSE +0 -0
  22. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/setup.cfg +0 -0
  23. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslog2cef.egg-info/dependency_links.txt +0 -0
  24. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslog2cef.egg-info/entry_points.txt +0 -0
  25. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslog2cef.egg-info/top_level.txt +0 -0
  26. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/__main__.py +0 -0
  27. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/adaptive.py +0 -0
  28. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/mappings/__init__.py +0 -0
  29. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/mappings/cisco_asa.json +0 -0
  30. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/mappings/cisco_ios.json +0 -0
  31. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/mappings/f5.json +0 -0
  32. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/mappings/fortinet.json +0 -0
  33. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/mappings/linux.json +0 -0
  34. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/mappings/sophos.json +0 -0
  35. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/mappings/vmware.json +0 -0
  36. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/net.py +0 -0
  37. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/syslogcef/validation.py +0 -0
  38. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/tests/test_cef.py +0 -0
  39. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/tests/test_net.py +0 -0
  40. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/tests/test_normalizer.py +0 -0
  41. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/tests/test_parsers.py +0 -0
  42. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/tests/test_parsers_extra.py +0 -0
  43. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/tests/test_pipeline.py +0 -0
  44. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/tests/test_real_world.py +0 -0
  45. {syslog2cef-0.1.4 → syslog2cef-0.1.5}/tests/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: syslog2cef
3
- Version: 0.1.4
3
+ Version: 0.1.5
4
4
  Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
5
  Author-email: Tamir Suliman <allamiro@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/allamiro/syslogcef
@@ -17,6 +17,8 @@ Provides-Extra: test
17
17
  Requires-Dist: pytest>=7; extra == "test"
18
18
  Provides-Extra: kafka
19
19
  Requires-Dist: kafka-python>=2.0; extra == "kafka"
20
+ Provides-Extra: fuzz
21
+ Requires-Dist: hypothesis>=6; extra == "fuzz"
20
22
  Dynamic: license-file
21
23
 
22
24
  <p align="center">
@@ -199,6 +201,55 @@ Field templates that reference missing keys resolve to an empty string
199
201
  rather than failing the event. See [docs/cef_fields.md](docs/cef_fields.md)
200
202
  for the full CEF extension dictionary.
201
203
 
204
+ ## Custom Parsers
205
+
206
+ When a device emits a format no built-in parser handles, add your own
207
+ detection regexes the same way you add mappings — no code required:
208
+
209
+ ```bash
210
+ syslogcef acme.log --patterns /etc/syslogcef/patterns.json
211
+ ```
212
+
213
+ ```json
214
+ {
215
+ "patterns": [
216
+ {
217
+ "name": "acme_fw",
218
+ "regex": "^ACME (?P<ts>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}) (?P<host>\\S+) (?P<app>\\w+)\\[(?P<pid>\\d+)\\]: (?P<msg>.*)$",
219
+ "timestamp_format": "%Y-%m-%d %H:%M:%S",
220
+ "priority": "after"
221
+ }
222
+ ]
223
+ }
224
+ ```
225
+
226
+ - Named groups map to event fields: `pri`, `host`, `app`, `pid`,
227
+ `msgid`, `msg`, and `ts`.
228
+ - A `ts` group requires `timestamp_format`: a strptime format,
229
+ `iso8601`, or `epoch`. Yearless formats get the same year-rollover
230
+ inference as the built-in parsers; an unparseable timestamp never
231
+ drops the event.
232
+ - `priority` is `"after"` (default: tried after the built-ins, before
233
+ the adaptive fallback) or `"before"` (tried first, overriding
234
+ built-ins).
235
+ - Pattern names work as `--mode` values, and files are validated at
236
+ startup — bad regexes, unknown groups, or duplicate names fail with
237
+ a clear message instead of mid-stream.
238
+ - Per-service-instance patterns: set
239
+ `EXTRA_ARGS=--patterns /etc/syslogcef/patterns.json` in the
240
+ instance's conf.d file.
241
+ - Your regexes run against untrusted log content — keep them anchored
242
+ and avoid nested quantifiers (ReDoS).
243
+
244
+ From Python, register full parser functions instead:
245
+
246
+ ```python
247
+ from syslogcef import register_parser, load_patterns
248
+
249
+ load_patterns("patterns.json") # same file format
250
+ register_parser("marker", my_parse_fn) # fn(line) -> ParsedEvent | None
251
+ ```
252
+
202
253
  ## Running as a Service
203
254
 
204
255
  The RPM and Debian packages install a systemd unit and an environment
@@ -209,6 +260,9 @@ configuration in `/etc/conf.d/syslogcef`):
209
260
  arguments for the converter.
210
261
  - `syslogcef.service` — runs `syslogcef --tail` against the configured
211
262
  input and appends CEF to the configured output.
263
+ - `syslogcef@.service` — template unit for running several independent
264
+ pipelines from `/etc/syslogcef/conf.d/` (see below).
265
+ - `/etc/logrotate.d/syslogcef` — daily rotation for flat `.cef` archives.
212
266
 
213
267
  ```bash
214
268
  sudo dnf install syslogcef-*.rpm
@@ -216,6 +270,70 @@ sudo vi /etc/syslogcef/syslogcef.conf
216
270
  sudo systemctl enable --now syslogcef
217
271
  ```
218
272
 
273
+ The environment file has three variables; everything else goes through
274
+ `EXTRA_ARGS`, which accepts any command line flag (`--mode`, `--mapping`,
275
+ `--listen`, `--send`, `--eps`, `--validate`, `--strict`,
276
+ `--multiprocess`, `--log-level`, ...):
277
+
278
+ ```bash
279
+ # File(s) to follow for new syslog lines, separated by spaces.
280
+ INPUT_FILE=/var/log/messages
281
+
282
+ # File that converted CEF events are appended to.
283
+ OUTPUT_FILE=/var/log/syslogcef/events.cef
284
+
285
+ # Extra arguments; leave INPUT_FILE empty and use --listen for a
286
+ # network daemon: EXTRA_ARGS=--listen udp:514 --send tcp://siem:514
287
+ EXTRA_ARGS=
288
+ ```
289
+
290
+ ### Timed output files
291
+
292
+ `OUTPUT_FILE` (and `--output` generally) accepts strftime codes; the
293
+ file is reopened whenever the rendered path changes and parent
294
+ directories are created automatically:
295
+
296
+ ```bash
297
+ # New file each hour, grouped in a directory per day:
298
+ OUTPUT_FILE=/var/log/syslogcef/%Y-%m-%d/events-%H.cef
299
+ ```
300
+
301
+ Unsupported `%` codes are rejected at startup (use `%%` for a literal
302
+ percent). Keep templated outputs in a dated subdirectory as shown above:
303
+ the installed logrotate snippet rotates every flat `.cef` file directly
304
+ under `/var/log/syslogcef/`, and a templated file rendering flat there
305
+ would be rotated twice. Prefer a stable filename if a downstream
306
+ collector reads the archive — flat files are rotated daily by logrotate
307
+ instead.
308
+
309
+ ### Multiple pipelines (one input per output)
310
+
311
+ To map specific inputs to specific outputs, run one instance of the
312
+ template unit per pipeline. Each instance reads its own file in
313
+ `/etc/syslogcef/conf.d/` (a commented `example.conf.sample` is
314
+ installed there) and has independent restart, logs, and options:
315
+
316
+ ```bash
317
+ sudo cp /etc/syslogcef/conf.d/example.conf.sample /etc/syslogcef/conf.d/secure.conf
318
+ sudo cp /etc/syslogcef/conf.d/example.conf.sample /etc/syslogcef/conf.d/firewall.conf
319
+ sudo vi /etc/syslogcef/conf.d/secure.conf # /var/log/secure -> secure.cef
320
+ sudo vi /etc/syslogcef/conf.d/firewall.conf # --listen udp:514 --mode cisco_seq
321
+ sudo systemctl enable --now syslogcef@secure syslogcef@firewall
322
+ ```
323
+
324
+ On Alpine, OpenRC gets the same result with symlinked services:
325
+
326
+ ```bash
327
+ sudo ln -s syslogcef /etc/init.d/syslogcef.firewall
328
+ sudo cp /etc/conf.d/syslogcef /etc/conf.d/syslogcef.firewall
329
+ sudo vi /etc/conf.d/syslogcef.firewall # set a distinct INPUT_FILE and OUTPUT_FILE
330
+ sudo rc-update add syslogcef.firewall && sudo rc-service syslogcef.firewall start
331
+ ```
332
+
333
+ Give every instance its own `INPUT_FILE` and `OUTPUT_FILE` — two
334
+ instances sharing them would process the same events twice and append
335
+ to the same file concurrently.
336
+
219
337
  See [packaging/rpm/](packaging/rpm/) for the spec file and build
220
338
  instructions, including GPG signing of the RPM.
221
339
 
@@ -178,6 +178,55 @@ Field templates that reference missing keys resolve to an empty string
178
178
  rather than failing the event. See [docs/cef_fields.md](docs/cef_fields.md)
179
179
  for the full CEF extension dictionary.
180
180
 
181
+ ## Custom Parsers
182
+
183
+ When a device emits a format no built-in parser handles, add your own
184
+ detection regexes the same way you add mappings — no code required:
185
+
186
+ ```bash
187
+ syslogcef acme.log --patterns /etc/syslogcef/patterns.json
188
+ ```
189
+
190
+ ```json
191
+ {
192
+ "patterns": [
193
+ {
194
+ "name": "acme_fw",
195
+ "regex": "^ACME (?P<ts>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}) (?P<host>\\S+) (?P<app>\\w+)\\[(?P<pid>\\d+)\\]: (?P<msg>.*)$",
196
+ "timestamp_format": "%Y-%m-%d %H:%M:%S",
197
+ "priority": "after"
198
+ }
199
+ ]
200
+ }
201
+ ```
202
+
203
+ - Named groups map to event fields: `pri`, `host`, `app`, `pid`,
204
+ `msgid`, `msg`, and `ts`.
205
+ - A `ts` group requires `timestamp_format`: a strptime format,
206
+ `iso8601`, or `epoch`. Yearless formats get the same year-rollover
207
+ inference as the built-in parsers; an unparseable timestamp never
208
+ drops the event.
209
+ - `priority` is `"after"` (default: tried after the built-ins, before
210
+ the adaptive fallback) or `"before"` (tried first, overriding
211
+ built-ins).
212
+ - Pattern names work as `--mode` values, and files are validated at
213
+ startup — bad regexes, unknown groups, or duplicate names fail with
214
+ a clear message instead of mid-stream.
215
+ - Per-service-instance patterns: set
216
+ `EXTRA_ARGS=--patterns /etc/syslogcef/patterns.json` in the
217
+ instance's conf.d file.
218
+ - Your regexes run against untrusted log content — keep them anchored
219
+ and avoid nested quantifiers (ReDoS).
220
+
221
+ From Python, register full parser functions instead:
222
+
223
+ ```python
224
+ from syslogcef import register_parser, load_patterns
225
+
226
+ load_patterns("patterns.json") # same file format
227
+ register_parser("marker", my_parse_fn) # fn(line) -> ParsedEvent | None
228
+ ```
229
+
181
230
  ## Running as a Service
182
231
 
183
232
  The RPM and Debian packages install a systemd unit and an environment
@@ -188,6 +237,9 @@ configuration in `/etc/conf.d/syslogcef`):
188
237
  arguments for the converter.
189
238
  - `syslogcef.service` — runs `syslogcef --tail` against the configured
190
239
  input and appends CEF to the configured output.
240
+ - `syslogcef@.service` — template unit for running several independent
241
+ pipelines from `/etc/syslogcef/conf.d/` (see below).
242
+ - `/etc/logrotate.d/syslogcef` — daily rotation for flat `.cef` archives.
191
243
 
192
244
  ```bash
193
245
  sudo dnf install syslogcef-*.rpm
@@ -195,6 +247,70 @@ sudo vi /etc/syslogcef/syslogcef.conf
195
247
  sudo systemctl enable --now syslogcef
196
248
  ```
197
249
 
250
+ The environment file has three variables; everything else goes through
251
+ `EXTRA_ARGS`, which accepts any command line flag (`--mode`, `--mapping`,
252
+ `--listen`, `--send`, `--eps`, `--validate`, `--strict`,
253
+ `--multiprocess`, `--log-level`, ...):
254
+
255
+ ```bash
256
+ # File(s) to follow for new syslog lines, separated by spaces.
257
+ INPUT_FILE=/var/log/messages
258
+
259
+ # File that converted CEF events are appended to.
260
+ OUTPUT_FILE=/var/log/syslogcef/events.cef
261
+
262
+ # Extra arguments; leave INPUT_FILE empty and use --listen for a
263
+ # network daemon: EXTRA_ARGS=--listen udp:514 --send tcp://siem:514
264
+ EXTRA_ARGS=
265
+ ```
266
+
267
+ ### Timed output files
268
+
269
+ `OUTPUT_FILE` (and `--output` generally) accepts strftime codes; the
270
+ file is reopened whenever the rendered path changes and parent
271
+ directories are created automatically:
272
+
273
+ ```bash
274
+ # New file each hour, grouped in a directory per day:
275
+ OUTPUT_FILE=/var/log/syslogcef/%Y-%m-%d/events-%H.cef
276
+ ```
277
+
278
+ Unsupported `%` codes are rejected at startup (use `%%` for a literal
279
+ percent). Keep templated outputs in a dated subdirectory as shown above:
280
+ the installed logrotate snippet rotates every flat `.cef` file directly
281
+ under `/var/log/syslogcef/`, and a templated file rendering flat there
282
+ would be rotated twice. Prefer a stable filename if a downstream
283
+ collector reads the archive — flat files are rotated daily by logrotate
284
+ instead.
285
+
286
+ ### Multiple pipelines (one input per output)
287
+
288
+ To map specific inputs to specific outputs, run one instance of the
289
+ template unit per pipeline. Each instance reads its own file in
290
+ `/etc/syslogcef/conf.d/` (a commented `example.conf.sample` is
291
+ installed there) and has independent restart, logs, and options:
292
+
293
+ ```bash
294
+ sudo cp /etc/syslogcef/conf.d/example.conf.sample /etc/syslogcef/conf.d/secure.conf
295
+ sudo cp /etc/syslogcef/conf.d/example.conf.sample /etc/syslogcef/conf.d/firewall.conf
296
+ sudo vi /etc/syslogcef/conf.d/secure.conf # /var/log/secure -> secure.cef
297
+ sudo vi /etc/syslogcef/conf.d/firewall.conf # --listen udp:514 --mode cisco_seq
298
+ sudo systemctl enable --now syslogcef@secure syslogcef@firewall
299
+ ```
300
+
301
+ On Alpine, OpenRC gets the same result with symlinked services:
302
+
303
+ ```bash
304
+ sudo ln -s syslogcef /etc/init.d/syslogcef.firewall
305
+ sudo cp /etc/conf.d/syslogcef /etc/conf.d/syslogcef.firewall
306
+ sudo vi /etc/conf.d/syslogcef.firewall # set a distinct INPUT_FILE and OUTPUT_FILE
307
+ sudo rc-update add syslogcef.firewall && sudo rc-service syslogcef.firewall start
308
+ ```
309
+
310
+ Give every instance its own `INPUT_FILE` and `OUTPUT_FILE` — two
311
+ instances sharing them would process the same events twice and append
312
+ to the same file concurrently.
313
+
198
314
  See [packaging/rpm/](packaging/rpm/) for the spec file and build
199
315
  instructions, including GPG signing of the RPM.
200
316
 
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "syslog2cef"
7
- version = "0.1.4"
7
+ version = "0.1.5"
8
8
  description = "Convert RFC3164/RFC5424 syslog lines into ArcSight CEF"
9
9
  authors = [{name = "Tamir Suliman", email = "allamiro@gmail.com"}]
10
10
  readme = "README.md"
@@ -29,6 +29,9 @@ test = [
29
29
  kafka = [
30
30
  "kafka-python>=2.0",
31
31
  ]
32
+ fuzz = [
33
+ "hypothesis>=6",
34
+ ]
32
35
 
33
36
  [project.scripts]
34
37
  syslogcef = "syslogcef.cli:main"
@@ -43,3 +46,11 @@ syslogcef = ["mappings/*.json"]
43
46
  [tool.pytest.ini_options]
44
47
  addopts = "-ra"
45
48
  pythonpath = ["."]
49
+
50
+ [tool.ruff]
51
+ target-version = "py39"
52
+
53
+ [tool.ruff.lint]
54
+ # Start conservative: syntax errors and pyflakes (undefined names,
55
+ # unused imports). Widen the selection as the codebase is cleaned up.
56
+ select = ["E9", "F"]
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: syslog2cef
3
- Version: 0.1.4
3
+ Version: 0.1.5
4
4
  Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
5
  Author-email: Tamir Suliman <allamiro@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/allamiro/syslogcef
@@ -17,6 +17,8 @@ Provides-Extra: test
17
17
  Requires-Dist: pytest>=7; extra == "test"
18
18
  Provides-Extra: kafka
19
19
  Requires-Dist: kafka-python>=2.0; extra == "kafka"
20
+ Provides-Extra: fuzz
21
+ Requires-Dist: hypothesis>=6; extra == "fuzz"
20
22
  Dynamic: license-file
21
23
 
22
24
  <p align="center">
@@ -199,6 +201,55 @@ Field templates that reference missing keys resolve to an empty string
199
201
  rather than failing the event. See [docs/cef_fields.md](docs/cef_fields.md)
200
202
  for the full CEF extension dictionary.
201
203
 
204
+ ## Custom Parsers
205
+
206
+ When a device emits a format no built-in parser handles, add your own
207
+ detection regexes the same way you add mappings — no code required:
208
+
209
+ ```bash
210
+ syslogcef acme.log --patterns /etc/syslogcef/patterns.json
211
+ ```
212
+
213
+ ```json
214
+ {
215
+ "patterns": [
216
+ {
217
+ "name": "acme_fw",
218
+ "regex": "^ACME (?P<ts>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}) (?P<host>\\S+) (?P<app>\\w+)\\[(?P<pid>\\d+)\\]: (?P<msg>.*)$",
219
+ "timestamp_format": "%Y-%m-%d %H:%M:%S",
220
+ "priority": "after"
221
+ }
222
+ ]
223
+ }
224
+ ```
225
+
226
+ - Named groups map to event fields: `pri`, `host`, `app`, `pid`,
227
+ `msgid`, `msg`, and `ts`.
228
+ - A `ts` group requires `timestamp_format`: a strptime format,
229
+ `iso8601`, or `epoch`. Yearless formats get the same year-rollover
230
+ inference as the built-in parsers; an unparseable timestamp never
231
+ drops the event.
232
+ - `priority` is `"after"` (default: tried after the built-ins, before
233
+ the adaptive fallback) or `"before"` (tried first, overriding
234
+ built-ins).
235
+ - Pattern names work as `--mode` values, and files are validated at
236
+ startup — bad regexes, unknown groups, or duplicate names fail with
237
+ a clear message instead of mid-stream.
238
+ - Per-service-instance patterns: set
239
+ `EXTRA_ARGS=--patterns /etc/syslogcef/patterns.json` in the
240
+ instance's conf.d file.
241
+ - Your regexes run against untrusted log content — keep them anchored
242
+ and avoid nested quantifiers (ReDoS).
243
+
244
+ From Python, register full parser functions instead:
245
+
246
+ ```python
247
+ from syslogcef import register_parser, load_patterns
248
+
249
+ load_patterns("patterns.json") # same file format
250
+ register_parser("marker", my_parse_fn) # fn(line) -> ParsedEvent | None
251
+ ```
252
+
202
253
  ## Running as a Service
203
254
 
204
255
  The RPM and Debian packages install a systemd unit and an environment
@@ -209,6 +260,9 @@ configuration in `/etc/conf.d/syslogcef`):
209
260
  arguments for the converter.
210
261
  - `syslogcef.service` — runs `syslogcef --tail` against the configured
211
262
  input and appends CEF to the configured output.
263
+ - `syslogcef@.service` — template unit for running several independent
264
+ pipelines from `/etc/syslogcef/conf.d/` (see below).
265
+ - `/etc/logrotate.d/syslogcef` — daily rotation for flat `.cef` archives.
212
266
 
213
267
  ```bash
214
268
  sudo dnf install syslogcef-*.rpm
@@ -216,6 +270,70 @@ sudo vi /etc/syslogcef/syslogcef.conf
216
270
  sudo systemctl enable --now syslogcef
217
271
  ```
218
272
 
273
+ The environment file has three variables; everything else goes through
274
+ `EXTRA_ARGS`, which accepts any command line flag (`--mode`, `--mapping`,
275
+ `--listen`, `--send`, `--eps`, `--validate`, `--strict`,
276
+ `--multiprocess`, `--log-level`, ...):
277
+
278
+ ```bash
279
+ # File(s) to follow for new syslog lines, separated by spaces.
280
+ INPUT_FILE=/var/log/messages
281
+
282
+ # File that converted CEF events are appended to.
283
+ OUTPUT_FILE=/var/log/syslogcef/events.cef
284
+
285
+ # Extra arguments; leave INPUT_FILE empty and use --listen for a
286
+ # network daemon: EXTRA_ARGS=--listen udp:514 --send tcp://siem:514
287
+ EXTRA_ARGS=
288
+ ```
289
+
290
+ ### Timed output files
291
+
292
+ `OUTPUT_FILE` (and `--output` generally) accepts strftime codes; the
293
+ file is reopened whenever the rendered path changes and parent
294
+ directories are created automatically:
295
+
296
+ ```bash
297
+ # New file each hour, grouped in a directory per day:
298
+ OUTPUT_FILE=/var/log/syslogcef/%Y-%m-%d/events-%H.cef
299
+ ```
300
+
301
+ Unsupported `%` codes are rejected at startup (use `%%` for a literal
302
+ percent). Keep templated outputs in a dated subdirectory as shown above:
303
+ the installed logrotate snippet rotates every flat `.cef` file directly
304
+ under `/var/log/syslogcef/`, and a templated file rendering flat there
305
+ would be rotated twice. Prefer a stable filename if a downstream
306
+ collector reads the archive — flat files are rotated daily by logrotate
307
+ instead.
308
+
309
+ ### Multiple pipelines (one input per output)
310
+
311
+ To map specific inputs to specific outputs, run one instance of the
312
+ template unit per pipeline. Each instance reads its own file in
313
+ `/etc/syslogcef/conf.d/` (a commented `example.conf.sample` is
314
+ installed there) and has independent restart, logs, and options:
315
+
316
+ ```bash
317
+ sudo cp /etc/syslogcef/conf.d/example.conf.sample /etc/syslogcef/conf.d/secure.conf
318
+ sudo cp /etc/syslogcef/conf.d/example.conf.sample /etc/syslogcef/conf.d/firewall.conf
319
+ sudo vi /etc/syslogcef/conf.d/secure.conf # /var/log/secure -> secure.cef
320
+ sudo vi /etc/syslogcef/conf.d/firewall.conf # --listen udp:514 --mode cisco_seq
321
+ sudo systemctl enable --now syslogcef@secure syslogcef@firewall
322
+ ```
323
+
324
+ On Alpine, OpenRC gets the same result with symlinked services:
325
+
326
+ ```bash
327
+ sudo ln -s syslogcef /etc/init.d/syslogcef.firewall
328
+ sudo cp /etc/conf.d/syslogcef /etc/conf.d/syslogcef.firewall
329
+ sudo vi /etc/conf.d/syslogcef.firewall # set a distinct INPUT_FILE and OUTPUT_FILE
330
+ sudo rc-update add syslogcef.firewall && sudo rc-service syslogcef.firewall start
331
+ ```
332
+
333
+ Give every instance its own `INPUT_FILE` and `OUTPUT_FILE` — two
334
+ instances sharing them would process the same events twice and append
335
+ to the same file concurrently.
336
+
219
337
  See [packaging/rpm/](packaging/rpm/) for the spec file and build
220
338
  instructions, including GPG signing of the RPM.
221
339
 
@@ -7,6 +7,7 @@ pyproject.toml
7
7
  ./syslogcef/api.py
8
8
  ./syslogcef/cef.py
9
9
  ./syslogcef/cli.py
10
+ ./syslogcef/custom.py
10
11
  ./syslogcef/net.py
11
12
  ./syslogcef/normalizer.py
12
13
  ./syslogcef/parsers.py
@@ -32,6 +33,7 @@ syslogcef/adaptive.py
32
33
  syslogcef/api.py
33
34
  syslogcef/cef.py
34
35
  syslogcef/cli.py
36
+ syslogcef/custom.py
35
37
  syslogcef/net.py
36
38
  syslogcef/normalizer.py
37
39
  syslogcef/parsers.py
@@ -48,6 +50,8 @@ syslogcef/mappings/vmware.json
48
50
  tests/test_adaptive.py
49
51
  tests/test_cef.py
50
52
  tests/test_cli.py
53
+ tests/test_custom_parsers.py
54
+ tests/test_fuzz_properties.py
51
55
  tests/test_net.py
52
56
  tests/test_normalizer.py
53
57
  tests/test_parsers.py
@@ -1,4 +1,7 @@
1
1
 
2
+ [fuzz]
3
+ hypothesis>=6
4
+
2
5
  [kafka]
3
6
  kafka-python>=2.0
4
7
 
@@ -10,10 +10,14 @@ conversion workflow.
10
10
  from __future__ import annotations
11
11
 
12
12
  from .api import convert_line, normalize_event, parse_syslog, to_cef
13
+ from .custom import PatternFileError, load_patterns, register_parser
13
14
 
14
15
  __all__ = [
15
16
  "convert_line",
16
17
  "normalize_event",
17
18
  "parse_syslog",
18
19
  "to_cef",
20
+ "load_patterns",
21
+ "register_parser",
22
+ "PatternFileError",
19
23
  ]
@@ -2,10 +2,10 @@ from __future__ import annotations
2
2
 
3
3
  import json
4
4
  import logging
5
- from dataclasses import dataclass, field
6
- from datetime import datetime, timezone
5
+ from dataclasses import dataclass
6
+ from datetime import datetime
7
7
  from pathlib import Path
8
- from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional
8
+ from typing import Any, Mapping, Optional
9
9
 
10
10
  from .parsers import ParsedEvent, autodetect_and_parse
11
11
  from .normalizer import NormalizedEvent, normalize
@@ -2,8 +2,7 @@ from __future__ import annotations
2
2
 
3
3
  import logging
4
4
  from dataclasses import dataclass
5
- from datetime import datetime
6
- from typing import Any, Dict, Mapping, Optional
5
+ from typing import Any, Dict, Mapping
7
6
 
8
7
  from .normalizer import NormalizedEvent
9
8
  from .utils import cef_escape, cef_escape_header