syslog2cef 0.3.2__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. {syslog2cef-0.3.2/syslog2cef.egg-info → syslog2cef-0.4.0}/PKG-INFO +87 -16
  2. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/README.md +80 -15
  3. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/pyproject.toml +7 -1
  4. {syslog2cef-0.3.2 → syslog2cef-0.4.0/syslog2cef.egg-info}/PKG-INFO +87 -16
  5. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/syslog2cef.egg-info/SOURCES.txt +2 -0
  6. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/syslogcef/__init__.py +6 -0
  7. syslog2cef-0.4.0/syslogcef/_version.py +12 -0
  8. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/syslogcef/adaptive.py +69 -19
  9. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/syslogcef/api.py +122 -11
  10. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/syslogcef/cef.py +41 -2
  11. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/syslogcef/cli.py +27 -1
  12. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/syslogcef/custom.py +6 -18
  13. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/syslogcef/mappings/cisco_asa.json +1 -1
  14. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/syslogcef/normalizer.py +46 -5
  15. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/syslogcef/parsers.py +43 -32
  16. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/syslogcef/utils.py +38 -3
  17. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/tests/test_adaptive.py +129 -0
  18. syslog2cef-0.4.0/tests/test_cef.py +230 -0
  19. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/tests/test_cli.py +43 -0
  20. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/tests/test_custom_parsers.py +18 -1
  21. syslog2cef-0.4.0/tests/test_normalizer.py +123 -0
  22. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/tests/test_parsers_extra.py +44 -0
  23. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/tests/test_pipeline.py +19 -2
  24. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/tests/test_utils.py +36 -0
  25. syslog2cef-0.3.2/tests/test_cef.py +0 -64
  26. syslog2cef-0.3.2/tests/test_normalizer.py +0 -49
  27. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/LICENSE +0 -0
  28. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/setup.cfg +0 -0
  29. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/syslog2cef.egg-info/dependency_links.txt +0 -0
  30. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/syslog2cef.egg-info/entry_points.txt +0 -0
  31. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/syslog2cef.egg-info/requires.txt +0 -0
  32. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/syslog2cef.egg-info/top_level.txt +0 -0
  33. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/syslogcef/__main__.py +0 -0
  34. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/syslogcef/dictionary.json +0 -0
  35. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/syslogcef/dictionary.py +0 -0
  36. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/syslogcef/mappings/__init__.py +0 -0
  37. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/syslogcef/mappings/cisco_ios.json +0 -0
  38. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/syslogcef/mappings/f5.json +0 -0
  39. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/syslogcef/mappings/fortinet.json +0 -0
  40. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/syslogcef/mappings/linux.json +0 -0
  41. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/syslogcef/mappings/sophos.json +0 -0
  42. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/syslogcef/mappings/vmware.json +0 -0
  43. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/syslogcef/net.py +0 -0
  44. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/syslogcef/validation.py +0 -0
  45. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/tests/test_cli_validation.py +0 -0
  46. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/tests/test_dictionary.py +0 -0
  47. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/tests/test_fuzz_grammar.py +0 -0
  48. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/tests/test_fuzz_properties.py +0 -0
  49. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/tests/test_macos_logs.py +0 -0
  50. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/tests/test_net.py +0 -0
  51. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/tests/test_parsers.py +0 -0
  52. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/tests/test_real_world.py +0 -0
  53. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/tests/test_rfc5424_epoch.py +0 -0
  54. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/tests/test_security_hardening.py +0 -0
  55. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/tests/test_tail_rotation.py +0 -0
  56. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/tests/test_validation.py +0 -0
  57. {syslog2cef-0.3.2 → syslog2cef-0.4.0}/tests/test_zipapp_safe.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: syslog2cef
3
- Version: 0.3.2
3
+ Version: 0.4.0
4
4
  Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
5
  Author-email: Tamir Suliman <allamiro@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/allamiro/syslogcef
@@ -8,6 +8,12 @@ Project-URL: Issues, https://github.com/allamiro/syslogcef/issues
8
8
  Keywords: syslog,cef,logging,security
9
9
  Classifier: Programming Language :: Python :: 3
10
10
  Classifier: Programming Language :: Python :: 3 :: Only
11
+ Classifier: Programming Language :: Python :: 3.9
12
+ Classifier: Programming Language :: Python :: 3.10
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Programming Language :: Python :: 3.13
16
+ Classifier: Programming Language :: Python :: 3.14
11
17
  Classifier: License :: OSI Approved :: MIT License
12
18
  Classifier: Topic :: System :: Logging
13
19
  Requires-Python: >=3.9
@@ -25,6 +31,19 @@ Dynamic: license-file
25
31
  <img src="https://raw.githubusercontent.com/allamiro/syslogcef/main/docs/logo.svg" alt="syslogcef logo" width="540">
26
32
  </p>
27
33
 
34
+ <p align="center">
35
+ <a href="https://github.com/allamiro/syslog2cef/actions/workflows/ci.yml"><img src="https://img.shields.io/github/actions/workflow/status/allamiro/syslog2cef/ci.yml?branch=main&label=CI" alt="CI status"></a>
36
+ <a href="https://pypi.org/project/syslog2cef/"><img src="https://img.shields.io/pypi/v/syslog2cef" alt="PyPI version"></a>
37
+ <a href="https://pypi.org/project/syslog2cef/"><img src="https://img.shields.io/pypi/pyversions/syslog2cef" alt="Supported Python versions"></a>
38
+ <a href="https://pypistats.org/packages/syslog2cef"><img src="https://img.shields.io/pypi/dm/syslog2cef" alt="PyPI downloads per month"></a>
39
+ <a href="https://copr.fedorainfracloud.org/coprs/allamiro/syslogcef/"><img src="https://copr.fedorainfracloud.org/coprs/allamiro/syslogcef/package/syslogcef/status_image/last_build.png" alt="COPR build status"></a>
40
+ <a href="https://github.com/allamiro/syslog2cef/pkgs/container/syslogcef"><img src="https://img.shields.io/badge/container-ghcr.io-blue?logo=docker" alt="Container image on ghcr.io"></a>
41
+ <a href="https://github.com/allamiro/syslog2cef/actions/workflows/fuzz.yml"><img src="https://img.shields.io/github/actions/workflow/status/allamiro/syslog2cef/fuzz.yml?branch=main&label=deep%20fuzz" alt="Deep fuzz status"></a>
42
+ <a href="https://github.com/astral-sh/ruff"><img src="https://img.shields.io/badge/lint-ruff-261230" alt="Linted with Ruff"></a>
43
+ <a href="LICENSE"><img src="https://img.shields.io/github/license/allamiro/syslog2cef" alt="MIT license"></a>
44
+ <a href="CITATION.cff"><img src="https://img.shields.io/badge/cite-CITATION.cff-informational" alt="Citation file"></a>
45
+ </p>
46
+
28
47
  # syslogcef
29
48
 
30
49
  syslogcef converts syslog events into ArcSight Common Event Format (CEF). It
@@ -36,11 +55,18 @@ platforms that consume CEF.
36
55
 
37
56
  - Automatic detection and parsing of multiple syslog dialects:
38
57
  RFC3164 (BSD), RFC5424 (including structured data), rsyslog JSON and
39
- file formats, and systemd journal exports (JSON, short, and ISO formats).
58
+ file formats, systemd journal exports (JSON, short, and ISO formats),
59
+ Cisco console/buffer lines, ISO-timestamp syslog, and key=value streams
60
+ (Fortinet, Sophos). Unknown layouts are learned adaptively.
40
61
  - Deterministic vendor/product mappings bundled for Cisco ASA, Cisco IOS,
41
- F5 BIG-IP, generic Linux, and VMware ESXi, with automatic mapping
42
- selection based on message content.
43
- - Custom mappings supplied as JSON files or Python dictionaries.
62
+ F5 BIG-IP, Fortinet FortiGate, Sophos XG Firewall, VMware ESXi, and
63
+ generic Linux, with automatic mapping selection based on message content.
64
+ - Custom mappings supplied as JSON files or Python dictionaries, and custom
65
+ parsers supplied as regex pattern files.
66
+ - Validation of CEF extension values against the ArcSight extension
67
+ dictionary (`--validate`, `--strict`).
68
+ - Network receiver (`--listen udp:514`) and forwarding to UDP, TCP, or
69
+ Kafka (`--send`), with optional rate limiting.
44
70
  - Escaping of untrusted log content in both CEF header and extension
45
71
  fields, so crafted messages cannot forge header fields or split records.
46
72
  - Streaming CLI with stdin/file input, tail (follow) mode across multiple
@@ -129,13 +155,21 @@ Options:
129
155
  | Option | Description |
130
156
  | ------ | ----------- |
131
157
  | `paths` | Input files; stdin is used when omitted. |
132
- | `-o, --output FILE` | Write CEF lines to a file instead of stdout. |
133
- | `--mode MODE` | Parser override: `rfc3164`, `rfc5424`, `rsyslog_json`, `rsyslog_file`, `journald_json`, `journald_short`, `journald_iso`. Auto-detected when omitted. |
158
+ | `-o, --output FILE` | Write CEF lines to a file instead of stdout. Accepts strftime codes for timed files (see "Timed output files"). |
159
+ | `-a, --append` | Append to `--output` instead of truncating it (implied for templated paths). |
160
+ | `--mode MODE` | Parser override: `rfc3164`, `rfc5424`, `rsyslog_json`, `rsyslog_file`, `journald_json`, `journald_short`, `journald_iso`, `cisco_seq`, `iso_syslog`, `kv`, or a name from `--patterns`. Auto-detected when omitted. |
134
161
  | `--mapping FILE` | Mapping JSON file. Auto-selected from message content when omitted. |
162
+ | `--patterns FILE` | Custom parser patterns (see "Custom Parsers"); may be repeated. |
135
163
  | `--tail` | Follow input files like `tail -f`. |
164
+ | `--listen PROTO:PORT` | Receive syslog over the network instead of files/stdin: `udp:514`, `tcp:5514`, or `udp:10.0.0.5:514`. |
165
+ | `--send URL` | Forward CEF to `udp://HOST:PORT`, `tcp://HOST:PORT`, or `kafka://BROKER:PORT/TOPIC` (needs the `kafka` extra). |
166
+ | `--eps N` | Rate-limit `--send` to at most N events per second. |
136
167
  | `--multiprocess` | Convert lines using a process pool. |
137
168
  | `--pool-size N` | Worker count for `--multiprocess` (default: CPU count minus one). |
169
+ | `--validate` | Check CEF extensions against the ArcSight dictionary; violations are logged as warnings. |
170
+ | `--strict` | Like `--validate`, but exit with an error on type or length violations. |
138
171
  | `--log-level LEVEL` | Python logging level (default `WARNING`). |
172
+ | `--version` | Print the version, copyright, license and author, then exit. |
139
173
 
140
174
  `python -m syslogcef` is equivalent to the `syslogcef` entry point.
141
175
 
@@ -161,7 +195,8 @@ cef = to_cef(normalized, mapping="my_mapping.json")
161
195
  ```
162
196
 
163
197
  Bundled mappings are importable from `syslogcef.mappings` (`CISCO_ASA`,
164
- `CISCO_IOS`, `F5`, `LINUX`, `VMWARE`, or `load_mapping(name)`).
198
+ `CISCO_IOS`, `F5`, `FORTINET`, `LINUX`, `SOPHOS`, `VMWARE`, or
199
+ `load_mapping(name)`).
165
200
 
166
201
  ## Mapping Files
167
202
 
@@ -181,7 +216,7 @@ normalized event's fields:
181
216
  "src": "%(src)s",
182
217
  "dst": "%(dst)s",
183
218
  "cs1Label": "rawEvent",
184
- "cs1": "%(raw_kv)s"
219
+ "cs1": "%(raw)s"
185
220
  }
186
221
  }
187
222
  ```
@@ -191,15 +226,31 @@ normalized event's fields:
191
226
  - `severity_map` translates syslog severity (0-7) to CEF severity (0-10);
192
227
  unmapped values pass through.
193
228
  - `extensions` maps CEF extension keys to templates. Extensions that
194
- resolve to an empty value are omitted.
229
+ resolve to an empty value are omitted, and so is a custom-field label
230
+ (`cs1Label`, `cn1Label`, ...) whose paired value is omitted.
195
231
  - Available template fields include `host`, `app`, `pid`, `msgid`, `msg`,
196
232
  `message_short` (first 120 characters), `raw`, `raw_kv`, `event_code`,
197
233
  `facility`, `severity`, `ts`, plus every key=value pair extracted from
198
234
  the message and any RFC5424 structured-data or journald fields.
199
235
 
200
236
  Field templates that reference missing keys resolve to an empty string
201
- rather than failing the event. See [docs/cef_fields.md](docs/cef_fields.md)
202
- for the full CEF extension dictionary.
237
+ rather than failing the event. Mapping files and Python mapping dictionaries
238
+ are validated before rendering; malformed templates, severity maps, and CEF
239
+ extension keys fail with a configuration error instead of corrupting a record.
240
+ See [docs/cef_fields.md](docs/cef_fields.md) for the full CEF extension
241
+ dictionary.
242
+
243
+ ## Timestamps without a year
244
+
245
+ RFC3164, journald short, Cisco console lines and many device formats
246
+ stamp `Jan 1 12:34:56` with no year. syslogcef dates such a timestamp to
247
+ its most recent occurrence that is not more than one day ahead of the
248
+ current time: log lines describe events that already happened. A `Jan 1`
249
+ line converted in November is dated January of the current year, while a
250
+ line stamped just after midnight on New Year's Eve (a sender clock or
251
+ timezone slightly ahead) lands in the new year. Pass `now=` to
252
+ `parse_syslog`/`convert_line` to date archived files relative to when
253
+ they were written.
203
254
 
204
255
  ## Field Dictionary
205
256
 
@@ -217,7 +268,10 @@ knowledge, derived from the ArcSight Extension Dictionary (see
217
268
  are applied during normalization for **every** event — including
218
269
  key=value pairs extracted from adaptively-parsed unknown formats — so
219
270
  mappings and validation always see canonical names. Original keys are
220
- preserved and an explicit canonical key is never overwritten.
271
+ preserved and an explicit canonical key is never overwritten. Source key
272
+ casing is normalized as well (`SRCIP`, `SrcIp`, and `srcip` all make `src`
273
+ available), while parsed syslog envelope fields such as host and severity
274
+ remain authoritative over same-named text in the message.
221
275
 
222
276
  This means a Fortinet-style `srcip=10.1.1.1 dstport=443` and an unknown
223
277
  device emitting the same pairs behind an unrecognized prefix both end up
@@ -249,9 +303,9 @@ syslogcef acme.log --patterns /etc/syslogcef/patterns.json
249
303
  - Named groups map to event fields: `pri`, `host`, `app`, `pid`,
250
304
  `msgid`, `msg`, and `ts`.
251
305
  - A `ts` group requires `timestamp_format`: a strptime format,
252
- `iso8601`, or `epoch`. Yearless formats get the same year-rollover
253
- inference as the built-in parsers; an unparseable timestamp never
254
- drops the event.
306
+ `iso8601`, or `epoch`. Yearless formats get the same year inference
307
+ as the built-in parsers (see "Timestamps without a year" below); an
308
+ unparseable timestamp never drops the event.
255
309
  - `priority` is `"after"` (default: tried after the built-ins, before
256
310
  the adaptive fallback) or `"before"` (tried first, overriding
257
311
  built-ins).
@@ -382,6 +436,23 @@ pytest
382
436
  Contributions are welcome — see [CONTRIBUTING.md](CONTRIBUTING.md). Notable
383
437
  changes are tracked in [CHANGELOG.md](CHANGELOG.md).
384
438
 
439
+ ## Citation
440
+
441
+ If you use syslog2cef in research or published work, please cite it. The
442
+ metadata is in [CITATION.cff](CITATION.cff), and GitHub's "Cite this
443
+ repository" button produces APA and BibTeX from it.
444
+
445
+ ```bibtex
446
+ @software{suliman_syslog2cef,
447
+ author = {Suliman, Tamir},
448
+ title = {syslog2cef: Convert syslog to ArcSight Common Event Format (CEF)},
449
+ version = {0.4.0},
450
+ year = {2026},
451
+ url = {https://github.com/allamiro/syslog2cef},
452
+ license = {MIT}
453
+ }
454
+ ```
455
+
385
456
  ## License
386
457
 
387
458
  MIT — see [LICENSE](LICENSE). Copyright (c) Tamir Suliman.
@@ -2,6 +2,19 @@
2
2
  <img src="https://raw.githubusercontent.com/allamiro/syslogcef/main/docs/logo.svg" alt="syslogcef logo" width="540">
3
3
  </p>
4
4
 
5
+ <p align="center">
6
+ <a href="https://github.com/allamiro/syslog2cef/actions/workflows/ci.yml"><img src="https://img.shields.io/github/actions/workflow/status/allamiro/syslog2cef/ci.yml?branch=main&label=CI" alt="CI status"></a>
7
+ <a href="https://pypi.org/project/syslog2cef/"><img src="https://img.shields.io/pypi/v/syslog2cef" alt="PyPI version"></a>
8
+ <a href="https://pypi.org/project/syslog2cef/"><img src="https://img.shields.io/pypi/pyversions/syslog2cef" alt="Supported Python versions"></a>
9
+ <a href="https://pypistats.org/packages/syslog2cef"><img src="https://img.shields.io/pypi/dm/syslog2cef" alt="PyPI downloads per month"></a>
10
+ <a href="https://copr.fedorainfracloud.org/coprs/allamiro/syslogcef/"><img src="https://copr.fedorainfracloud.org/coprs/allamiro/syslogcef/package/syslogcef/status_image/last_build.png" alt="COPR build status"></a>
11
+ <a href="https://github.com/allamiro/syslog2cef/pkgs/container/syslogcef"><img src="https://img.shields.io/badge/container-ghcr.io-blue?logo=docker" alt="Container image on ghcr.io"></a>
12
+ <a href="https://github.com/allamiro/syslog2cef/actions/workflows/fuzz.yml"><img src="https://img.shields.io/github/actions/workflow/status/allamiro/syslog2cef/fuzz.yml?branch=main&label=deep%20fuzz" alt="Deep fuzz status"></a>
13
+ <a href="https://github.com/astral-sh/ruff"><img src="https://img.shields.io/badge/lint-ruff-261230" alt="Linted with Ruff"></a>
14
+ <a href="LICENSE"><img src="https://img.shields.io/github/license/allamiro/syslog2cef" alt="MIT license"></a>
15
+ <a href="CITATION.cff"><img src="https://img.shields.io/badge/cite-CITATION.cff-informational" alt="Citation file"></a>
16
+ </p>
17
+
5
18
  # syslogcef
6
19
 
7
20
  syslogcef converts syslog events into ArcSight Common Event Format (CEF). It
@@ -13,11 +26,18 @@ platforms that consume CEF.
13
26
 
14
27
  - Automatic detection and parsing of multiple syslog dialects:
15
28
  RFC3164 (BSD), RFC5424 (including structured data), rsyslog JSON and
16
- file formats, and systemd journal exports (JSON, short, and ISO formats).
29
+ file formats, systemd journal exports (JSON, short, and ISO formats),
30
+ Cisco console/buffer lines, ISO-timestamp syslog, and key=value streams
31
+ (Fortinet, Sophos). Unknown layouts are learned adaptively.
17
32
  - Deterministic vendor/product mappings bundled for Cisco ASA, Cisco IOS,
18
- F5 BIG-IP, generic Linux, and VMware ESXi, with automatic mapping
19
- selection based on message content.
20
- - Custom mappings supplied as JSON files or Python dictionaries.
33
+ F5 BIG-IP, Fortinet FortiGate, Sophos XG Firewall, VMware ESXi, and
34
+ generic Linux, with automatic mapping selection based on message content.
35
+ - Custom mappings supplied as JSON files or Python dictionaries, and custom
36
+ parsers supplied as regex pattern files.
37
+ - Validation of CEF extension values against the ArcSight extension
38
+ dictionary (`--validate`, `--strict`).
39
+ - Network receiver (`--listen udp:514`) and forwarding to UDP, TCP, or
40
+ Kafka (`--send`), with optional rate limiting.
21
41
  - Escaping of untrusted log content in both CEF header and extension
22
42
  fields, so crafted messages cannot forge header fields or split records.
23
43
  - Streaming CLI with stdin/file input, tail (follow) mode across multiple
@@ -106,13 +126,21 @@ Options:
106
126
  | Option | Description |
107
127
  | ------ | ----------- |
108
128
  | `paths` | Input files; stdin is used when omitted. |
109
- | `-o, --output FILE` | Write CEF lines to a file instead of stdout. |
110
- | `--mode MODE` | Parser override: `rfc3164`, `rfc5424`, `rsyslog_json`, `rsyslog_file`, `journald_json`, `journald_short`, `journald_iso`. Auto-detected when omitted. |
129
+ | `-o, --output FILE` | Write CEF lines to a file instead of stdout. Accepts strftime codes for timed files (see "Timed output files"). |
130
+ | `-a, --append` | Append to `--output` instead of truncating it (implied for templated paths). |
131
+ | `--mode MODE` | Parser override: `rfc3164`, `rfc5424`, `rsyslog_json`, `rsyslog_file`, `journald_json`, `journald_short`, `journald_iso`, `cisco_seq`, `iso_syslog`, `kv`, or a name from `--patterns`. Auto-detected when omitted. |
111
132
  | `--mapping FILE` | Mapping JSON file. Auto-selected from message content when omitted. |
133
+ | `--patterns FILE` | Custom parser patterns (see "Custom Parsers"); may be repeated. |
112
134
  | `--tail` | Follow input files like `tail -f`. |
135
+ | `--listen PROTO:PORT` | Receive syslog over the network instead of files/stdin: `udp:514`, `tcp:5514`, or `udp:10.0.0.5:514`. |
136
+ | `--send URL` | Forward CEF to `udp://HOST:PORT`, `tcp://HOST:PORT`, or `kafka://BROKER:PORT/TOPIC` (needs the `kafka` extra). |
137
+ | `--eps N` | Rate-limit `--send` to at most N events per second. |
113
138
  | `--multiprocess` | Convert lines using a process pool. |
114
139
  | `--pool-size N` | Worker count for `--multiprocess` (default: CPU count minus one). |
140
+ | `--validate` | Check CEF extensions against the ArcSight dictionary; violations are logged as warnings. |
141
+ | `--strict` | Like `--validate`, but exit with an error on type or length violations. |
115
142
  | `--log-level LEVEL` | Python logging level (default `WARNING`). |
143
+ | `--version` | Print the version, copyright, license and author, then exit. |
116
144
 
117
145
  `python -m syslogcef` is equivalent to the `syslogcef` entry point.
118
146
 
@@ -138,7 +166,8 @@ cef = to_cef(normalized, mapping="my_mapping.json")
138
166
  ```
139
167
 
140
168
  Bundled mappings are importable from `syslogcef.mappings` (`CISCO_ASA`,
141
- `CISCO_IOS`, `F5`, `LINUX`, `VMWARE`, or `load_mapping(name)`).
169
+ `CISCO_IOS`, `F5`, `FORTINET`, `LINUX`, `SOPHOS`, `VMWARE`, or
170
+ `load_mapping(name)`).
142
171
 
143
172
  ## Mapping Files
144
173
 
@@ -158,7 +187,7 @@ normalized event's fields:
158
187
  "src": "%(src)s",
159
188
  "dst": "%(dst)s",
160
189
  "cs1Label": "rawEvent",
161
- "cs1": "%(raw_kv)s"
190
+ "cs1": "%(raw)s"
162
191
  }
163
192
  }
164
193
  ```
@@ -168,15 +197,31 @@ normalized event's fields:
168
197
  - `severity_map` translates syslog severity (0-7) to CEF severity (0-10);
169
198
  unmapped values pass through.
170
199
  - `extensions` maps CEF extension keys to templates. Extensions that
171
- resolve to an empty value are omitted.
200
+ resolve to an empty value are omitted, and so is a custom-field label
201
+ (`cs1Label`, `cn1Label`, ...) whose paired value is omitted.
172
202
  - Available template fields include `host`, `app`, `pid`, `msgid`, `msg`,
173
203
  `message_short` (first 120 characters), `raw`, `raw_kv`, `event_code`,
174
204
  `facility`, `severity`, `ts`, plus every key=value pair extracted from
175
205
  the message and any RFC5424 structured-data or journald fields.
176
206
 
177
207
  Field templates that reference missing keys resolve to an empty string
178
- rather than failing the event. See [docs/cef_fields.md](docs/cef_fields.md)
179
- for the full CEF extension dictionary.
208
+ rather than failing the event. Mapping files and Python mapping dictionaries
209
+ are validated before rendering; malformed templates, severity maps, and CEF
210
+ extension keys fail with a configuration error instead of corrupting a record.
211
+ See [docs/cef_fields.md](docs/cef_fields.md) for the full CEF extension
212
+ dictionary.
213
+
214
+ ## Timestamps without a year
215
+
216
+ RFC3164, journald short, Cisco console lines and many device formats
217
+ stamp `Jan 1 12:34:56` with no year. syslogcef dates such a timestamp to
218
+ its most recent occurrence that is not more than one day ahead of the
219
+ current time: log lines describe events that already happened. A `Jan 1`
220
+ line converted in November is dated January of the current year, while a
221
+ line stamped just after midnight on New Year's Eve (a sender clock or
222
+ timezone slightly ahead) lands in the new year. Pass `now=` to
223
+ `parse_syslog`/`convert_line` to date archived files relative to when
224
+ they were written.
180
225
 
181
226
  ## Field Dictionary
182
227
 
@@ -194,7 +239,10 @@ knowledge, derived from the ArcSight Extension Dictionary (see
194
239
  are applied during normalization for **every** event — including
195
240
  key=value pairs extracted from adaptively-parsed unknown formats — so
196
241
  mappings and validation always see canonical names. Original keys are
197
- preserved and an explicit canonical key is never overwritten.
242
+ preserved and an explicit canonical key is never overwritten. Source key
243
+ casing is normalized as well (`SRCIP`, `SrcIp`, and `srcip` all make `src`
244
+ available), while parsed syslog envelope fields such as host and severity
245
+ remain authoritative over same-named text in the message.
198
246
 
199
247
  This means a Fortinet-style `srcip=10.1.1.1 dstport=443` and an unknown
200
248
  device emitting the same pairs behind an unrecognized prefix both end up
@@ -226,9 +274,9 @@ syslogcef acme.log --patterns /etc/syslogcef/patterns.json
226
274
  - Named groups map to event fields: `pri`, `host`, `app`, `pid`,
227
275
  `msgid`, `msg`, and `ts`.
228
276
  - A `ts` group requires `timestamp_format`: a strptime format,
229
- `iso8601`, or `epoch`. Yearless formats get the same year-rollover
230
- inference as the built-in parsers; an unparseable timestamp never
231
- drops the event.
277
+ `iso8601`, or `epoch`. Yearless formats get the same year inference
278
+ as the built-in parsers (see "Timestamps without a year" below); an
279
+ unparseable timestamp never drops the event.
232
280
  - `priority` is `"after"` (default: tried after the built-ins, before
233
281
  the adaptive fallback) or `"before"` (tried first, overriding
234
282
  built-ins).
@@ -359,6 +407,23 @@ pytest
359
407
  Contributions are welcome — see [CONTRIBUTING.md](CONTRIBUTING.md). Notable
360
408
  changes are tracked in [CHANGELOG.md](CHANGELOG.md).
361
409
 
410
+ ## Citation
411
+
412
+ If you use syslog2cef in research or published work, please cite it. The
413
+ metadata is in [CITATION.cff](CITATION.cff), and GitHub's "Cite this
414
+ repository" button produces APA and BibTeX from it.
415
+
416
+ ```bibtex
417
+ @software{suliman_syslog2cef,
418
+ author = {Suliman, Tamir},
419
+ title = {syslog2cef: Convert syslog to ArcSight Common Event Format (CEF)},
420
+ version = {0.4.0},
421
+ year = {2026},
422
+ url = {https://github.com/allamiro/syslog2cef},
423
+ license = {MIT}
424
+ }
425
+ ```
426
+
362
427
  ## License
363
428
 
364
429
  MIT — see [LICENSE](LICENSE). Copyright (c) Tamir Suliman.
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "syslog2cef"
7
- version = "0.3.2"
7
+ version = "0.4.0"
8
8
  description = "Convert RFC3164/RFC5424 syslog lines into ArcSight CEF"
9
9
  authors = [{name = "Tamir Suliman", email = "allamiro@gmail.com"}]
10
10
  readme = "README.md"
@@ -12,6 +12,12 @@ requires-python = ">=3.9"
12
12
  classifiers = [
13
13
  "Programming Language :: Python :: 3",
14
14
  "Programming Language :: Python :: 3 :: Only",
15
+ "Programming Language :: Python :: 3.9",
16
+ "Programming Language :: Python :: 3.10",
17
+ "Programming Language :: Python :: 3.11",
18
+ "Programming Language :: Python :: 3.12",
19
+ "Programming Language :: Python :: 3.13",
20
+ "Programming Language :: Python :: 3.14",
15
21
  "License :: OSI Approved :: MIT License",
16
22
  "Topic :: System :: Logging",
17
23
  ]
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: syslog2cef
3
- Version: 0.3.2
3
+ Version: 0.4.0
4
4
  Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
5
  Author-email: Tamir Suliman <allamiro@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/allamiro/syslogcef
@@ -8,6 +8,12 @@ Project-URL: Issues, https://github.com/allamiro/syslogcef/issues
8
8
  Keywords: syslog,cef,logging,security
9
9
  Classifier: Programming Language :: Python :: 3
10
10
  Classifier: Programming Language :: Python :: 3 :: Only
11
+ Classifier: Programming Language :: Python :: 3.9
12
+ Classifier: Programming Language :: Python :: 3.10
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Programming Language :: Python :: 3.13
16
+ Classifier: Programming Language :: Python :: 3.14
11
17
  Classifier: License :: OSI Approved :: MIT License
12
18
  Classifier: Topic :: System :: Logging
13
19
  Requires-Python: >=3.9
@@ -25,6 +31,19 @@ Dynamic: license-file
25
31
  <img src="https://raw.githubusercontent.com/allamiro/syslogcef/main/docs/logo.svg" alt="syslogcef logo" width="540">
26
32
  </p>
27
33
 
34
+ <p align="center">
35
+ <a href="https://github.com/allamiro/syslog2cef/actions/workflows/ci.yml"><img src="https://img.shields.io/github/actions/workflow/status/allamiro/syslog2cef/ci.yml?branch=main&label=CI" alt="CI status"></a>
36
+ <a href="https://pypi.org/project/syslog2cef/"><img src="https://img.shields.io/pypi/v/syslog2cef" alt="PyPI version"></a>
37
+ <a href="https://pypi.org/project/syslog2cef/"><img src="https://img.shields.io/pypi/pyversions/syslog2cef" alt="Supported Python versions"></a>
38
+ <a href="https://pypistats.org/packages/syslog2cef"><img src="https://img.shields.io/pypi/dm/syslog2cef" alt="PyPI downloads per month"></a>
39
+ <a href="https://copr.fedorainfracloud.org/coprs/allamiro/syslogcef/"><img src="https://copr.fedorainfracloud.org/coprs/allamiro/syslogcef/package/syslogcef/status_image/last_build.png" alt="COPR build status"></a>
40
+ <a href="https://github.com/allamiro/syslog2cef/pkgs/container/syslogcef"><img src="https://img.shields.io/badge/container-ghcr.io-blue?logo=docker" alt="Container image on ghcr.io"></a>
41
+ <a href="https://github.com/allamiro/syslog2cef/actions/workflows/fuzz.yml"><img src="https://img.shields.io/github/actions/workflow/status/allamiro/syslog2cef/fuzz.yml?branch=main&label=deep%20fuzz" alt="Deep fuzz status"></a>
42
+ <a href="https://github.com/astral-sh/ruff"><img src="https://img.shields.io/badge/lint-ruff-261230" alt="Linted with Ruff"></a>
43
+ <a href="LICENSE"><img src="https://img.shields.io/github/license/allamiro/syslog2cef" alt="MIT license"></a>
44
+ <a href="CITATION.cff"><img src="https://img.shields.io/badge/cite-CITATION.cff-informational" alt="Citation file"></a>
45
+ </p>
46
+
28
47
  # syslogcef
29
48
 
30
49
  syslogcef converts syslog events into ArcSight Common Event Format (CEF). It
@@ -36,11 +55,18 @@ platforms that consume CEF.
36
55
 
37
56
  - Automatic detection and parsing of multiple syslog dialects:
38
57
  RFC3164 (BSD), RFC5424 (including structured data), rsyslog JSON and
39
- file formats, and systemd journal exports (JSON, short, and ISO formats).
58
+ file formats, systemd journal exports (JSON, short, and ISO formats),
59
+ Cisco console/buffer lines, ISO-timestamp syslog, and key=value streams
60
+ (Fortinet, Sophos). Unknown layouts are learned adaptively.
40
61
  - Deterministic vendor/product mappings bundled for Cisco ASA, Cisco IOS,
41
- F5 BIG-IP, generic Linux, and VMware ESXi, with automatic mapping
42
- selection based on message content.
43
- - Custom mappings supplied as JSON files or Python dictionaries.
62
+ F5 BIG-IP, Fortinet FortiGate, Sophos XG Firewall, VMware ESXi, and
63
+ generic Linux, with automatic mapping selection based on message content.
64
+ - Custom mappings supplied as JSON files or Python dictionaries, and custom
65
+ parsers supplied as regex pattern files.
66
+ - Validation of CEF extension values against the ArcSight extension
67
+ dictionary (`--validate`, `--strict`).
68
+ - Network receiver (`--listen udp:514`) and forwarding to UDP, TCP, or
69
+ Kafka (`--send`), with optional rate limiting.
44
70
  - Escaping of untrusted log content in both CEF header and extension
45
71
  fields, so crafted messages cannot forge header fields or split records.
46
72
  - Streaming CLI with stdin/file input, tail (follow) mode across multiple
@@ -129,13 +155,21 @@ Options:
129
155
  | Option | Description |
130
156
  | ------ | ----------- |
131
157
  | `paths` | Input files; stdin is used when omitted. |
132
- | `-o, --output FILE` | Write CEF lines to a file instead of stdout. |
133
- | `--mode MODE` | Parser override: `rfc3164`, `rfc5424`, `rsyslog_json`, `rsyslog_file`, `journald_json`, `journald_short`, `journald_iso`. Auto-detected when omitted. |
158
+ | `-o, --output FILE` | Write CEF lines to a file instead of stdout. Accepts strftime codes for timed files (see "Timed output files"). |
159
+ | `-a, --append` | Append to `--output` instead of truncating it (implied for templated paths). |
160
+ | `--mode MODE` | Parser override: `rfc3164`, `rfc5424`, `rsyslog_json`, `rsyslog_file`, `journald_json`, `journald_short`, `journald_iso`, `cisco_seq`, `iso_syslog`, `kv`, or a name from `--patterns`. Auto-detected when omitted. |
134
161
  | `--mapping FILE` | Mapping JSON file. Auto-selected from message content when omitted. |
162
+ | `--patterns FILE` | Custom parser patterns (see "Custom Parsers"); may be repeated. |
135
163
  | `--tail` | Follow input files like `tail -f`. |
164
+ | `--listen PROTO:PORT` | Receive syslog over the network instead of files/stdin: `udp:514`, `tcp:5514`, or `udp:10.0.0.5:514`. |
165
+ | `--send URL` | Forward CEF to `udp://HOST:PORT`, `tcp://HOST:PORT`, or `kafka://BROKER:PORT/TOPIC` (needs the `kafka` extra). |
166
+ | `--eps N` | Rate-limit `--send` to at most N events per second. |
136
167
  | `--multiprocess` | Convert lines using a process pool. |
137
168
  | `--pool-size N` | Worker count for `--multiprocess` (default: CPU count minus one). |
169
+ | `--validate` | Check CEF extensions against the ArcSight dictionary; violations are logged as warnings. |
170
+ | `--strict` | Like `--validate`, but exit with an error on type or length violations. |
138
171
  | `--log-level LEVEL` | Python logging level (default `WARNING`). |
172
+ | `--version` | Print the version, copyright, license and author, then exit. |
139
173
 
140
174
  `python -m syslogcef` is equivalent to the `syslogcef` entry point.
141
175
 
@@ -161,7 +195,8 @@ cef = to_cef(normalized, mapping="my_mapping.json")
161
195
  ```
162
196
 
163
197
  Bundled mappings are importable from `syslogcef.mappings` (`CISCO_ASA`,
164
- `CISCO_IOS`, `F5`, `LINUX`, `VMWARE`, or `load_mapping(name)`).
198
+ `CISCO_IOS`, `F5`, `FORTINET`, `LINUX`, `SOPHOS`, `VMWARE`, or
199
+ `load_mapping(name)`).
165
200
 
166
201
  ## Mapping Files
167
202
 
@@ -181,7 +216,7 @@ normalized event's fields:
181
216
  "src": "%(src)s",
182
217
  "dst": "%(dst)s",
183
218
  "cs1Label": "rawEvent",
184
- "cs1": "%(raw_kv)s"
219
+ "cs1": "%(raw)s"
185
220
  }
186
221
  }
187
222
  ```
@@ -191,15 +226,31 @@ normalized event's fields:
191
226
  - `severity_map` translates syslog severity (0-7) to CEF severity (0-10);
192
227
  unmapped values pass through.
193
228
  - `extensions` maps CEF extension keys to templates. Extensions that
194
- resolve to an empty value are omitted.
229
+ resolve to an empty value are omitted, and so is a custom-field label
230
+ (`cs1Label`, `cn1Label`, ...) whose paired value is omitted.
195
231
  - Available template fields include `host`, `app`, `pid`, `msgid`, `msg`,
196
232
  `message_short` (first 120 characters), `raw`, `raw_kv`, `event_code`,
197
233
  `facility`, `severity`, `ts`, plus every key=value pair extracted from
198
234
  the message and any RFC5424 structured-data or journald fields.
199
235
 
200
236
  Field templates that reference missing keys resolve to an empty string
201
- rather than failing the event. See [docs/cef_fields.md](docs/cef_fields.md)
202
- for the full CEF extension dictionary.
237
+ rather than failing the event. Mapping files and Python mapping dictionaries
238
+ are validated before rendering; malformed templates, severity maps, and CEF
239
+ extension keys fail with a configuration error instead of corrupting a record.
240
+ See [docs/cef_fields.md](docs/cef_fields.md) for the full CEF extension
241
+ dictionary.
242
+
243
+ ## Timestamps without a year
244
+
245
+ RFC3164, journald short, Cisco console lines and many device formats
246
+ stamp `Jan 1 12:34:56` with no year. syslogcef dates such a timestamp to
247
+ its most recent occurrence that is not more than one day ahead of the
248
+ current time: log lines describe events that already happened. A `Jan 1`
249
+ line converted in November is dated January of the current year, while a
250
+ line stamped just after midnight on New Year's Eve (a sender clock or
251
+ timezone slightly ahead) lands in the new year. Pass `now=` to
252
+ `parse_syslog`/`convert_line` to date archived files relative to when
253
+ they were written.
203
254
 
204
255
  ## Field Dictionary
205
256
 
@@ -217,7 +268,10 @@ knowledge, derived from the ArcSight Extension Dictionary (see
217
268
  are applied during normalization for **every** event — including
218
269
  key=value pairs extracted from adaptively-parsed unknown formats — so
219
270
  mappings and validation always see canonical names. Original keys are
220
- preserved and an explicit canonical key is never overwritten.
271
+ preserved and an explicit canonical key is never overwritten. Source key
272
+ casing is normalized as well (`SRCIP`, `SrcIp`, and `srcip` all make `src`
273
+ available), while parsed syslog envelope fields such as host and severity
274
+ remain authoritative over same-named text in the message.
221
275
 
222
276
  This means a Fortinet-style `srcip=10.1.1.1 dstport=443` and an unknown
223
277
  device emitting the same pairs behind an unrecognized prefix both end up
@@ -249,9 +303,9 @@ syslogcef acme.log --patterns /etc/syslogcef/patterns.json
249
303
  - Named groups map to event fields: `pri`, `host`, `app`, `pid`,
250
304
  `msgid`, `msg`, and `ts`.
251
305
  - A `ts` group requires `timestamp_format`: a strptime format,
252
- `iso8601`, or `epoch`. Yearless formats get the same year-rollover
253
- inference as the built-in parsers; an unparseable timestamp never
254
- drops the event.
306
+ `iso8601`, or `epoch`. Yearless formats get the same year inference
307
+ as the built-in parsers (see "Timestamps without a year" below); an
308
+ unparseable timestamp never drops the event.
255
309
  - `priority` is `"after"` (default: tried after the built-ins, before
256
310
  the adaptive fallback) or `"before"` (tried first, overriding
257
311
  built-ins).
@@ -382,6 +436,23 @@ pytest
382
436
  Contributions are welcome — see [CONTRIBUTING.md](CONTRIBUTING.md). Notable
383
437
  changes are tracked in [CHANGELOG.md](CHANGELOG.md).
384
438
 
439
+ ## Citation
440
+
441
+ If you use syslog2cef in research or published work, please cite it. The
442
+ metadata is in [CITATION.cff](CITATION.cff), and GitHub's "Cite this
443
+ repository" button produces APA and BibTeX from it.
444
+
445
+ ```bibtex
446
+ @software{suliman_syslog2cef,
447
+ author = {Suliman, Tamir},
448
+ title = {syslog2cef: Convert syslog to ArcSight Common Event Format (CEF)},
449
+ version = {0.4.0},
450
+ year = {2026},
451
+ url = {https://github.com/allamiro/syslog2cef},
452
+ license = {MIT}
453
+ }
454
+ ```
455
+
385
456
  ## License
386
457
 
387
458
  MIT — see [LICENSE](LICENSE). Copyright (c) Tamir Suliman.
@@ -3,6 +3,7 @@ README.md
3
3
  pyproject.toml
4
4
  ./syslogcef/__init__.py
5
5
  ./syslogcef/__main__.py
6
+ ./syslogcef/_version.py
6
7
  ./syslogcef/adaptive.py
7
8
  ./syslogcef/api.py
8
9
  ./syslogcef/cef.py
@@ -31,6 +32,7 @@ syslog2cef.egg-info/requires.txt
31
32
  syslog2cef.egg-info/top_level.txt
32
33
  syslogcef/__init__.py
33
34
  syslogcef/__main__.py
35
+ syslogcef/_version.py
34
36
  syslogcef/adaptive.py
35
37
  syslogcef/api.py
36
38
  syslogcef/cef.py
@@ -9,10 +9,16 @@ conversion workflow.
9
9
  """
10
10
  from __future__ import annotations
11
11
 
12
+ from ._version import __author__, __copyright__, __email__, __license__, __version__
12
13
  from .api import StreamConverter, convert_line, normalize_event, parse_syslog, to_cef
13
14
  from .custom import PatternFileError, load_patterns, register_parser
14
15
 
15
16
  __all__ = [
17
+ "__version__",
18
+ "__author__",
19
+ "__email__",
20
+ "__copyright__",
21
+ "__license__",
16
22
  "StreamConverter",
17
23
  "convert_line",
18
24
  "normalize_event",
@@ -0,0 +1,12 @@
1
+ """Package version and author, shown by ``syslogcef --version``.
2
+
3
+ Kept in step with ``version`` in pyproject.toml (checked by the test
4
+ suite). A module constant rather than importlib.metadata, because the
5
+ standalone .pyz has no installed distribution metadata to read.
6
+ """
7
+
8
+ __version__ = "0.4.0"
9
+ __author__ = "Tamir Suliman"
10
+ __email__ = "allamiro@gmail.com"
11
+ __copyright__ = "Copyright (c) 2023 Tamir Suliman"
12
+ __license__ = "MIT"