syslog2cef 0.3.3__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/PKG-INFO +78 -13
  2. syslog2cef-0.3.3/syslog2cef.egg-info/PKG-INFO → syslog2cef-0.4.0/README.md +71 -35
  3. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/pyproject.toml +7 -1
  4. syslog2cef-0.3.3/README.md → syslog2cef-0.4.0/syslog2cef.egg-info/PKG-INFO +100 -12
  5. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslog2cef.egg-info/SOURCES.txt +2 -0
  6. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/__init__.py +6 -0
  7. syslog2cef-0.4.0/syslogcef/_version.py +12 -0
  8. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/cef.py +27 -0
  9. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/cli.py +27 -1
  10. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/custom.py +6 -18
  11. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/mappings/cisco_asa.json +1 -1
  12. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/parsers.py +3 -13
  13. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/utils.py +28 -2
  14. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_cef.py +44 -0
  15. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_cli.py +43 -0
  16. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_custom_parsers.py +18 -1
  17. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_parsers_extra.py +44 -0
  18. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/LICENSE +0 -0
  19. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/setup.cfg +0 -0
  20. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslog2cef.egg-info/dependency_links.txt +0 -0
  21. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslog2cef.egg-info/entry_points.txt +0 -0
  22. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslog2cef.egg-info/requires.txt +0 -0
  23. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslog2cef.egg-info/top_level.txt +0 -0
  24. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/__main__.py +0 -0
  25. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/adaptive.py +0 -0
  26. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/api.py +0 -0
  27. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/dictionary.json +0 -0
  28. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/dictionary.py +0 -0
  29. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/mappings/__init__.py +0 -0
  30. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/mappings/cisco_ios.json +0 -0
  31. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/mappings/f5.json +0 -0
  32. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/mappings/fortinet.json +0 -0
  33. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/mappings/linux.json +0 -0
  34. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/mappings/sophos.json +0 -0
  35. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/mappings/vmware.json +0 -0
  36. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/net.py +0 -0
  37. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/normalizer.py +0 -0
  38. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/validation.py +0 -0
  39. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_adaptive.py +0 -0
  40. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_cli_validation.py +0 -0
  41. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_dictionary.py +0 -0
  42. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_fuzz_grammar.py +0 -0
  43. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_fuzz_properties.py +0 -0
  44. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_macos_logs.py +0 -0
  45. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_net.py +0 -0
  46. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_normalizer.py +0 -0
  47. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_parsers.py +0 -0
  48. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_pipeline.py +0 -0
  49. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_real_world.py +0 -0
  50. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_rfc5424_epoch.py +0 -0
  51. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_security_hardening.py +0 -0
  52. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_tail_rotation.py +0 -0
  53. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_utils.py +0 -0
  54. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_validation.py +0 -0
  55. {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_zipapp_safe.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: syslog2cef
3
- Version: 0.3.3
3
+ Version: 0.4.0
4
4
  Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
5
  Author-email: Tamir Suliman <allamiro@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/allamiro/syslogcef
@@ -8,6 +8,12 @@ Project-URL: Issues, https://github.com/allamiro/syslogcef/issues
8
8
  Keywords: syslog,cef,logging,security
9
9
  Classifier: Programming Language :: Python :: 3
10
10
  Classifier: Programming Language :: Python :: 3 :: Only
11
+ Classifier: Programming Language :: Python :: 3.9
12
+ Classifier: Programming Language :: Python :: 3.10
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Programming Language :: Python :: 3.13
16
+ Classifier: Programming Language :: Python :: 3.14
11
17
  Classifier: License :: OSI Approved :: MIT License
12
18
  Classifier: Topic :: System :: Logging
13
19
  Requires-Python: >=3.9
@@ -25,6 +31,19 @@ Dynamic: license-file
25
31
  <img src="https://raw.githubusercontent.com/allamiro/syslogcef/main/docs/logo.svg" alt="syslogcef logo" width="540">
26
32
  </p>
27
33
 
34
+ <p align="center">
35
+ <a href="https://github.com/allamiro/syslog2cef/actions/workflows/ci.yml"><img src="https://img.shields.io/github/actions/workflow/status/allamiro/syslog2cef/ci.yml?branch=main&label=CI" alt="CI status"></a>
36
+ <a href="https://pypi.org/project/syslog2cef/"><img src="https://img.shields.io/pypi/v/syslog2cef" alt="PyPI version"></a>
37
+ <a href="https://pypi.org/project/syslog2cef/"><img src="https://img.shields.io/pypi/pyversions/syslog2cef" alt="Supported Python versions"></a>
38
+ <a href="https://pypistats.org/packages/syslog2cef"><img src="https://img.shields.io/pypi/dm/syslog2cef" alt="PyPI downloads per month"></a>
39
+ <a href="https://copr.fedorainfracloud.org/coprs/allamiro/syslogcef/"><img src="https://copr.fedorainfracloud.org/coprs/allamiro/syslogcef/package/syslogcef/status_image/last_build.png" alt="COPR build status"></a>
40
+ <a href="https://github.com/allamiro/syslog2cef/pkgs/container/syslogcef"><img src="https://img.shields.io/badge/container-ghcr.io-blue?logo=docker" alt="Container image on ghcr.io"></a>
41
+ <a href="https://github.com/allamiro/syslog2cef/actions/workflows/fuzz.yml"><img src="https://img.shields.io/github/actions/workflow/status/allamiro/syslog2cef/fuzz.yml?branch=main&label=deep%20fuzz" alt="Deep fuzz status"></a>
42
+ <a href="https://github.com/astral-sh/ruff"><img src="https://img.shields.io/badge/lint-ruff-261230" alt="Linted with Ruff"></a>
43
+ <a href="LICENSE"><img src="https://img.shields.io/github/license/allamiro/syslog2cef" alt="MIT license"></a>
44
+ <a href="CITATION.cff"><img src="https://img.shields.io/badge/cite-CITATION.cff-informational" alt="Citation file"></a>
45
+ </p>
46
+
28
47
  # syslogcef
29
48
 
30
49
  syslogcef converts syslog events into ArcSight Common Event Format (CEF). It
@@ -36,11 +55,18 @@ platforms that consume CEF.
36
55
 
37
56
  - Automatic detection and parsing of multiple syslog dialects:
38
57
  RFC3164 (BSD), RFC5424 (including structured data), rsyslog JSON and
39
- file formats, and systemd journal exports (JSON, short, and ISO formats).
58
+ file formats, systemd journal exports (JSON, short, and ISO formats),
59
+ Cisco console/buffer lines, ISO-timestamp syslog, and key=value streams
60
+ (Fortinet, Sophos). Unknown layouts are learned adaptively.
40
61
  - Deterministic vendor/product mappings bundled for Cisco ASA, Cisco IOS,
41
- F5 BIG-IP, generic Linux, and VMware ESXi, with automatic mapping
42
- selection based on message content.
43
- - Custom mappings supplied as JSON files or Python dictionaries.
62
+ F5 BIG-IP, Fortinet FortiGate, Sophos XG Firewall, VMware ESXi, and
63
+ generic Linux, with automatic mapping selection based on message content.
64
+ - Custom mappings supplied as JSON files or Python dictionaries, and custom
65
+ parsers supplied as regex pattern files.
66
+ - Validation of CEF extension values against the ArcSight extension
67
+ dictionary (`--validate`, `--strict`).
68
+ - Network receiver (`--listen udp:514`) and forwarding to UDP, TCP, or
69
+ Kafka (`--send`), with optional rate limiting.
44
70
  - Escaping of untrusted log content in both CEF header and extension
45
71
  fields, so crafted messages cannot forge header fields or split records.
46
72
  - Streaming CLI with stdin/file input, tail (follow) mode across multiple
@@ -129,13 +155,21 @@ Options:
129
155
  | Option | Description |
130
156
  | ------ | ----------- |
131
157
  | `paths` | Input files; stdin is used when omitted. |
132
- | `-o, --output FILE` | Write CEF lines to a file instead of stdout. |
133
- | `--mode MODE` | Parser override: `rfc3164`, `rfc5424`, `rsyslog_json`, `rsyslog_file`, `journald_json`, `journald_short`, `journald_iso`. Auto-detected when omitted. |
158
+ | `-o, --output FILE` | Write CEF lines to a file instead of stdout. Accepts strftime codes for timed files (see "Timed output files"). |
159
+ | `-a, --append` | Append to `--output` instead of truncating it (implied for templated paths). |
160
+ | `--mode MODE` | Parser override: `rfc3164`, `rfc5424`, `rsyslog_json`, `rsyslog_file`, `journald_json`, `journald_short`, `journald_iso`, `cisco_seq`, `iso_syslog`, `kv`, or a name from `--patterns`. Auto-detected when omitted. |
134
161
  | `--mapping FILE` | Mapping JSON file. Auto-selected from message content when omitted. |
162
+ | `--patterns FILE` | Custom parser patterns (see "Custom Parsers"); may be repeated. |
135
163
  | `--tail` | Follow input files like `tail -f`. |
164
+ | `--listen PROTO:PORT` | Receive syslog over the network instead of files/stdin: `udp:514`, `tcp:5514`, or `udp:10.0.0.5:514`. |
165
+ | `--send URL` | Forward CEF to `udp://HOST:PORT`, `tcp://HOST:PORT`, or `kafka://BROKER:PORT/TOPIC` (needs the `kafka` extra). |
166
+ | `--eps N` | Rate-limit `--send` to at most N events per second. |
136
167
  | `--multiprocess` | Convert lines using a process pool. |
137
168
  | `--pool-size N` | Worker count for `--multiprocess` (default: CPU count minus one). |
169
+ | `--validate` | Check CEF extensions against the ArcSight dictionary; violations are logged as warnings. |
170
+ | `--strict` | Like `--validate`, but exit with an error on type or length violations. |
138
171
  | `--log-level LEVEL` | Python logging level (default `WARNING`). |
172
+ | `--version` | Print the version, copyright, license and author, then exit. |
139
173
 
140
174
  `python -m syslogcef` is equivalent to the `syslogcef` entry point.
141
175
 
@@ -161,7 +195,8 @@ cef = to_cef(normalized, mapping="my_mapping.json")
161
195
  ```
162
196
 
163
197
  Bundled mappings are importable from `syslogcef.mappings` (`CISCO_ASA`,
164
- `CISCO_IOS`, `F5`, `LINUX`, `VMWARE`, or `load_mapping(name)`).
198
+ `CISCO_IOS`, `F5`, `FORTINET`, `LINUX`, `SOPHOS`, `VMWARE`, or
199
+ `load_mapping(name)`).
165
200
 
166
201
  ## Mapping Files
167
202
 
@@ -181,7 +216,7 @@ normalized event's fields:
181
216
  "src": "%(src)s",
182
217
  "dst": "%(dst)s",
183
218
  "cs1Label": "rawEvent",
184
- "cs1": "%(raw_kv)s"
219
+ "cs1": "%(raw)s"
185
220
  }
186
221
  }
187
222
  ```
@@ -191,7 +226,8 @@ normalized event's fields:
191
226
  - `severity_map` translates syslog severity (0-7) to CEF severity (0-10);
192
227
  unmapped values pass through.
193
228
  - `extensions` maps CEF extension keys to templates. Extensions that
194
- resolve to an empty value are omitted.
229
+ resolve to an empty value are omitted, and so is a custom-field label
230
+ (`cs1Label`, `cn1Label`, ...) whose paired value is omitted.
195
231
  - Available template fields include `host`, `app`, `pid`, `msgid`, `msg`,
196
232
  `message_short` (first 120 characters), `raw`, `raw_kv`, `event_code`,
197
233
  `facility`, `severity`, `ts`, plus every key=value pair extracted from
@@ -204,6 +240,18 @@ extension keys fail with a configuration error instead of corrupting a record.
204
240
  See [docs/cef_fields.md](docs/cef_fields.md) for the full CEF extension
205
241
  dictionary.
206
242
 
243
+ ## Timestamps without a year
244
+
245
+ RFC3164, journald short, Cisco console lines and many device formats
246
+ stamp `Jan 1 12:34:56` with no year. syslogcef dates such a timestamp to
247
+ its most recent occurrence that is not more than one day ahead of the
248
+ current time: log lines describe events that already happened. A `Jan 1`
249
+ line converted in November is dated January of the current year, while a
250
+ line stamped just after midnight on New Year's Eve (a sender clock or
251
+ timezone slightly ahead) lands in the new year. Pass `now=` to
252
+ `parse_syslog`/`convert_line` to date archived files relative to when
253
+ they were written.
254
+
207
255
  ## Field Dictionary
208
256
 
209
257
  `syslogcef/dictionary.json` is the single source of truth for CEF field
@@ -255,9 +303,9 @@ syslogcef acme.log --patterns /etc/syslogcef/patterns.json
255
303
  - Named groups map to event fields: `pri`, `host`, `app`, `pid`,
256
304
  `msgid`, `msg`, and `ts`.
257
305
  - A `ts` group requires `timestamp_format`: a strptime format,
258
- `iso8601`, or `epoch`. Yearless formats get the same year-rollover
259
- inference as the built-in parsers; an unparseable timestamp never
260
- drops the event.
306
+ `iso8601`, or `epoch`. Yearless formats get the same year inference
307
+ as the built-in parsers (see "Timestamps without a year" below); an
308
+ unparseable timestamp never drops the event.
261
309
  - `priority` is `"after"` (default: tried after the built-ins, before
262
310
  the adaptive fallback) or `"before"` (tried first, overriding
263
311
  built-ins).
@@ -388,6 +436,23 @@ pytest
388
436
  Contributions are welcome — see [CONTRIBUTING.md](CONTRIBUTING.md). Notable
389
437
  changes are tracked in [CHANGELOG.md](CHANGELOG.md).
390
438
 
439
+ ## Citation
440
+
441
+ If you use syslog2cef in research or published work, please cite it. The
442
+ metadata is in [CITATION.cff](CITATION.cff), and GitHub's "Cite this
443
+ repository" button produces APA and BibTeX from it.
444
+
445
+ ```bibtex
446
+ @software{suliman_syslog2cef,
447
+ author = {Suliman, Tamir},
448
+ title = {syslog2cef: Convert syslog to ArcSight Common Event Format (CEF)},
449
+ version = {0.4.0},
450
+ year = {2026},
451
+ url = {https://github.com/allamiro/syslog2cef},
452
+ license = {MIT}
453
+ }
454
+ ```
455
+
391
456
  ## License
392
457
 
393
458
  MIT — see [LICENSE](LICENSE). Copyright (c) Tamir Suliman.
@@ -1,30 +1,20 @@
1
- Metadata-Version: 2.4
2
- Name: syslog2cef
3
- Version: 0.3.3
4
- Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
- Author-email: Tamir Suliman <allamiro@gmail.com>
6
- Project-URL: Homepage, https://github.com/allamiro/syslogcef
7
- Project-URL: Issues, https://github.com/allamiro/syslogcef/issues
8
- Keywords: syslog,cef,logging,security
9
- Classifier: Programming Language :: Python :: 3
10
- Classifier: Programming Language :: Python :: 3 :: Only
11
- Classifier: License :: OSI Approved :: MIT License
12
- Classifier: Topic :: System :: Logging
13
- Requires-Python: >=3.9
14
- Description-Content-Type: text/markdown
15
- License-File: LICENSE
16
- Provides-Extra: test
17
- Requires-Dist: pytest>=7; extra == "test"
18
- Provides-Extra: kafka
19
- Requires-Dist: kafka-python>=2.0; extra == "kafka"
20
- Provides-Extra: fuzz
21
- Requires-Dist: hypothesis>=6; extra == "fuzz"
22
- Dynamic: license-file
23
-
24
1
  <p align="center">
25
2
  <img src="https://raw.githubusercontent.com/allamiro/syslogcef/main/docs/logo.svg" alt="syslogcef logo" width="540">
26
3
  </p>
27
4
 
5
+ <p align="center">
6
+ <a href="https://github.com/allamiro/syslog2cef/actions/workflows/ci.yml"><img src="https://img.shields.io/github/actions/workflow/status/allamiro/syslog2cef/ci.yml?branch=main&label=CI" alt="CI status"></a>
7
+ <a href="https://pypi.org/project/syslog2cef/"><img src="https://img.shields.io/pypi/v/syslog2cef" alt="PyPI version"></a>
8
+ <a href="https://pypi.org/project/syslog2cef/"><img src="https://img.shields.io/pypi/pyversions/syslog2cef" alt="Supported Python versions"></a>
9
+ <a href="https://pypistats.org/packages/syslog2cef"><img src="https://img.shields.io/pypi/dm/syslog2cef" alt="PyPI downloads per month"></a>
10
+ <a href="https://copr.fedorainfracloud.org/coprs/allamiro/syslogcef/"><img src="https://copr.fedorainfracloud.org/coprs/allamiro/syslogcef/package/syslogcef/status_image/last_build.png" alt="COPR build status"></a>
11
+ <a href="https://github.com/allamiro/syslog2cef/pkgs/container/syslogcef"><img src="https://img.shields.io/badge/container-ghcr.io-blue?logo=docker" alt="Container image on ghcr.io"></a>
12
+ <a href="https://github.com/allamiro/syslog2cef/actions/workflows/fuzz.yml"><img src="https://img.shields.io/github/actions/workflow/status/allamiro/syslog2cef/fuzz.yml?branch=main&label=deep%20fuzz" alt="Deep fuzz status"></a>
13
+ <a href="https://github.com/astral-sh/ruff"><img src="https://img.shields.io/badge/lint-ruff-261230" alt="Linted with Ruff"></a>
14
+ <a href="LICENSE"><img src="https://img.shields.io/github/license/allamiro/syslog2cef" alt="MIT license"></a>
15
+ <a href="CITATION.cff"><img src="https://img.shields.io/badge/cite-CITATION.cff-informational" alt="Citation file"></a>
16
+ </p>
17
+
28
18
  # syslogcef
29
19
 
30
20
  syslogcef converts syslog events into ArcSight Common Event Format (CEF). It
@@ -36,11 +26,18 @@ platforms that consume CEF.
36
26
 
37
27
  - Automatic detection and parsing of multiple syslog dialects:
38
28
  RFC3164 (BSD), RFC5424 (including structured data), rsyslog JSON and
39
- file formats, and systemd journal exports (JSON, short, and ISO formats).
29
+ file formats, systemd journal exports (JSON, short, and ISO formats),
30
+ Cisco console/buffer lines, ISO-timestamp syslog, and key=value streams
31
+ (Fortinet, Sophos). Unknown layouts are learned adaptively.
40
32
  - Deterministic vendor/product mappings bundled for Cisco ASA, Cisco IOS,
41
- F5 BIG-IP, generic Linux, and VMware ESXi, with automatic mapping
42
- selection based on message content.
43
- - Custom mappings supplied as JSON files or Python dictionaries.
33
+ F5 BIG-IP, Fortinet FortiGate, Sophos XG Firewall, VMware ESXi, and
34
+ generic Linux, with automatic mapping selection based on message content.
35
+ - Custom mappings supplied as JSON files or Python dictionaries, and custom
36
+ parsers supplied as regex pattern files.
37
+ - Validation of CEF extension values against the ArcSight extension
38
+ dictionary (`--validate`, `--strict`).
39
+ - Network receiver (`--listen udp:514`) and forwarding to UDP, TCP, or
40
+ Kafka (`--send`), with optional rate limiting.
44
41
  - Escaping of untrusted log content in both CEF header and extension
45
42
  fields, so crafted messages cannot forge header fields or split records.
46
43
  - Streaming CLI with stdin/file input, tail (follow) mode across multiple
@@ -129,13 +126,21 @@ Options:
129
126
  | Option | Description |
130
127
  | ------ | ----------- |
131
128
  | `paths` | Input files; stdin is used when omitted. |
132
- | `-o, --output FILE` | Write CEF lines to a file instead of stdout. |
133
- | `--mode MODE` | Parser override: `rfc3164`, `rfc5424`, `rsyslog_json`, `rsyslog_file`, `journald_json`, `journald_short`, `journald_iso`. Auto-detected when omitted. |
129
+ | `-o, --output FILE` | Write CEF lines to a file instead of stdout. Accepts strftime codes for timed files (see "Timed output files"). |
130
+ | `-a, --append` | Append to `--output` instead of truncating it (implied for templated paths). |
131
+ | `--mode MODE` | Parser override: `rfc3164`, `rfc5424`, `rsyslog_json`, `rsyslog_file`, `journald_json`, `journald_short`, `journald_iso`, `cisco_seq`, `iso_syslog`, `kv`, or a name from `--patterns`. Auto-detected when omitted. |
134
132
  | `--mapping FILE` | Mapping JSON file. Auto-selected from message content when omitted. |
133
+ | `--patterns FILE` | Custom parser patterns (see "Custom Parsers"); may be repeated. |
135
134
  | `--tail` | Follow input files like `tail -f`. |
135
+ | `--listen PROTO:PORT` | Receive syslog over the network instead of files/stdin: `udp:514`, `tcp:5514`, or `udp:10.0.0.5:514`. |
136
+ | `--send URL` | Forward CEF to `udp://HOST:PORT`, `tcp://HOST:PORT`, or `kafka://BROKER:PORT/TOPIC` (needs the `kafka` extra). |
137
+ | `--eps N` | Rate-limit `--send` to at most N events per second. |
136
138
  | `--multiprocess` | Convert lines using a process pool. |
137
139
  | `--pool-size N` | Worker count for `--multiprocess` (default: CPU count minus one). |
140
+ | `--validate` | Check CEF extensions against the ArcSight dictionary; violations are logged as warnings. |
141
+ | `--strict` | Like `--validate`, but exit with an error on type or length violations. |
138
142
  | `--log-level LEVEL` | Python logging level (default `WARNING`). |
143
+ | `--version` | Print the version, copyright, license and author, then exit. |
139
144
 
140
145
  `python -m syslogcef` is equivalent to the `syslogcef` entry point.
141
146
 
@@ -161,7 +166,8 @@ cef = to_cef(normalized, mapping="my_mapping.json")
161
166
  ```
162
167
 
163
168
  Bundled mappings are importable from `syslogcef.mappings` (`CISCO_ASA`,
164
- `CISCO_IOS`, `F5`, `LINUX`, `VMWARE`, or `load_mapping(name)`).
169
+ `CISCO_IOS`, `F5`, `FORTINET`, `LINUX`, `SOPHOS`, `VMWARE`, or
170
+ `load_mapping(name)`).
165
171
 
166
172
  ## Mapping Files
167
173
 
@@ -181,7 +187,7 @@ normalized event's fields:
181
187
  "src": "%(src)s",
182
188
  "dst": "%(dst)s",
183
189
  "cs1Label": "rawEvent",
184
- "cs1": "%(raw_kv)s"
190
+ "cs1": "%(raw)s"
185
191
  }
186
192
  }
187
193
  ```
@@ -191,7 +197,8 @@ normalized event's fields:
191
197
  - `severity_map` translates syslog severity (0-7) to CEF severity (0-10);
192
198
  unmapped values pass through.
193
199
  - `extensions` maps CEF extension keys to templates. Extensions that
194
- resolve to an empty value are omitted.
200
+ resolve to an empty value are omitted, and so is a custom-field label
201
+ (`cs1Label`, `cn1Label`, ...) whose paired value is omitted.
195
202
  - Available template fields include `host`, `app`, `pid`, `msgid`, `msg`,
196
203
  `message_short` (first 120 characters), `raw`, `raw_kv`, `event_code`,
197
204
  `facility`, `severity`, `ts`, plus every key=value pair extracted from
@@ -204,6 +211,18 @@ extension keys fail with a configuration error instead of corrupting a record.
204
211
  See [docs/cef_fields.md](docs/cef_fields.md) for the full CEF extension
205
212
  dictionary.
206
213
 
214
+ ## Timestamps without a year
215
+
216
+ RFC3164, journald short, Cisco console lines and many device formats
217
+ stamp `Jan 1 12:34:56` with no year. syslogcef dates such a timestamp to
218
+ its most recent occurrence that is not more than one day ahead of the
219
+ current time: log lines describe events that already happened. A `Jan 1`
220
+ line converted in November is dated January of the current year, while a
221
+ line stamped just after midnight on New Year's Eve (a sender clock or
222
+ timezone slightly ahead) lands in the new year. Pass `now=` to
223
+ `parse_syslog`/`convert_line` to date archived files relative to when
224
+ they were written.
225
+
207
226
  ## Field Dictionary
208
227
 
209
228
  `syslogcef/dictionary.json` is the single source of truth for CEF field
@@ -255,9 +274,9 @@ syslogcef acme.log --patterns /etc/syslogcef/patterns.json
255
274
  - Named groups map to event fields: `pri`, `host`, `app`, `pid`,
256
275
  `msgid`, `msg`, and `ts`.
257
276
  - A `ts` group requires `timestamp_format`: a strptime format,
258
- `iso8601`, or `epoch`. Yearless formats get the same year-rollover
259
- inference as the built-in parsers; an unparseable timestamp never
260
- drops the event.
277
+ `iso8601`, or `epoch`. Yearless formats get the same year inference
278
+ as the built-in parsers (see "Timestamps without a year" below); an
279
+ unparseable timestamp never drops the event.
261
280
  - `priority` is `"after"` (default: tried after the built-ins, before
262
281
  the adaptive fallback) or `"before"` (tried first, overriding
263
282
  built-ins).
@@ -388,6 +407,23 @@ pytest
388
407
  Contributions are welcome — see [CONTRIBUTING.md](CONTRIBUTING.md). Notable
389
408
  changes are tracked in [CHANGELOG.md](CHANGELOG.md).
390
409
 
410
+ ## Citation
411
+
412
+ If you use syslog2cef in research or published work, please cite it. The
413
+ metadata is in [CITATION.cff](CITATION.cff), and GitHub's "Cite this
414
+ repository" button produces APA and BibTeX from it.
415
+
416
+ ```bibtex
417
+ @software{suliman_syslog2cef,
418
+ author = {Suliman, Tamir},
419
+ title = {syslog2cef: Convert syslog to ArcSight Common Event Format (CEF)},
420
+ version = {0.4.0},
421
+ year = {2026},
422
+ url = {https://github.com/allamiro/syslog2cef},
423
+ license = {MIT}
424
+ }
425
+ ```
426
+
391
427
  ## License
392
428
 
393
429
  MIT — see [LICENSE](LICENSE). Copyright (c) Tamir Suliman.
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "syslog2cef"
7
- version = "0.3.3"
7
+ version = "0.4.0"
8
8
  description = "Convert RFC3164/RFC5424 syslog lines into ArcSight CEF"
9
9
  authors = [{name = "Tamir Suliman", email = "allamiro@gmail.com"}]
10
10
  readme = "README.md"
@@ -12,6 +12,12 @@ requires-python = ">=3.9"
12
12
  classifiers = [
13
13
  "Programming Language :: Python :: 3",
14
14
  "Programming Language :: Python :: 3 :: Only",
15
+ "Programming Language :: Python :: 3.9",
16
+ "Programming Language :: Python :: 3.10",
17
+ "Programming Language :: Python :: 3.11",
18
+ "Programming Language :: Python :: 3.12",
19
+ "Programming Language :: Python :: 3.13",
20
+ "Programming Language :: Python :: 3.14",
15
21
  "License :: OSI Approved :: MIT License",
16
22
  "Topic :: System :: Logging",
17
23
  ]
@@ -1,7 +1,49 @@
1
+ Metadata-Version: 2.4
2
+ Name: syslog2cef
3
+ Version: 0.4.0
4
+ Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
+ Author-email: Tamir Suliman <allamiro@gmail.com>
6
+ Project-URL: Homepage, https://github.com/allamiro/syslogcef
7
+ Project-URL: Issues, https://github.com/allamiro/syslogcef/issues
8
+ Keywords: syslog,cef,logging,security
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: Programming Language :: Python :: 3 :: Only
11
+ Classifier: Programming Language :: Python :: 3.9
12
+ Classifier: Programming Language :: Python :: 3.10
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Programming Language :: Python :: 3.13
16
+ Classifier: Programming Language :: Python :: 3.14
17
+ Classifier: License :: OSI Approved :: MIT License
18
+ Classifier: Topic :: System :: Logging
19
+ Requires-Python: >=3.9
20
+ Description-Content-Type: text/markdown
21
+ License-File: LICENSE
22
+ Provides-Extra: test
23
+ Requires-Dist: pytest>=7; extra == "test"
24
+ Provides-Extra: kafka
25
+ Requires-Dist: kafka-python>=2.0; extra == "kafka"
26
+ Provides-Extra: fuzz
27
+ Requires-Dist: hypothesis>=6; extra == "fuzz"
28
+ Dynamic: license-file
29
+
1
30
  <p align="center">
2
31
  <img src="https://raw.githubusercontent.com/allamiro/syslogcef/main/docs/logo.svg" alt="syslogcef logo" width="540">
3
32
  </p>
4
33
 
34
+ <p align="center">
35
+ <a href="https://github.com/allamiro/syslog2cef/actions/workflows/ci.yml"><img src="https://img.shields.io/github/actions/workflow/status/allamiro/syslog2cef/ci.yml?branch=main&label=CI" alt="CI status"></a>
36
+ <a href="https://pypi.org/project/syslog2cef/"><img src="https://img.shields.io/pypi/v/syslog2cef" alt="PyPI version"></a>
37
+ <a href="https://pypi.org/project/syslog2cef/"><img src="https://img.shields.io/pypi/pyversions/syslog2cef" alt="Supported Python versions"></a>
38
+ <a href="https://pypistats.org/packages/syslog2cef"><img src="https://img.shields.io/pypi/dm/syslog2cef" alt="PyPI downloads per month"></a>
39
+ <a href="https://copr.fedorainfracloud.org/coprs/allamiro/syslogcef/"><img src="https://copr.fedorainfracloud.org/coprs/allamiro/syslogcef/package/syslogcef/status_image/last_build.png" alt="COPR build status"></a>
40
+ <a href="https://github.com/allamiro/syslog2cef/pkgs/container/syslogcef"><img src="https://img.shields.io/badge/container-ghcr.io-blue?logo=docker" alt="Container image on ghcr.io"></a>
41
+ <a href="https://github.com/allamiro/syslog2cef/actions/workflows/fuzz.yml"><img src="https://img.shields.io/github/actions/workflow/status/allamiro/syslog2cef/fuzz.yml?branch=main&label=deep%20fuzz" alt="Deep fuzz status"></a>
42
+ <a href="https://github.com/astral-sh/ruff"><img src="https://img.shields.io/badge/lint-ruff-261230" alt="Linted with Ruff"></a>
43
+ <a href="LICENSE"><img src="https://img.shields.io/github/license/allamiro/syslog2cef" alt="MIT license"></a>
44
+ <a href="CITATION.cff"><img src="https://img.shields.io/badge/cite-CITATION.cff-informational" alt="Citation file"></a>
45
+ </p>
46
+
5
47
  # syslogcef
6
48
 
7
49
  syslogcef converts syslog events into ArcSight Common Event Format (CEF). It
@@ -13,11 +55,18 @@ platforms that consume CEF.
13
55
 
14
56
  - Automatic detection and parsing of multiple syslog dialects:
15
57
  RFC3164 (BSD), RFC5424 (including structured data), rsyslog JSON and
16
- file formats, and systemd journal exports (JSON, short, and ISO formats).
58
+ file formats, systemd journal exports (JSON, short, and ISO formats),
59
+ Cisco console/buffer lines, ISO-timestamp syslog, and key=value streams
60
+ (Fortinet, Sophos). Unknown layouts are learned adaptively.
17
61
  - Deterministic vendor/product mappings bundled for Cisco ASA, Cisco IOS,
18
- F5 BIG-IP, generic Linux, and VMware ESXi, with automatic mapping
19
- selection based on message content.
20
- - Custom mappings supplied as JSON files or Python dictionaries.
62
+ F5 BIG-IP, Fortinet FortiGate, Sophos XG Firewall, VMware ESXi, and
63
+ generic Linux, with automatic mapping selection based on message content.
64
+ - Custom mappings supplied as JSON files or Python dictionaries, and custom
65
+ parsers supplied as regex pattern files.
66
+ - Validation of CEF extension values against the ArcSight extension
67
+ dictionary (`--validate`, `--strict`).
68
+ - Network receiver (`--listen udp:514`) and forwarding to UDP, TCP, or
69
+ Kafka (`--send`), with optional rate limiting.
21
70
  - Escaping of untrusted log content in both CEF header and extension
22
71
  fields, so crafted messages cannot forge header fields or split records.
23
72
  - Streaming CLI with stdin/file input, tail (follow) mode across multiple
@@ -106,13 +155,21 @@ Options:
106
155
  | Option | Description |
107
156
  | ------ | ----------- |
108
157
  | `paths` | Input files; stdin is used when omitted. |
109
- | `-o, --output FILE` | Write CEF lines to a file instead of stdout. |
110
- | `--mode MODE` | Parser override: `rfc3164`, `rfc5424`, `rsyslog_json`, `rsyslog_file`, `journald_json`, `journald_short`, `journald_iso`. Auto-detected when omitted. |
158
+ | `-o, --output FILE` | Write CEF lines to a file instead of stdout. Accepts strftime codes for timed files (see "Timed output files"). |
159
+ | `-a, --append` | Append to `--output` instead of truncating it (implied for templated paths). |
160
+ | `--mode MODE` | Parser override: `rfc3164`, `rfc5424`, `rsyslog_json`, `rsyslog_file`, `journald_json`, `journald_short`, `journald_iso`, `cisco_seq`, `iso_syslog`, `kv`, or a name from `--patterns`. Auto-detected when omitted. |
111
161
  | `--mapping FILE` | Mapping JSON file. Auto-selected from message content when omitted. |
162
+ | `--patterns FILE` | Custom parser patterns (see "Custom Parsers"); may be repeated. |
112
163
  | `--tail` | Follow input files like `tail -f`. |
164
+ | `--listen PROTO:PORT` | Receive syslog over the network instead of files/stdin: `udp:514`, `tcp:5514`, or `udp:10.0.0.5:514`. |
165
+ | `--send URL` | Forward CEF to `udp://HOST:PORT`, `tcp://HOST:PORT`, or `kafka://BROKER:PORT/TOPIC` (needs the `kafka` extra). |
166
+ | `--eps N` | Rate-limit `--send` to at most N events per second. |
113
167
  | `--multiprocess` | Convert lines using a process pool. |
114
168
  | `--pool-size N` | Worker count for `--multiprocess` (default: CPU count minus one). |
169
+ | `--validate` | Check CEF extensions against the ArcSight dictionary; violations are logged as warnings. |
170
+ | `--strict` | Like `--validate`, but exit with an error on type or length violations. |
115
171
  | `--log-level LEVEL` | Python logging level (default `WARNING`). |
172
+ | `--version` | Print the version, copyright, license and author, then exit. |
116
173
 
117
174
  `python -m syslogcef` is equivalent to the `syslogcef` entry point.
118
175
 
@@ -138,7 +195,8 @@ cef = to_cef(normalized, mapping="my_mapping.json")
138
195
  ```
139
196
 
140
197
  Bundled mappings are importable from `syslogcef.mappings` (`CISCO_ASA`,
141
- `CISCO_IOS`, `F5`, `LINUX`, `VMWARE`, or `load_mapping(name)`).
198
+ `CISCO_IOS`, `F5`, `FORTINET`, `LINUX`, `SOPHOS`, `VMWARE`, or
199
+ `load_mapping(name)`).
142
200
 
143
201
  ## Mapping Files
144
202
 
@@ -158,7 +216,7 @@ normalized event's fields:
158
216
  "src": "%(src)s",
159
217
  "dst": "%(dst)s",
160
218
  "cs1Label": "rawEvent",
161
- "cs1": "%(raw_kv)s"
219
+ "cs1": "%(raw)s"
162
220
  }
163
221
  }
164
222
  ```
@@ -168,7 +226,8 @@ normalized event's fields:
168
226
  - `severity_map` translates syslog severity (0-7) to CEF severity (0-10);
169
227
  unmapped values pass through.
170
228
  - `extensions` maps CEF extension keys to templates. Extensions that
171
- resolve to an empty value are omitted.
229
+ resolve to an empty value are omitted, and so is a custom-field label
230
+ (`cs1Label`, `cn1Label`, ...) whose paired value is omitted.
172
231
  - Available template fields include `host`, `app`, `pid`, `msgid`, `msg`,
173
232
  `message_short` (first 120 characters), `raw`, `raw_kv`, `event_code`,
174
233
  `facility`, `severity`, `ts`, plus every key=value pair extracted from
@@ -181,6 +240,18 @@ extension keys fail with a configuration error instead of corrupting a record.
181
240
  See [docs/cef_fields.md](docs/cef_fields.md) for the full CEF extension
182
241
  dictionary.
183
242
 
243
+ ## Timestamps without a year
244
+
245
+ RFC3164, journald short, Cisco console lines and many device formats
246
+ stamp `Jan 1 12:34:56` with no year. syslogcef dates such a timestamp to
247
+ its most recent occurrence that is not more than one day ahead of the
248
+ current time: log lines describe events that already happened. A `Jan 1`
249
+ line converted in November is dated January of the current year, while a
250
+ line stamped just after midnight on New Year's Eve (a sender clock or
251
+ timezone slightly ahead) lands in the new year. Pass `now=` to
252
+ `parse_syslog`/`convert_line` to date archived files relative to when
253
+ they were written.
254
+
184
255
  ## Field Dictionary
185
256
 
186
257
  `syslogcef/dictionary.json` is the single source of truth for CEF field
@@ -232,9 +303,9 @@ syslogcef acme.log --patterns /etc/syslogcef/patterns.json
232
303
  - Named groups map to event fields: `pri`, `host`, `app`, `pid`,
233
304
  `msgid`, `msg`, and `ts`.
234
305
  - A `ts` group requires `timestamp_format`: a strptime format,
235
- `iso8601`, or `epoch`. Yearless formats get the same year-rollover
236
- inference as the built-in parsers; an unparseable timestamp never
237
- drops the event.
306
+ `iso8601`, or `epoch`. Yearless formats get the same year inference
307
+ as the built-in parsers (see "Timestamps without a year" below); an
308
+ unparseable timestamp never drops the event.
238
309
  - `priority` is `"after"` (default: tried after the built-ins, before
239
310
  the adaptive fallback) or `"before"` (tried first, overriding
240
311
  built-ins).
@@ -365,6 +436,23 @@ pytest
365
436
  Contributions are welcome — see [CONTRIBUTING.md](CONTRIBUTING.md). Notable
366
437
  changes are tracked in [CHANGELOG.md](CHANGELOG.md).
367
438
 
439
+ ## Citation
440
+
441
+ If you use syslog2cef in research or published work, please cite it. The
442
+ metadata is in [CITATION.cff](CITATION.cff), and GitHub's "Cite this
443
+ repository" button produces APA and BibTeX from it.
444
+
445
+ ```bibtex
446
+ @software{suliman_syslog2cef,
447
+ author = {Suliman, Tamir},
448
+ title = {syslog2cef: Convert syslog to ArcSight Common Event Format (CEF)},
449
+ version = {0.4.0},
450
+ year = {2026},
451
+ url = {https://github.com/allamiro/syslog2cef},
452
+ license = {MIT}
453
+ }
454
+ ```
455
+
368
456
  ## License
369
457
 
370
458
  MIT — see [LICENSE](LICENSE). Copyright (c) Tamir Suliman.
@@ -3,6 +3,7 @@ README.md
3
3
  pyproject.toml
4
4
  ./syslogcef/__init__.py
5
5
  ./syslogcef/__main__.py
6
+ ./syslogcef/_version.py
6
7
  ./syslogcef/adaptive.py
7
8
  ./syslogcef/api.py
8
9
  ./syslogcef/cef.py
@@ -31,6 +32,7 @@ syslog2cef.egg-info/requires.txt
31
32
  syslog2cef.egg-info/top_level.txt
32
33
  syslogcef/__init__.py
33
34
  syslogcef/__main__.py
35
+ syslogcef/_version.py
34
36
  syslogcef/adaptive.py
35
37
  syslogcef/api.py
36
38
  syslogcef/cef.py
@@ -9,10 +9,16 @@ conversion workflow.
9
9
  """
10
10
  from __future__ import annotations
11
11
 
12
+ from ._version import __author__, __copyright__, __email__, __license__, __version__
12
13
  from .api import StreamConverter, convert_line, normalize_event, parse_syslog, to_cef
13
14
  from .custom import PatternFileError, load_patterns, register_parser
14
15
 
15
16
  __all__ = [
17
+ "__version__",
18
+ "__author__",
19
+ "__email__",
20
+ "__copyright__",
21
+ "__license__",
16
22
  "StreamConverter",
17
23
  "convert_line",
18
24
  "normalize_event",
@@ -0,0 +1,12 @@
1
+ """Package version and author, shown by ``syslogcef --version``.
2
+
3
+ Kept in step with ``version`` in pyproject.toml (checked by the test
4
+ suite). A module constant rather than importlib.metadata, because the
5
+ standalone .pyz has no installed distribution metadata to read.
6
+ """
7
+
8
+ __version__ = "0.4.0"
9
+ __author__ = "Tamir Suliman"
10
+ __email__ = "allamiro@gmail.com"
11
+ __copyright__ = "Copyright (c) 2023 Tamir Suliman"
12
+ __license__ = "MIT"
@@ -2,6 +2,7 @@ from __future__ import annotations
2
2
 
3
3
  import logging
4
4
  from dataclasses import dataclass
5
+ from functools import lru_cache
5
6
  from typing import Any, Dict, Mapping
6
7
 
7
8
  from .normalizer import NormalizedEvent
@@ -15,6 +16,11 @@ HEADER_KEYS = (
15
16
  "deviceVendor", "deviceProduct", "deviceVersion", "eventClassId", "name",
16
17
  )
17
18
 
19
+ # Custom-field labels pair with the key they describe: cs1Label -> cs1.
20
+ # Only the dictionary's own *Label keys pair this way; a custom extension
21
+ # that merely ends in "Label" (classificationLabel) is independent data.
22
+ _LABEL_SUFFIX = "Label"
23
+
18
24
 
19
25
  DEFAULT_MAPPING = {
20
26
  "deviceVendor": "Generic",
@@ -54,6 +60,19 @@ DEFAULT_MAPPING = {
54
60
  }
55
61
 
56
62
 
63
+ @lru_cache(maxsize=1)
64
+ def _label_pairs() -> Dict[str, str]:
65
+ """Dictionary custom-field label -> the key it labels (cs1Label -> cs1)."""
66
+ from .dictionary import cef_keys
67
+
68
+ keys = cef_keys()
69
+ return {
70
+ key: key[: -len(_LABEL_SUFFIX)]
71
+ for key in keys
72
+ if key.endswith(_LABEL_SUFFIX) and key[: -len(_LABEL_SUFFIX)] in keys
73
+ }
74
+
75
+
57
76
  @dataclass
58
77
  class CEFEvent:
59
78
  header: Dict[str, str]
@@ -133,6 +152,14 @@ class MappingResolver:
133
152
  value = self._format(template, fields)
134
153
  if value:
135
154
  raw[key] = str(value)
155
+ # A label (cs1Label, cn1Label, flexString1Label, ...) is a constant
156
+ # and always renders, while its paired value is dropped when the
157
+ # template resolves empty. A label without its value describes
158
+ # nothing, so drop it too (#111).
159
+ labels = _label_pairs()
160
+ for key in [k for k in raw if k in labels]:
161
+ if labels[key] not in raw:
162
+ del raw[key]
136
163
  return raw
137
164
 
138
165
  def resolve_extensions(self, fields: Mapping[str, Any]) -> Dict[str, str]:
@@ -10,6 +10,7 @@ from multiprocessing import Pool, cpu_count
10
10
  from pathlib import Path
11
11
  from typing import Any, Iterable, Iterator, Optional
12
12
 
13
+ from ._version import __author__, __copyright__, __email__, __license__, __version__
13
14
  from .api import StreamConverter, convert_line
14
15
 
15
16
  logger = logging.getLogger(__name__)
@@ -381,8 +382,27 @@ def _optional_path(value: str) -> Optional[Path]:
381
382
  return Path(value) if value else None
382
383
 
383
384
 
385
+ # GNU-style --version text: name and version first (scripts parse it),
386
+ # then copyright, license, and authorship.
387
+ VERSION_TEXT = (
388
+ f"syslogcef {__version__}\n"
389
+ f"{__copyright__}\n"
390
+ f"License {__license__}: <https://opensource.org/licenses/{__license__}>\n"
391
+ "This is free software: you are free to change and redistribute it.\n"
392
+ "There is NO WARRANTY, to the extent permitted by law.\n"
393
+ "\n"
394
+ f"Written by {__author__} <{__email__}>."
395
+ )
396
+
397
+
384
398
  def main(argv: Optional[list[str]] = None) -> int:
385
- parser = argparse.ArgumentParser(description="Convert syslog lines to ArcSight CEF")
399
+ # RawDescriptionHelpFormatter keeps VERSION_TEXT's line breaks; argparse
400
+ # otherwise reflows the version string onto one line. Option help is
401
+ # still wrapped as before.
402
+ parser = argparse.ArgumentParser(
403
+ description="Convert syslog lines to ArcSight CEF",
404
+ formatter_class=argparse.RawDescriptionHelpFormatter,
405
+ )
386
406
  parser.add_argument("paths", nargs="*", type=Path, help="Input files (defaults to stdin)")
387
407
  parser.add_argument("-o", "--output", type=_optional_path, help="Output file; strftime codes such as %%Y-%%m-%%d or %%H start a new file when the rendered path changes (templated paths always append and parent directories are created). Use %%%% for a literal percent; unsupported codes are rejected. An empty value means stdout.")
388
408
  parser.add_argument("-a", "--append", action="store_true", help="Append to --output instead of truncating it (implied for templated paths)")
@@ -398,6 +418,12 @@ def main(argv: Optional[list[str]] = None) -> int:
398
418
  parser.add_argument("--validate", action="store_true", help="Validate CEF extensions against the ArcSight dictionary; violations are logged as warnings")
399
419
  parser.add_argument("--strict", action="store_true", help="Like --validate, but exit with an error on type/length violations")
400
420
  parser.add_argument("--log-level", default="WARNING", help="Logging level")
421
+ parser.add_argument(
422
+ "--version",
423
+ action="version",
424
+ version=VERSION_TEXT,
425
+ help="Show the version, license and author, then exit",
426
+ )
401
427
 
402
428
  args = parser.parse_args(argv)
403
429
  logging.basicConfig(level=getattr(logging, args.log_level.upper(), logging.WARNING))
@@ -40,12 +40,12 @@ from __future__ import annotations
40
40
  import json
41
41
  import re
42
42
  from dataclasses import dataclass
43
- from datetime import datetime, timedelta, timezone
43
+ from datetime import datetime, timezone
44
44
  from pathlib import Path
45
45
  from typing import Callable, Dict, List, Optional, Union
46
46
 
47
47
  from .parsers import PARSERS, ParsedEvent
48
- from .utils import convert_pri, parse_iso8601
48
+ from .utils import convert_pri, latest_past_occurrence, parse_iso8601
49
49
 
50
50
  _EVENT_GROUPS = {"pri", "host", "app", "pid", "msgid", "msg"}
51
51
  _TS_SPECIAL_FORMATS = {"iso8601", "epoch"}
@@ -126,10 +126,10 @@ def _validate_strptime_format(fmt: str) -> bool:
126
126
 
127
127
 
128
128
  def _year_with_rollover(parsed: datetime, ref: datetime) -> datetime:
129
- """Assign the most plausible year to a yearless timestamp.
129
+ """Assign the year to a yearless timestamp, as the built-in parsers do.
130
130
 
131
- Keeps every parsed component (tz offset, microseconds); handles
132
- Feb 29 by picking the nearest leap year when needed.
131
+ Keeps every parsed component (tz offset, microseconds); see
132
+ :func:`syslogcef.utils.latest_past_occurrence` for the rule.
133
133
  """
134
134
 
135
135
  def with_year(year: int) -> Optional[datetime]:
@@ -138,19 +138,7 @@ def _year_with_rollover(parsed: datetime, ref: datetime) -> datetime:
138
138
  except ValueError: # Feb 29 in a non-leap year
139
139
  return None
140
140
 
141
- candidate = with_year(ref.year)
142
- if candidate is None:
143
- # Feb 29: pick the occurrence closest in time, not the first
144
- # leap year found in either direction.
145
- candidates = [c for off in range(-4, 5) if off and (c := with_year(ref.year + off))]
146
- if not candidates:
147
- return parsed
148
- return min(candidates, key=lambda c: abs(c - ref))
149
- if candidate - ref > timedelta(days=180):
150
- return with_year(ref.year - 1) or candidate
151
- if ref - candidate > timedelta(days=180):
152
- return with_year(ref.year + 1) or candidate
153
- return candidate
141
+ return latest_past_occurrence(with_year, ref) or parsed
154
142
 
155
143
 
156
144
  def _parse_timestamp(text: str, fmt: str, *, now: Optional[datetime] = None) -> datetime:
@@ -21,6 +21,6 @@
21
21
  "dpt": "%(dpt)s",
22
22
  "proto": "%(protocol)s",
23
23
  "cs1Label": "rawEvent",
24
- "cs1": "%(raw_kv)s"
24
+ "cs1": "%(raw)s"
25
25
  }
26
26
  }
@@ -12,6 +12,7 @@ from .utils import (
12
12
  JournalJSONMapping,
13
13
  convert_pri,
14
14
  guess_hostname,
15
+ latest_past_occurrence,
15
16
  month_abbr_to_int,
16
17
  parse_iso8601,
17
18
  parse_key_value_pairs,
@@ -151,20 +152,9 @@ def _infer_timestamp(month: int, day: int, time_str: str, *, now: Optional[datet
151
152
  except ValueError: # Feb 29 in a non-leap year
152
153
  return None
153
154
 
154
- candidate = build(now.year)
155
+ candidate = latest_past_occurrence(build, now)
155
156
  if candidate is None:
156
- # A yearless Feb 29 line in a non-leap runtime year must not
157
- # crash the parser: pick the occurrence closest in time (not
158
- # merely the first leap year found in either direction).
159
- candidates = [c for off in range(-4, 5) if off and (c := build(now.year + off))]
160
- if not candidates:
161
- raise ValueError(f"impossible calendar date: month={month} day={day}")
162
- return min(candidates, key=lambda c: abs(c - now))
163
- # Handle rollover into previous year.
164
- if candidate - now > timedelta(days=180):
165
- candidate = build(now.year - 1) or candidate
166
- elif now - candidate > timedelta(days=180):
167
- candidate = build(now.year + 1) or candidate
157
+ raise ValueError(f"impossible calendar date: month={month} day={day}")
168
158
  return candidate
169
159
 
170
160
 
@@ -5,8 +5,8 @@ import logging
5
5
  import re
6
6
  import socket
7
7
  from dataclasses import dataclass
8
- from datetime import datetime, timezone
9
- from typing import Any, Dict, Mapping, Optional
8
+ from datetime import datetime, timedelta, timezone
9
+ from typing import Any, Callable, Dict, Mapping, Optional
10
10
 
11
11
  logger = logging.getLogger(__name__)
12
12
 
@@ -104,6 +104,32 @@ def parse_iso8601(value: str) -> datetime:
104
104
  raise ValueError(f"Invalid ISO8601 timestamp: {value!r}") from exc
105
105
 
106
106
 
107
+ # How far ahead of "now" a yearless timestamp may sit and still count as
108
+ # the current occurrence: covers senders in timezones ahead of UTC (up to
109
+ # +14h, since yearless local times are read as UTC) plus modest clock skew.
110
+ YEARLESS_FUTURE_TOLERANCE = timedelta(days=1)
111
+
112
+
113
+ def latest_past_occurrence(
114
+ build: Callable[[int], Optional[datetime]], now: datetime
115
+ ) -> Optional[datetime]:
116
+ """Pick the year for a timestamp that carries none.
117
+
118
+ ``build(year)`` returns the timestamp in that year, or ``None`` when the
119
+ date does not exist then (Feb 29). Logs describe events that already
120
+ happened, so the answer is the most recent occurrence no later than
121
+ ``now`` plus :data:`YEARLESS_FUTURE_TOLERANCE`. Next year only wins at
122
+ the year boundary (a Jan 1 00:00:05 line received at Dec 31 23:59:58).
123
+ Leap days recur at most eight years apart, which bounds the search.
124
+ """
125
+
126
+ for year in range(now.year + 1, now.year - 9, -1):
127
+ candidate = build(year)
128
+ if candidate is not None and candidate - now <= YEARLESS_FUTURE_TOLERANCE:
129
+ return candidate
130
+ return None
131
+
132
+
107
133
  def ensure_tzaware(ts: Optional[datetime]) -> Optional[datetime]:
108
134
  if ts is None:
109
135
  return None
@@ -184,3 +184,47 @@ def test_full_conversion_syntax_is_accepted(template):
184
184
  # Flags, width, precision and length modifiers are all valid syntax;
185
185
  # whether a given value satisfies the conversion is data-dependent.
186
186
  assert convert_line("plain log", mapping={"name": template}).startswith("CEF:0|")
187
+
188
+
189
+ def _extensions(cef: str) -> str:
190
+ return cef.split("|", 7)[7]
191
+
192
+
193
+ def test_asa_line_without_kv_keeps_raw_event_value():
194
+ # Issue #111: the ASA mapping took cs1 from raw_kv, which is empty for
195
+ # most ASA messages, so cs1 was dropped but cs1Label=rawEvent remained.
196
+ line = "<166>Jan 1 12:34:56 fw1 %ASA-6-302013: Built inbound TCP connection"
197
+ ext = _extensions(convert_line(line))
198
+
199
+ assert "cs1Label=rawEvent" in ext
200
+ assert "cs1=<166>Jan 1 12:34:56 fw1 %ASA-6-302013: Built inbound TCP connection" in ext
201
+
202
+
203
+ @pytest.mark.parametrize("name", ["cisco_asa", "cisco_ios", "f5", "fortinet", "linux", "sophos", "vmware"])
204
+ def test_bundled_mappings_carry_the_raw_line_in_cs1(name):
205
+ assert load_mapping(name)["extensions"]["cs1"] == "%(raw)s"
206
+
207
+
208
+ @pytest.mark.parametrize(
209
+ "label, value",
210
+ [("cs2Label", "cs2"), ("cn1Label", "cn1"), ("c6a1Label", "c6a1"),
211
+ ("deviceCustomDate1Label", "deviceCustomDate1"), ("flexString1Label", "flexString1")],
212
+ )
213
+ def test_label_is_dropped_when_its_value_is_empty(label, value):
214
+ mapping = {"extensions": {label: "Session", value: "%(session)s"}}
215
+
216
+ without = _extensions(convert_line("srcip=1.2.3.4 action=x", mapping=mapping))
217
+ assert f"{label}=" not in without
218
+ assert f" {value}=" not in f" {without}"
219
+
220
+ with_value = _extensions(convert_line("srcip=1.2.3.4 session=abc", mapping=mapping))
221
+ assert f"{label}=Session" in with_value
222
+ assert f"{value}=abc" in with_value
223
+
224
+
225
+ def test_custom_key_ending_in_label_is_not_treated_as_a_label():
226
+ # Only the dictionary's custom-field labels pair with a value; an
227
+ # independent custom key that merely ends in "Label" keeps its data.
228
+ ext = _extensions(convert_line("plain log", mapping={"extensions": {"classificationLabel": "Confidential"}}))
229
+
230
+ assert "classificationLabel=Confidential" in ext
@@ -176,3 +176,46 @@ def test_cli_output_template_end_to_end(tmp_path: Path):
176
176
  files = list((tmp_path / "cef").rglob("events.cef"))
177
177
  assert len(files) == 1
178
178
  assert "CEF:0" in files[0].read_text(encoding="utf-8")
179
+
180
+
181
+ def _run_cli(*args: str) -> subprocess.CompletedProcess:
182
+ return subprocess.run(
183
+ [sys.executable, "-m", "syslogcef", *args],
184
+ capture_output=True,
185
+ text=True,
186
+ check=True,
187
+ )
188
+
189
+
190
+ def test_version_shows_version_and_author():
191
+ out = _run_cli("--version").stdout
192
+
193
+ from syslogcef import __version__
194
+
195
+ lines = out.splitlines()
196
+ # First line is "name version" so scripts can parse it.
197
+ assert lines[0] == f"syslogcef {__version__}"
198
+ assert "Copyright (c) 2023 Tamir Suliman" in lines
199
+ assert lines[-1] == "Written by Tamir Suliman <allamiro@gmail.com>."
200
+
201
+
202
+ def test_conversion_output_has_no_author_line():
203
+ # stdout carries CEF records for SIEMs; nothing else may be printed.
204
+ result = subprocess.run(
205
+ [sys.executable, "-m", "syslogcef"],
206
+ input="<166>Jan 1 12:34:56 router1 %ASA-6-302013: Built inbound TCP connection\n",
207
+ capture_output=True,
208
+ text=True,
209
+ check=True,
210
+ )
211
+ assert result.stdout.splitlines()[0].startswith("CEF:0|")
212
+ assert len(result.stdout.splitlines()) == 1
213
+
214
+
215
+ def test_package_version_matches_pyproject():
216
+ import re
217
+
218
+ from syslogcef import __version__
219
+
220
+ pyproject = (Path(__file__).resolve().parent.parent / "pyproject.toml").read_text(encoding="utf-8")
221
+ assert re.search(r'^version = "([^"]+)"', pyproject, re.M).group(1) == __version__
@@ -287,4 +287,21 @@ def test_cli_rejects_unknown_mode(tmp_path: Path):
287
287
  from syslogcef.cli import main
288
288
 
289
289
  with pytest.raises(SystemExit):
290
- main(["--mode", "nonexistent"])
290
+ main(["--mode", "nonexistent"])
291
+
292
+ def test_yearless_pattern_timestamp_is_dated_in_the_past(tmp_path: Path):
293
+ # Issue #112: custom patterns share the built-in year inference.
294
+ load_patterns(
295
+ write_patterns(
296
+ tmp_path,
297
+ {"name": "past", "regex": r"^P (?P<ts>\w{3} \d{2} \d{2}:\d{2}:\d{2}) (?P<msg>.*)$", "timestamp_format": "%b %d %H:%M:%S"},
298
+ )
299
+ )
300
+ parse = custom.get_registered("past")
301
+
302
+ late_november = datetime(2026, 11, 20, tzinfo=timezone.utc)
303
+ assert parse("P Jan 01 12:34:56 x", now=late_november).ts.year == 2026
304
+ assert parse("P Nov 25 10:00:00 x", now=late_november).ts.year == 2025
305
+
306
+ new_years_eve = datetime(2026, 12, 31, 23, 59, 58, tzinfo=timezone.utc)
307
+ assert parse("P Jan 01 00:00:05 x", now=new_years_eve).ts.year == 2027
@@ -73,3 +73,47 @@ def test_rfc3164_year_rollover_previous_year():
73
73
  line = "<14>Dec 31 23:59:59 host1 app: year rollover"
74
74
  result = autodetect_and_parse(line, now=datetime(2024, 1, 2, tzinfo=timezone.utc))
75
75
  assert result.ts.year == 2023
76
+
77
+
78
+ # Issue #112: a yearless timestamp describes an event that already
79
+ # happened, so it takes the most recent occurrence that is not later than
80
+ # now plus a small clock-skew/timezone tolerance — never a date months
81
+ # ahead just because it is nearer than last year's.
82
+
83
+ @pytest.mark.parametrize(
84
+ "line, now, year",
85
+ [
86
+ # Jan 1 seen in late November: earlier this year, not next year.
87
+ ("<14>Jan 1 12:34:56 h1 sshd[22]: x", datetime(2026, 11, 20, tzinfo=timezone.utc), 2026),
88
+ # Year boundary: a line stamped just after midnight by a clock
89
+ # slightly ahead lands in the year that is about to begin.
90
+ ("<14>Jan 1 00:00:05 h1 app: x", datetime(2026, 12, 31, 23, 59, 58, tzinfo=timezone.utc), 2027),
91
+ # Later today (sender in a timezone ahead of UTC) stays this year.
92
+ ("<14>Nov 20 22:00:00 h1 app: x", datetime(2026, 11, 20, 9, 0, tzinfo=timezone.utc), 2026),
93
+ # Several days in the future can only be last year's date.
94
+ ("<14>Nov 25 10:00:00 h1 app: x", datetime(2026, 11, 20, tzinfo=timezone.utc), 2025),
95
+ # Six-plus months back is still the past, not next year.
96
+ ("<14>Mar 3 10:00:00 h1 app: x", datetime(2026, 11, 20, tzinfo=timezone.utc), 2026),
97
+ ],
98
+ )
99
+ def test_yearless_timestamp_is_dated_in_the_past(line, now, year):
100
+ assert autodetect_and_parse(line, now=now).ts.year == year
101
+
102
+
103
+ def test_yearless_feb29_takes_the_last_leap_day_in_the_past():
104
+ from syslogcef.parsers import _infer_timestamp
105
+
106
+ # 2028-02-29 is nearer to 2027-03-01 than 2024-02-29, but it has not
107
+ # happened yet.
108
+ ts = _infer_timestamp(2, 29, "12:00:00", now=datetime(2027, 3, 1, tzinfo=timezone.utc))
109
+ assert ts.year == 2024
110
+
111
+
112
+ def test_adaptive_mon_day_is_dated_in_the_past():
113
+ import re
114
+
115
+ from syslogcef.adaptive import TIMESTAMP_LIB
116
+
117
+ src, conv = next((s, c) for n, s, c in TIMESTAMP_LIB if n == "mon_day")
118
+ ts = conv(re.compile(src).match("Jan 1 12:34:56"), datetime(2026, 11, 20, tzinfo=timezone.utc))
119
+ assert ts.year == 2026
File without changes
File without changes
File without changes
File without changes
File without changes