syslog2cef 0.3.3__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/PKG-INFO +78 -13
- syslog2cef-0.3.3/syslog2cef.egg-info/PKG-INFO → syslog2cef-0.4.0/README.md +71 -35
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/pyproject.toml +7 -1
- syslog2cef-0.3.3/README.md → syslog2cef-0.4.0/syslog2cef.egg-info/PKG-INFO +100 -12
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslog2cef.egg-info/SOURCES.txt +2 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/__init__.py +6 -0
- syslog2cef-0.4.0/syslogcef/_version.py +12 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/cef.py +27 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/cli.py +27 -1
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/custom.py +6 -18
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/mappings/cisco_asa.json +1 -1
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/parsers.py +3 -13
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/utils.py +28 -2
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_cef.py +44 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_cli.py +43 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_custom_parsers.py +18 -1
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_parsers_extra.py +44 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/LICENSE +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/setup.cfg +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslog2cef.egg-info/dependency_links.txt +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslog2cef.egg-info/entry_points.txt +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslog2cef.egg-info/requires.txt +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslog2cef.egg-info/top_level.txt +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/__main__.py +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/adaptive.py +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/api.py +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/dictionary.json +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/dictionary.py +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/mappings/__init__.py +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/mappings/cisco_ios.json +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/mappings/f5.json +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/mappings/fortinet.json +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/mappings/linux.json +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/mappings/sophos.json +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/mappings/vmware.json +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/net.py +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/normalizer.py +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/syslogcef/validation.py +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_adaptive.py +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_cli_validation.py +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_dictionary.py +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_fuzz_grammar.py +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_fuzz_properties.py +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_macos_logs.py +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_net.py +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_normalizer.py +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_parsers.py +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_pipeline.py +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_real_world.py +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_rfc5424_epoch.py +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_security_hardening.py +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_tail_rotation.py +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_utils.py +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_validation.py +0 -0
- {syslog2cef-0.3.3 → syslog2cef-0.4.0}/tests/test_zipapp_safe.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: syslog2cef
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
|
|
5
5
|
Author-email: Tamir Suliman <allamiro@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/allamiro/syslogcef
|
|
@@ -8,6 +8,12 @@ Project-URL: Issues, https://github.com/allamiro/syslogcef/issues
|
|
|
8
8
|
Keywords: syslog,cef,logging,security
|
|
9
9
|
Classifier: Programming Language :: Python :: 3
|
|
10
10
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
11
17
|
Classifier: License :: OSI Approved :: MIT License
|
|
12
18
|
Classifier: Topic :: System :: Logging
|
|
13
19
|
Requires-Python: >=3.9
|
|
@@ -25,6 +31,19 @@ Dynamic: license-file
|
|
|
25
31
|
<img src="https://raw.githubusercontent.com/allamiro/syslogcef/main/docs/logo.svg" alt="syslogcef logo" width="540">
|
|
26
32
|
</p>
|
|
27
33
|
|
|
34
|
+
<p align="center">
|
|
35
|
+
<a href="https://github.com/allamiro/syslog2cef/actions/workflows/ci.yml"><img src="https://img.shields.io/github/actions/workflow/status/allamiro/syslog2cef/ci.yml?branch=main&label=CI" alt="CI status"></a>
|
|
36
|
+
<a href="https://pypi.org/project/syslog2cef/"><img src="https://img.shields.io/pypi/v/syslog2cef" alt="PyPI version"></a>
|
|
37
|
+
<a href="https://pypi.org/project/syslog2cef/"><img src="https://img.shields.io/pypi/pyversions/syslog2cef" alt="Supported Python versions"></a>
|
|
38
|
+
<a href="https://pypistats.org/packages/syslog2cef"><img src="https://img.shields.io/pypi/dm/syslog2cef" alt="PyPI downloads per month"></a>
|
|
39
|
+
<a href="https://copr.fedorainfracloud.org/coprs/allamiro/syslogcef/"><img src="https://copr.fedorainfracloud.org/coprs/allamiro/syslogcef/package/syslogcef/status_image/last_build.png" alt="COPR build status"></a>
|
|
40
|
+
<a href="https://github.com/allamiro/syslog2cef/pkgs/container/syslogcef"><img src="https://img.shields.io/badge/container-ghcr.io-blue?logo=docker" alt="Container image on ghcr.io"></a>
|
|
41
|
+
<a href="https://github.com/allamiro/syslog2cef/actions/workflows/fuzz.yml"><img src="https://img.shields.io/github/actions/workflow/status/allamiro/syslog2cef/fuzz.yml?branch=main&label=deep%20fuzz" alt="Deep fuzz status"></a>
|
|
42
|
+
<a href="https://github.com/astral-sh/ruff"><img src="https://img.shields.io/badge/lint-ruff-261230" alt="Linted with Ruff"></a>
|
|
43
|
+
<a href="LICENSE"><img src="https://img.shields.io/github/license/allamiro/syslog2cef" alt="MIT license"></a>
|
|
44
|
+
<a href="CITATION.cff"><img src="https://img.shields.io/badge/cite-CITATION.cff-informational" alt="Citation file"></a>
|
|
45
|
+
</p>
|
|
46
|
+
|
|
28
47
|
# syslogcef
|
|
29
48
|
|
|
30
49
|
syslogcef converts syslog events into ArcSight Common Event Format (CEF). It
|
|
@@ -36,11 +55,18 @@ platforms that consume CEF.
|
|
|
36
55
|
|
|
37
56
|
- Automatic detection and parsing of multiple syslog dialects:
|
|
38
57
|
RFC3164 (BSD), RFC5424 (including structured data), rsyslog JSON and
|
|
39
|
-
file formats,
|
|
58
|
+
file formats, systemd journal exports (JSON, short, and ISO formats),
|
|
59
|
+
Cisco console/buffer lines, ISO-timestamp syslog, and key=value streams
|
|
60
|
+
(Fortinet, Sophos). Unknown layouts are learned adaptively.
|
|
40
61
|
- Deterministic vendor/product mappings bundled for Cisco ASA, Cisco IOS,
|
|
41
|
-
F5 BIG-IP,
|
|
42
|
-
selection based on message content.
|
|
43
|
-
- Custom mappings supplied as JSON files or Python dictionaries
|
|
62
|
+
F5 BIG-IP, Fortinet FortiGate, Sophos XG Firewall, VMware ESXi, and
|
|
63
|
+
generic Linux, with automatic mapping selection based on message content.
|
|
64
|
+
- Custom mappings supplied as JSON files or Python dictionaries, and custom
|
|
65
|
+
parsers supplied as regex pattern files.
|
|
66
|
+
- Validation of CEF extension values against the ArcSight extension
|
|
67
|
+
dictionary (`--validate`, `--strict`).
|
|
68
|
+
- Network receiver (`--listen udp:514`) and forwarding to UDP, TCP, or
|
|
69
|
+
Kafka (`--send`), with optional rate limiting.
|
|
44
70
|
- Escaping of untrusted log content in both CEF header and extension
|
|
45
71
|
fields, so crafted messages cannot forge header fields or split records.
|
|
46
72
|
- Streaming CLI with stdin/file input, tail (follow) mode across multiple
|
|
@@ -129,13 +155,21 @@ Options:
|
|
|
129
155
|
| Option | Description |
|
|
130
156
|
| ------ | ----------- |
|
|
131
157
|
| `paths` | Input files; stdin is used when omitted. |
|
|
132
|
-
| `-o, --output FILE` | Write CEF lines to a file instead of stdout. |
|
|
133
|
-
|
|
|
158
|
+
| `-o, --output FILE` | Write CEF lines to a file instead of stdout. Accepts strftime codes for timed files (see "Timed output files"). |
|
|
159
|
+
| `-a, --append` | Append to `--output` instead of truncating it (implied for templated paths). |
|
|
160
|
+
| `--mode MODE` | Parser override: `rfc3164`, `rfc5424`, `rsyslog_json`, `rsyslog_file`, `journald_json`, `journald_short`, `journald_iso`, `cisco_seq`, `iso_syslog`, `kv`, or a name from `--patterns`. Auto-detected when omitted. |
|
|
134
161
|
| `--mapping FILE` | Mapping JSON file. Auto-selected from message content when omitted. |
|
|
162
|
+
| `--patterns FILE` | Custom parser patterns (see "Custom Parsers"); may be repeated. |
|
|
135
163
|
| `--tail` | Follow input files like `tail -f`. |
|
|
164
|
+
| `--listen PROTO:PORT` | Receive syslog over the network instead of files/stdin: `udp:514`, `tcp:5514`, or `udp:10.0.0.5:514`. |
|
|
165
|
+
| `--send URL` | Forward CEF to `udp://HOST:PORT`, `tcp://HOST:PORT`, or `kafka://BROKER:PORT/TOPIC` (needs the `kafka` extra). |
|
|
166
|
+
| `--eps N` | Rate-limit `--send` to at most N events per second. |
|
|
136
167
|
| `--multiprocess` | Convert lines using a process pool. |
|
|
137
168
|
| `--pool-size N` | Worker count for `--multiprocess` (default: CPU count minus one). |
|
|
169
|
+
| `--validate` | Check CEF extensions against the ArcSight dictionary; violations are logged as warnings. |
|
|
170
|
+
| `--strict` | Like `--validate`, but exit with an error on type or length violations. |
|
|
138
171
|
| `--log-level LEVEL` | Python logging level (default `WARNING`). |
|
|
172
|
+
| `--version` | Print the version, copyright, license and author, then exit. |
|
|
139
173
|
|
|
140
174
|
`python -m syslogcef` is equivalent to the `syslogcef` entry point.
|
|
141
175
|
|
|
@@ -161,7 +195,8 @@ cef = to_cef(normalized, mapping="my_mapping.json")
|
|
|
161
195
|
```
|
|
162
196
|
|
|
163
197
|
Bundled mappings are importable from `syslogcef.mappings` (`CISCO_ASA`,
|
|
164
|
-
`CISCO_IOS`, `F5`, `LINUX`, `VMWARE`, or
|
|
198
|
+
`CISCO_IOS`, `F5`, `FORTINET`, `LINUX`, `SOPHOS`, `VMWARE`, or
|
|
199
|
+
`load_mapping(name)`).
|
|
165
200
|
|
|
166
201
|
## Mapping Files
|
|
167
202
|
|
|
@@ -181,7 +216,7 @@ normalized event's fields:
|
|
|
181
216
|
"src": "%(src)s",
|
|
182
217
|
"dst": "%(dst)s",
|
|
183
218
|
"cs1Label": "rawEvent",
|
|
184
|
-
"cs1": "%(
|
|
219
|
+
"cs1": "%(raw)s"
|
|
185
220
|
}
|
|
186
221
|
}
|
|
187
222
|
```
|
|
@@ -191,7 +226,8 @@ normalized event's fields:
|
|
|
191
226
|
- `severity_map` translates syslog severity (0-7) to CEF severity (0-10);
|
|
192
227
|
unmapped values pass through.
|
|
193
228
|
- `extensions` maps CEF extension keys to templates. Extensions that
|
|
194
|
-
resolve to an empty value are omitted
|
|
229
|
+
resolve to an empty value are omitted, and so is a custom-field label
|
|
230
|
+
(`cs1Label`, `cn1Label`, ...) whose paired value is omitted.
|
|
195
231
|
- Available template fields include `host`, `app`, `pid`, `msgid`, `msg`,
|
|
196
232
|
`message_short` (first 120 characters), `raw`, `raw_kv`, `event_code`,
|
|
197
233
|
`facility`, `severity`, `ts`, plus every key=value pair extracted from
|
|
@@ -204,6 +240,18 @@ extension keys fail with a configuration error instead of corrupting a record.
|
|
|
204
240
|
See [docs/cef_fields.md](docs/cef_fields.md) for the full CEF extension
|
|
205
241
|
dictionary.
|
|
206
242
|
|
|
243
|
+
## Timestamps without a year
|
|
244
|
+
|
|
245
|
+
RFC3164, journald short, Cisco console lines and many device formats
|
|
246
|
+
stamp `Jan 1 12:34:56` with no year. syslogcef dates such a timestamp to
|
|
247
|
+
its most recent occurrence that is not more than one day ahead of the
|
|
248
|
+
current time: log lines describe events that already happened. A `Jan 1`
|
|
249
|
+
line converted in November is dated January of the current year, while a
|
|
250
|
+
line stamped just after midnight on New Year's Eve (a sender clock or
|
|
251
|
+
timezone slightly ahead) lands in the new year. Pass `now=` to
|
|
252
|
+
`parse_syslog`/`convert_line` to date archived files relative to when
|
|
253
|
+
they were written.
|
|
254
|
+
|
|
207
255
|
## Field Dictionary
|
|
208
256
|
|
|
209
257
|
`syslogcef/dictionary.json` is the single source of truth for CEF field
|
|
@@ -255,9 +303,9 @@ syslogcef acme.log --patterns /etc/syslogcef/patterns.json
|
|
|
255
303
|
- Named groups map to event fields: `pri`, `host`, `app`, `pid`,
|
|
256
304
|
`msgid`, `msg`, and `ts`.
|
|
257
305
|
- A `ts` group requires `timestamp_format`: a strptime format,
|
|
258
|
-
`iso8601`, or `epoch`. Yearless formats get the same year
|
|
259
|
-
|
|
260
|
-
drops the event.
|
|
306
|
+
`iso8601`, or `epoch`. Yearless formats get the same year inference
|
|
307
|
+
as the built-in parsers (see "Timestamps without a year" below); an
|
|
308
|
+
unparseable timestamp never drops the event.
|
|
261
309
|
- `priority` is `"after"` (default: tried after the built-ins, before
|
|
262
310
|
the adaptive fallback) or `"before"` (tried first, overriding
|
|
263
311
|
built-ins).
|
|
@@ -388,6 +436,23 @@ pytest
|
|
|
388
436
|
Contributions are welcome — see [CONTRIBUTING.md](CONTRIBUTING.md). Notable
|
|
389
437
|
changes are tracked in [CHANGELOG.md](CHANGELOG.md).
|
|
390
438
|
|
|
439
|
+
## Citation
|
|
440
|
+
|
|
441
|
+
If you use syslog2cef in research or published work, please cite it. The
|
|
442
|
+
metadata is in [CITATION.cff](CITATION.cff), and GitHub's "Cite this
|
|
443
|
+
repository" button produces APA and BibTeX from it.
|
|
444
|
+
|
|
445
|
+
```bibtex
|
|
446
|
+
@software{suliman_syslog2cef,
|
|
447
|
+
author = {Suliman, Tamir},
|
|
448
|
+
title = {syslog2cef: Convert syslog to ArcSight Common Event Format (CEF)},
|
|
449
|
+
version = {0.4.0},
|
|
450
|
+
year = {2026},
|
|
451
|
+
url = {https://github.com/allamiro/syslog2cef},
|
|
452
|
+
license = {MIT}
|
|
453
|
+
}
|
|
454
|
+
```
|
|
455
|
+
|
|
391
456
|
## License
|
|
392
457
|
|
|
393
458
|
MIT — see [LICENSE](LICENSE). Copyright (c) Tamir Suliman.
|
|
@@ -1,30 +1,20 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: syslog2cef
|
|
3
|
-
Version: 0.3.3
|
|
4
|
-
Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
|
|
5
|
-
Author-email: Tamir Suliman <allamiro@gmail.com>
|
|
6
|
-
Project-URL: Homepage, https://github.com/allamiro/syslogcef
|
|
7
|
-
Project-URL: Issues, https://github.com/allamiro/syslogcef/issues
|
|
8
|
-
Keywords: syslog,cef,logging,security
|
|
9
|
-
Classifier: Programming Language :: Python :: 3
|
|
10
|
-
Classifier: Programming Language :: Python :: 3 :: Only
|
|
11
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
-
Classifier: Topic :: System :: Logging
|
|
13
|
-
Requires-Python: >=3.9
|
|
14
|
-
Description-Content-Type: text/markdown
|
|
15
|
-
License-File: LICENSE
|
|
16
|
-
Provides-Extra: test
|
|
17
|
-
Requires-Dist: pytest>=7; extra == "test"
|
|
18
|
-
Provides-Extra: kafka
|
|
19
|
-
Requires-Dist: kafka-python>=2.0; extra == "kafka"
|
|
20
|
-
Provides-Extra: fuzz
|
|
21
|
-
Requires-Dist: hypothesis>=6; extra == "fuzz"
|
|
22
|
-
Dynamic: license-file
|
|
23
|
-
|
|
24
1
|
<p align="center">
|
|
25
2
|
<img src="https://raw.githubusercontent.com/allamiro/syslogcef/main/docs/logo.svg" alt="syslogcef logo" width="540">
|
|
26
3
|
</p>
|
|
27
4
|
|
|
5
|
+
<p align="center">
|
|
6
|
+
<a href="https://github.com/allamiro/syslog2cef/actions/workflows/ci.yml"><img src="https://img.shields.io/github/actions/workflow/status/allamiro/syslog2cef/ci.yml?branch=main&label=CI" alt="CI status"></a>
|
|
7
|
+
<a href="https://pypi.org/project/syslog2cef/"><img src="https://img.shields.io/pypi/v/syslog2cef" alt="PyPI version"></a>
|
|
8
|
+
<a href="https://pypi.org/project/syslog2cef/"><img src="https://img.shields.io/pypi/pyversions/syslog2cef" alt="Supported Python versions"></a>
|
|
9
|
+
<a href="https://pypistats.org/packages/syslog2cef"><img src="https://img.shields.io/pypi/dm/syslog2cef" alt="PyPI downloads per month"></a>
|
|
10
|
+
<a href="https://copr.fedorainfracloud.org/coprs/allamiro/syslogcef/"><img src="https://copr.fedorainfracloud.org/coprs/allamiro/syslogcef/package/syslogcef/status_image/last_build.png" alt="COPR build status"></a>
|
|
11
|
+
<a href="https://github.com/allamiro/syslog2cef/pkgs/container/syslogcef"><img src="https://img.shields.io/badge/container-ghcr.io-blue?logo=docker" alt="Container image on ghcr.io"></a>
|
|
12
|
+
<a href="https://github.com/allamiro/syslog2cef/actions/workflows/fuzz.yml"><img src="https://img.shields.io/github/actions/workflow/status/allamiro/syslog2cef/fuzz.yml?branch=main&label=deep%20fuzz" alt="Deep fuzz status"></a>
|
|
13
|
+
<a href="https://github.com/astral-sh/ruff"><img src="https://img.shields.io/badge/lint-ruff-261230" alt="Linted with Ruff"></a>
|
|
14
|
+
<a href="LICENSE"><img src="https://img.shields.io/github/license/allamiro/syslog2cef" alt="MIT license"></a>
|
|
15
|
+
<a href="CITATION.cff"><img src="https://img.shields.io/badge/cite-CITATION.cff-informational" alt="Citation file"></a>
|
|
16
|
+
</p>
|
|
17
|
+
|
|
28
18
|
# syslogcef
|
|
29
19
|
|
|
30
20
|
syslogcef converts syslog events into ArcSight Common Event Format (CEF). It
|
|
@@ -36,11 +26,18 @@ platforms that consume CEF.
|
|
|
36
26
|
|
|
37
27
|
- Automatic detection and parsing of multiple syslog dialects:
|
|
38
28
|
RFC3164 (BSD), RFC5424 (including structured data), rsyslog JSON and
|
|
39
|
-
file formats,
|
|
29
|
+
file formats, systemd journal exports (JSON, short, and ISO formats),
|
|
30
|
+
Cisco console/buffer lines, ISO-timestamp syslog, and key=value streams
|
|
31
|
+
(Fortinet, Sophos). Unknown layouts are learned adaptively.
|
|
40
32
|
- Deterministic vendor/product mappings bundled for Cisco ASA, Cisco IOS,
|
|
41
|
-
F5 BIG-IP,
|
|
42
|
-
selection based on message content.
|
|
43
|
-
- Custom mappings supplied as JSON files or Python dictionaries
|
|
33
|
+
F5 BIG-IP, Fortinet FortiGate, Sophos XG Firewall, VMware ESXi, and
|
|
34
|
+
generic Linux, with automatic mapping selection based on message content.
|
|
35
|
+
- Custom mappings supplied as JSON files or Python dictionaries, and custom
|
|
36
|
+
parsers supplied as regex pattern files.
|
|
37
|
+
- Validation of CEF extension values against the ArcSight extension
|
|
38
|
+
dictionary (`--validate`, `--strict`).
|
|
39
|
+
- Network receiver (`--listen udp:514`) and forwarding to UDP, TCP, or
|
|
40
|
+
Kafka (`--send`), with optional rate limiting.
|
|
44
41
|
- Escaping of untrusted log content in both CEF header and extension
|
|
45
42
|
fields, so crafted messages cannot forge header fields or split records.
|
|
46
43
|
- Streaming CLI with stdin/file input, tail (follow) mode across multiple
|
|
@@ -129,13 +126,21 @@ Options:
|
|
|
129
126
|
| Option | Description |
|
|
130
127
|
| ------ | ----------- |
|
|
131
128
|
| `paths` | Input files; stdin is used when omitted. |
|
|
132
|
-
| `-o, --output FILE` | Write CEF lines to a file instead of stdout. |
|
|
133
|
-
|
|
|
129
|
+
| `-o, --output FILE` | Write CEF lines to a file instead of stdout. Accepts strftime codes for timed files (see "Timed output files"). |
|
|
130
|
+
| `-a, --append` | Append to `--output` instead of truncating it (implied for templated paths). |
|
|
131
|
+
| `--mode MODE` | Parser override: `rfc3164`, `rfc5424`, `rsyslog_json`, `rsyslog_file`, `journald_json`, `journald_short`, `journald_iso`, `cisco_seq`, `iso_syslog`, `kv`, or a name from `--patterns`. Auto-detected when omitted. |
|
|
134
132
|
| `--mapping FILE` | Mapping JSON file. Auto-selected from message content when omitted. |
|
|
133
|
+
| `--patterns FILE` | Custom parser patterns (see "Custom Parsers"); may be repeated. |
|
|
135
134
|
| `--tail` | Follow input files like `tail -f`. |
|
|
135
|
+
| `--listen PROTO:PORT` | Receive syslog over the network instead of files/stdin: `udp:514`, `tcp:5514`, or `udp:10.0.0.5:514`. |
|
|
136
|
+
| `--send URL` | Forward CEF to `udp://HOST:PORT`, `tcp://HOST:PORT`, or `kafka://BROKER:PORT/TOPIC` (needs the `kafka` extra). |
|
|
137
|
+
| `--eps N` | Rate-limit `--send` to at most N events per second. |
|
|
136
138
|
| `--multiprocess` | Convert lines using a process pool. |
|
|
137
139
|
| `--pool-size N` | Worker count for `--multiprocess` (default: CPU count minus one). |
|
|
140
|
+
| `--validate` | Check CEF extensions against the ArcSight dictionary; violations are logged as warnings. |
|
|
141
|
+
| `--strict` | Like `--validate`, but exit with an error on type or length violations. |
|
|
138
142
|
| `--log-level LEVEL` | Python logging level (default `WARNING`). |
|
|
143
|
+
| `--version` | Print the version, copyright, license and author, then exit. |
|
|
139
144
|
|
|
140
145
|
`python -m syslogcef` is equivalent to the `syslogcef` entry point.
|
|
141
146
|
|
|
@@ -161,7 +166,8 @@ cef = to_cef(normalized, mapping="my_mapping.json")
|
|
|
161
166
|
```
|
|
162
167
|
|
|
163
168
|
Bundled mappings are importable from `syslogcef.mappings` (`CISCO_ASA`,
|
|
164
|
-
`CISCO_IOS`, `F5`, `LINUX`, `VMWARE`, or
|
|
169
|
+
`CISCO_IOS`, `F5`, `FORTINET`, `LINUX`, `SOPHOS`, `VMWARE`, or
|
|
170
|
+
`load_mapping(name)`).
|
|
165
171
|
|
|
166
172
|
## Mapping Files
|
|
167
173
|
|
|
@@ -181,7 +187,7 @@ normalized event's fields:
|
|
|
181
187
|
"src": "%(src)s",
|
|
182
188
|
"dst": "%(dst)s",
|
|
183
189
|
"cs1Label": "rawEvent",
|
|
184
|
-
"cs1": "%(
|
|
190
|
+
"cs1": "%(raw)s"
|
|
185
191
|
}
|
|
186
192
|
}
|
|
187
193
|
```
|
|
@@ -191,7 +197,8 @@ normalized event's fields:
|
|
|
191
197
|
- `severity_map` translates syslog severity (0-7) to CEF severity (0-10);
|
|
192
198
|
unmapped values pass through.
|
|
193
199
|
- `extensions` maps CEF extension keys to templates. Extensions that
|
|
194
|
-
resolve to an empty value are omitted
|
|
200
|
+
resolve to an empty value are omitted, and so is a custom-field label
|
|
201
|
+
(`cs1Label`, `cn1Label`, ...) whose paired value is omitted.
|
|
195
202
|
- Available template fields include `host`, `app`, `pid`, `msgid`, `msg`,
|
|
196
203
|
`message_short` (first 120 characters), `raw`, `raw_kv`, `event_code`,
|
|
197
204
|
`facility`, `severity`, `ts`, plus every key=value pair extracted from
|
|
@@ -204,6 +211,18 @@ extension keys fail with a configuration error instead of corrupting a record.
|
|
|
204
211
|
See [docs/cef_fields.md](docs/cef_fields.md) for the full CEF extension
|
|
205
212
|
dictionary.
|
|
206
213
|
|
|
214
|
+
## Timestamps without a year
|
|
215
|
+
|
|
216
|
+
RFC3164, journald short, Cisco console lines and many device formats
|
|
217
|
+
stamp `Jan 1 12:34:56` with no year. syslogcef dates such a timestamp to
|
|
218
|
+
its most recent occurrence that is not more than one day ahead of the
|
|
219
|
+
current time: log lines describe events that already happened. A `Jan 1`
|
|
220
|
+
line converted in November is dated January of the current year, while a
|
|
221
|
+
line stamped just after midnight on New Year's Eve (a sender clock or
|
|
222
|
+
timezone slightly ahead) lands in the new year. Pass `now=` to
|
|
223
|
+
`parse_syslog`/`convert_line` to date archived files relative to when
|
|
224
|
+
they were written.
|
|
225
|
+
|
|
207
226
|
## Field Dictionary
|
|
208
227
|
|
|
209
228
|
`syslogcef/dictionary.json` is the single source of truth for CEF field
|
|
@@ -255,9 +274,9 @@ syslogcef acme.log --patterns /etc/syslogcef/patterns.json
|
|
|
255
274
|
- Named groups map to event fields: `pri`, `host`, `app`, `pid`,
|
|
256
275
|
`msgid`, `msg`, and `ts`.
|
|
257
276
|
- A `ts` group requires `timestamp_format`: a strptime format,
|
|
258
|
-
`iso8601`, or `epoch`. Yearless formats get the same year
|
|
259
|
-
|
|
260
|
-
drops the event.
|
|
277
|
+
`iso8601`, or `epoch`. Yearless formats get the same year inference
|
|
278
|
+
as the built-in parsers (see "Timestamps without a year" below); an
|
|
279
|
+
unparseable timestamp never drops the event.
|
|
261
280
|
- `priority` is `"after"` (default: tried after the built-ins, before
|
|
262
281
|
the adaptive fallback) or `"before"` (tried first, overriding
|
|
263
282
|
built-ins).
|
|
@@ -388,6 +407,23 @@ pytest
|
|
|
388
407
|
Contributions are welcome — see [CONTRIBUTING.md](CONTRIBUTING.md). Notable
|
|
389
408
|
changes are tracked in [CHANGELOG.md](CHANGELOG.md).
|
|
390
409
|
|
|
410
|
+
## Citation
|
|
411
|
+
|
|
412
|
+
If you use syslog2cef in research or published work, please cite it. The
|
|
413
|
+
metadata is in [CITATION.cff](CITATION.cff), and GitHub's "Cite this
|
|
414
|
+
repository" button produces APA and BibTeX from it.
|
|
415
|
+
|
|
416
|
+
```bibtex
|
|
417
|
+
@software{suliman_syslog2cef,
|
|
418
|
+
author = {Suliman, Tamir},
|
|
419
|
+
title = {syslog2cef: Convert syslog to ArcSight Common Event Format (CEF)},
|
|
420
|
+
version = {0.4.0},
|
|
421
|
+
year = {2026},
|
|
422
|
+
url = {https://github.com/allamiro/syslog2cef},
|
|
423
|
+
license = {MIT}
|
|
424
|
+
}
|
|
425
|
+
```
|
|
426
|
+
|
|
391
427
|
## License
|
|
392
428
|
|
|
393
429
|
MIT — see [LICENSE](LICENSE). Copyright (c) Tamir Suliman.
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "syslog2cef"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.4.0"
|
|
8
8
|
description = "Convert RFC3164/RFC5424 syslog lines into ArcSight CEF"
|
|
9
9
|
authors = [{name = "Tamir Suliman", email = "allamiro@gmail.com"}]
|
|
10
10
|
readme = "README.md"
|
|
@@ -12,6 +12,12 @@ requires-python = ">=3.9"
|
|
|
12
12
|
classifiers = [
|
|
13
13
|
"Programming Language :: Python :: 3",
|
|
14
14
|
"Programming Language :: Python :: 3 :: Only",
|
|
15
|
+
"Programming Language :: Python :: 3.9",
|
|
16
|
+
"Programming Language :: Python :: 3.10",
|
|
17
|
+
"Programming Language :: Python :: 3.11",
|
|
18
|
+
"Programming Language :: Python :: 3.12",
|
|
19
|
+
"Programming Language :: Python :: 3.13",
|
|
20
|
+
"Programming Language :: Python :: 3.14",
|
|
15
21
|
"License :: OSI Approved :: MIT License",
|
|
16
22
|
"Topic :: System :: Logging",
|
|
17
23
|
]
|
|
@@ -1,7 +1,49 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: syslog2cef
|
|
3
|
+
Version: 0.4.0
|
|
4
|
+
Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
|
|
5
|
+
Author-email: Tamir Suliman <allamiro@gmail.com>
|
|
6
|
+
Project-URL: Homepage, https://github.com/allamiro/syslogcef
|
|
7
|
+
Project-URL: Issues, https://github.com/allamiro/syslogcef/issues
|
|
8
|
+
Keywords: syslog,cef,logging,security
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
17
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
18
|
+
Classifier: Topic :: System :: Logging
|
|
19
|
+
Requires-Python: >=3.9
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Provides-Extra: test
|
|
23
|
+
Requires-Dist: pytest>=7; extra == "test"
|
|
24
|
+
Provides-Extra: kafka
|
|
25
|
+
Requires-Dist: kafka-python>=2.0; extra == "kafka"
|
|
26
|
+
Provides-Extra: fuzz
|
|
27
|
+
Requires-Dist: hypothesis>=6; extra == "fuzz"
|
|
28
|
+
Dynamic: license-file
|
|
29
|
+
|
|
1
30
|
<p align="center">
|
|
2
31
|
<img src="https://raw.githubusercontent.com/allamiro/syslogcef/main/docs/logo.svg" alt="syslogcef logo" width="540">
|
|
3
32
|
</p>
|
|
4
33
|
|
|
34
|
+
<p align="center">
|
|
35
|
+
<a href="https://github.com/allamiro/syslog2cef/actions/workflows/ci.yml"><img src="https://img.shields.io/github/actions/workflow/status/allamiro/syslog2cef/ci.yml?branch=main&label=CI" alt="CI status"></a>
|
|
36
|
+
<a href="https://pypi.org/project/syslog2cef/"><img src="https://img.shields.io/pypi/v/syslog2cef" alt="PyPI version"></a>
|
|
37
|
+
<a href="https://pypi.org/project/syslog2cef/"><img src="https://img.shields.io/pypi/pyversions/syslog2cef" alt="Supported Python versions"></a>
|
|
38
|
+
<a href="https://pypistats.org/packages/syslog2cef"><img src="https://img.shields.io/pypi/dm/syslog2cef" alt="PyPI downloads per month"></a>
|
|
39
|
+
<a href="https://copr.fedorainfracloud.org/coprs/allamiro/syslogcef/"><img src="https://copr.fedorainfracloud.org/coprs/allamiro/syslogcef/package/syslogcef/status_image/last_build.png" alt="COPR build status"></a>
|
|
40
|
+
<a href="https://github.com/allamiro/syslog2cef/pkgs/container/syslogcef"><img src="https://img.shields.io/badge/container-ghcr.io-blue?logo=docker" alt="Container image on ghcr.io"></a>
|
|
41
|
+
<a href="https://github.com/allamiro/syslog2cef/actions/workflows/fuzz.yml"><img src="https://img.shields.io/github/actions/workflow/status/allamiro/syslog2cef/fuzz.yml?branch=main&label=deep%20fuzz" alt="Deep fuzz status"></a>
|
|
42
|
+
<a href="https://github.com/astral-sh/ruff"><img src="https://img.shields.io/badge/lint-ruff-261230" alt="Linted with Ruff"></a>
|
|
43
|
+
<a href="LICENSE"><img src="https://img.shields.io/github/license/allamiro/syslog2cef" alt="MIT license"></a>
|
|
44
|
+
<a href="CITATION.cff"><img src="https://img.shields.io/badge/cite-CITATION.cff-informational" alt="Citation file"></a>
|
|
45
|
+
</p>
|
|
46
|
+
|
|
5
47
|
# syslogcef
|
|
6
48
|
|
|
7
49
|
syslogcef converts syslog events into ArcSight Common Event Format (CEF). It
|
|
@@ -13,11 +55,18 @@ platforms that consume CEF.
|
|
|
13
55
|
|
|
14
56
|
- Automatic detection and parsing of multiple syslog dialects:
|
|
15
57
|
RFC3164 (BSD), RFC5424 (including structured data), rsyslog JSON and
|
|
16
|
-
file formats,
|
|
58
|
+
file formats, systemd journal exports (JSON, short, and ISO formats),
|
|
59
|
+
Cisco console/buffer lines, ISO-timestamp syslog, and key=value streams
|
|
60
|
+
(Fortinet, Sophos). Unknown layouts are learned adaptively.
|
|
17
61
|
- Deterministic vendor/product mappings bundled for Cisco ASA, Cisco IOS,
|
|
18
|
-
F5 BIG-IP,
|
|
19
|
-
selection based on message content.
|
|
20
|
-
- Custom mappings supplied as JSON files or Python dictionaries
|
|
62
|
+
F5 BIG-IP, Fortinet FortiGate, Sophos XG Firewall, VMware ESXi, and
|
|
63
|
+
generic Linux, with automatic mapping selection based on message content.
|
|
64
|
+
- Custom mappings supplied as JSON files or Python dictionaries, and custom
|
|
65
|
+
parsers supplied as regex pattern files.
|
|
66
|
+
- Validation of CEF extension values against the ArcSight extension
|
|
67
|
+
dictionary (`--validate`, `--strict`).
|
|
68
|
+
- Network receiver (`--listen udp:514`) and forwarding to UDP, TCP, or
|
|
69
|
+
Kafka (`--send`), with optional rate limiting.
|
|
21
70
|
- Escaping of untrusted log content in both CEF header and extension
|
|
22
71
|
fields, so crafted messages cannot forge header fields or split records.
|
|
23
72
|
- Streaming CLI with stdin/file input, tail (follow) mode across multiple
|
|
@@ -106,13 +155,21 @@ Options:
|
|
|
106
155
|
| Option | Description |
|
|
107
156
|
| ------ | ----------- |
|
|
108
157
|
| `paths` | Input files; stdin is used when omitted. |
|
|
109
|
-
| `-o, --output FILE` | Write CEF lines to a file instead of stdout. |
|
|
110
|
-
|
|
|
158
|
+
| `-o, --output FILE` | Write CEF lines to a file instead of stdout. Accepts strftime codes for timed files (see "Timed output files"). |
|
|
159
|
+
| `-a, --append` | Append to `--output` instead of truncating it (implied for templated paths). |
|
|
160
|
+
| `--mode MODE` | Parser override: `rfc3164`, `rfc5424`, `rsyslog_json`, `rsyslog_file`, `journald_json`, `journald_short`, `journald_iso`, `cisco_seq`, `iso_syslog`, `kv`, or a name from `--patterns`. Auto-detected when omitted. |
|
|
111
161
|
| `--mapping FILE` | Mapping JSON file. Auto-selected from message content when omitted. |
|
|
162
|
+
| `--patterns FILE` | Custom parser patterns (see "Custom Parsers"); may be repeated. |
|
|
112
163
|
| `--tail` | Follow input files like `tail -f`. |
|
|
164
|
+
| `--listen PROTO:PORT` | Receive syslog over the network instead of files/stdin: `udp:514`, `tcp:5514`, or `udp:10.0.0.5:514`. |
|
|
165
|
+
| `--send URL` | Forward CEF to `udp://HOST:PORT`, `tcp://HOST:PORT`, or `kafka://BROKER:PORT/TOPIC` (needs the `kafka` extra). |
|
|
166
|
+
| `--eps N` | Rate-limit `--send` to at most N events per second. |
|
|
113
167
|
| `--multiprocess` | Convert lines using a process pool. |
|
|
114
168
|
| `--pool-size N` | Worker count for `--multiprocess` (default: CPU count minus one). |
|
|
169
|
+
| `--validate` | Check CEF extensions against the ArcSight dictionary; violations are logged as warnings. |
|
|
170
|
+
| `--strict` | Like `--validate`, but exit with an error on type or length violations. |
|
|
115
171
|
| `--log-level LEVEL` | Python logging level (default `WARNING`). |
|
|
172
|
+
| `--version` | Print the version, copyright, license and author, then exit. |
|
|
116
173
|
|
|
117
174
|
`python -m syslogcef` is equivalent to the `syslogcef` entry point.
|
|
118
175
|
|
|
@@ -138,7 +195,8 @@ cef = to_cef(normalized, mapping="my_mapping.json")
|
|
|
138
195
|
```
|
|
139
196
|
|
|
140
197
|
Bundled mappings are importable from `syslogcef.mappings` (`CISCO_ASA`,
|
|
141
|
-
`CISCO_IOS`, `F5`, `LINUX`, `VMWARE`, or
|
|
198
|
+
`CISCO_IOS`, `F5`, `FORTINET`, `LINUX`, `SOPHOS`, `VMWARE`, or
|
|
199
|
+
`load_mapping(name)`).
|
|
142
200
|
|
|
143
201
|
## Mapping Files
|
|
144
202
|
|
|
@@ -158,7 +216,7 @@ normalized event's fields:
|
|
|
158
216
|
"src": "%(src)s",
|
|
159
217
|
"dst": "%(dst)s",
|
|
160
218
|
"cs1Label": "rawEvent",
|
|
161
|
-
"cs1": "%(
|
|
219
|
+
"cs1": "%(raw)s"
|
|
162
220
|
}
|
|
163
221
|
}
|
|
164
222
|
```
|
|
@@ -168,7 +226,8 @@ normalized event's fields:
|
|
|
168
226
|
- `severity_map` translates syslog severity (0-7) to CEF severity (0-10);
|
|
169
227
|
unmapped values pass through.
|
|
170
228
|
- `extensions` maps CEF extension keys to templates. Extensions that
|
|
171
|
-
resolve to an empty value are omitted
|
|
229
|
+
resolve to an empty value are omitted, and so is a custom-field label
|
|
230
|
+
(`cs1Label`, `cn1Label`, ...) whose paired value is omitted.
|
|
172
231
|
- Available template fields include `host`, `app`, `pid`, `msgid`, `msg`,
|
|
173
232
|
`message_short` (first 120 characters), `raw`, `raw_kv`, `event_code`,
|
|
174
233
|
`facility`, `severity`, `ts`, plus every key=value pair extracted from
|
|
@@ -181,6 +240,18 @@ extension keys fail with a configuration error instead of corrupting a record.
|
|
|
181
240
|
See [docs/cef_fields.md](docs/cef_fields.md) for the full CEF extension
|
|
182
241
|
dictionary.
|
|
183
242
|
|
|
243
|
+
## Timestamps without a year
|
|
244
|
+
|
|
245
|
+
RFC3164, journald short, Cisco console lines and many device formats
|
|
246
|
+
stamp `Jan 1 12:34:56` with no year. syslogcef dates such a timestamp to
|
|
247
|
+
its most recent occurrence that is not more than one day ahead of the
|
|
248
|
+
current time: log lines describe events that already happened. A `Jan 1`
|
|
249
|
+
line converted in November is dated January of the current year, while a
|
|
250
|
+
line stamped just after midnight on New Year's Eve (a sender clock or
|
|
251
|
+
timezone slightly ahead) lands in the new year. Pass `now=` to
|
|
252
|
+
`parse_syslog`/`convert_line` to date archived files relative to when
|
|
253
|
+
they were written.
|
|
254
|
+
|
|
184
255
|
## Field Dictionary
|
|
185
256
|
|
|
186
257
|
`syslogcef/dictionary.json` is the single source of truth for CEF field
|
|
@@ -232,9 +303,9 @@ syslogcef acme.log --patterns /etc/syslogcef/patterns.json
|
|
|
232
303
|
- Named groups map to event fields: `pri`, `host`, `app`, `pid`,
|
|
233
304
|
`msgid`, `msg`, and `ts`.
|
|
234
305
|
- A `ts` group requires `timestamp_format`: a strptime format,
|
|
235
|
-
`iso8601`, or `epoch`. Yearless formats get the same year
|
|
236
|
-
|
|
237
|
-
drops the event.
|
|
306
|
+
`iso8601`, or `epoch`. Yearless formats get the same year inference
|
|
307
|
+
as the built-in parsers (see "Timestamps without a year" below); an
|
|
308
|
+
unparseable timestamp never drops the event.
|
|
238
309
|
- `priority` is `"after"` (default: tried after the built-ins, before
|
|
239
310
|
the adaptive fallback) or `"before"` (tried first, overriding
|
|
240
311
|
built-ins).
|
|
@@ -365,6 +436,23 @@ pytest
|
|
|
365
436
|
Contributions are welcome — see [CONTRIBUTING.md](CONTRIBUTING.md). Notable
|
|
366
437
|
changes are tracked in [CHANGELOG.md](CHANGELOG.md).
|
|
367
438
|
|
|
439
|
+
## Citation
|
|
440
|
+
|
|
441
|
+
If you use syslog2cef in research or published work, please cite it. The
|
|
442
|
+
metadata is in [CITATION.cff](CITATION.cff), and GitHub's "Cite this
|
|
443
|
+
repository" button produces APA and BibTeX from it.
|
|
444
|
+
|
|
445
|
+
```bibtex
|
|
446
|
+
@software{suliman_syslog2cef,
|
|
447
|
+
author = {Suliman, Tamir},
|
|
448
|
+
title = {syslog2cef: Convert syslog to ArcSight Common Event Format (CEF)},
|
|
449
|
+
version = {0.4.0},
|
|
450
|
+
year = {2026},
|
|
451
|
+
url = {https://github.com/allamiro/syslog2cef},
|
|
452
|
+
license = {MIT}
|
|
453
|
+
}
|
|
454
|
+
```
|
|
455
|
+
|
|
368
456
|
## License
|
|
369
457
|
|
|
370
458
|
MIT — see [LICENSE](LICENSE). Copyright (c) Tamir Suliman.
|
|
@@ -3,6 +3,7 @@ README.md
|
|
|
3
3
|
pyproject.toml
|
|
4
4
|
./syslogcef/__init__.py
|
|
5
5
|
./syslogcef/__main__.py
|
|
6
|
+
./syslogcef/_version.py
|
|
6
7
|
./syslogcef/adaptive.py
|
|
7
8
|
./syslogcef/api.py
|
|
8
9
|
./syslogcef/cef.py
|
|
@@ -31,6 +32,7 @@ syslog2cef.egg-info/requires.txt
|
|
|
31
32
|
syslog2cef.egg-info/top_level.txt
|
|
32
33
|
syslogcef/__init__.py
|
|
33
34
|
syslogcef/__main__.py
|
|
35
|
+
syslogcef/_version.py
|
|
34
36
|
syslogcef/adaptive.py
|
|
35
37
|
syslogcef/api.py
|
|
36
38
|
syslogcef/cef.py
|
|
@@ -9,10 +9,16 @@ conversion workflow.
|
|
|
9
9
|
"""
|
|
10
10
|
from __future__ import annotations
|
|
11
11
|
|
|
12
|
+
from ._version import __author__, __copyright__, __email__, __license__, __version__
|
|
12
13
|
from .api import StreamConverter, convert_line, normalize_event, parse_syslog, to_cef
|
|
13
14
|
from .custom import PatternFileError, load_patterns, register_parser
|
|
14
15
|
|
|
15
16
|
__all__ = [
|
|
17
|
+
"__version__",
|
|
18
|
+
"__author__",
|
|
19
|
+
"__email__",
|
|
20
|
+
"__copyright__",
|
|
21
|
+
"__license__",
|
|
16
22
|
"StreamConverter",
|
|
17
23
|
"convert_line",
|
|
18
24
|
"normalize_event",
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""Package version and author, shown by ``syslogcef --version``.
|
|
2
|
+
|
|
3
|
+
Kept in step with ``version`` in pyproject.toml (checked by the test
|
|
4
|
+
suite). A module constant rather than importlib.metadata, because the
|
|
5
|
+
standalone .pyz has no installed distribution metadata to read.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
__version__ = "0.4.0"
|
|
9
|
+
__author__ = "Tamir Suliman"
|
|
10
|
+
__email__ = "allamiro@gmail.com"
|
|
11
|
+
__copyright__ = "Copyright (c) 2023 Tamir Suliman"
|
|
12
|
+
__license__ = "MIT"
|
|
@@ -2,6 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
4
|
from dataclasses import dataclass
|
|
5
|
+
from functools import lru_cache
|
|
5
6
|
from typing import Any, Dict, Mapping
|
|
6
7
|
|
|
7
8
|
from .normalizer import NormalizedEvent
|
|
@@ -15,6 +16,11 @@ HEADER_KEYS = (
|
|
|
15
16
|
"deviceVendor", "deviceProduct", "deviceVersion", "eventClassId", "name",
|
|
16
17
|
)
|
|
17
18
|
|
|
19
|
+
# Custom-field labels pair with the key they describe: cs1Label -> cs1.
|
|
20
|
+
# Only the dictionary's own *Label keys pair this way; a custom extension
|
|
21
|
+
# that merely ends in "Label" (classificationLabel) is independent data.
|
|
22
|
+
_LABEL_SUFFIX = "Label"
|
|
23
|
+
|
|
18
24
|
|
|
19
25
|
DEFAULT_MAPPING = {
|
|
20
26
|
"deviceVendor": "Generic",
|
|
@@ -54,6 +60,19 @@ DEFAULT_MAPPING = {
|
|
|
54
60
|
}
|
|
55
61
|
|
|
56
62
|
|
|
63
|
+
@lru_cache(maxsize=1)
|
|
64
|
+
def _label_pairs() -> Dict[str, str]:
|
|
65
|
+
"""Dictionary custom-field label -> the key it labels (cs1Label -> cs1)."""
|
|
66
|
+
from .dictionary import cef_keys
|
|
67
|
+
|
|
68
|
+
keys = cef_keys()
|
|
69
|
+
return {
|
|
70
|
+
key: key[: -len(_LABEL_SUFFIX)]
|
|
71
|
+
for key in keys
|
|
72
|
+
if key.endswith(_LABEL_SUFFIX) and key[: -len(_LABEL_SUFFIX)] in keys
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
|
|
57
76
|
@dataclass
|
|
58
77
|
class CEFEvent:
|
|
59
78
|
header: Dict[str, str]
|
|
@@ -133,6 +152,14 @@ class MappingResolver:
|
|
|
133
152
|
value = self._format(template, fields)
|
|
134
153
|
if value:
|
|
135
154
|
raw[key] = str(value)
|
|
155
|
+
# A label (cs1Label, cn1Label, flexString1Label, ...) is a constant
|
|
156
|
+
# and always renders, while its paired value is dropped when the
|
|
157
|
+
# template resolves empty. A label without its value describes
|
|
158
|
+
# nothing, so drop it too (#111).
|
|
159
|
+
labels = _label_pairs()
|
|
160
|
+
for key in [k for k in raw if k in labels]:
|
|
161
|
+
if labels[key] not in raw:
|
|
162
|
+
del raw[key]
|
|
136
163
|
return raw
|
|
137
164
|
|
|
138
165
|
def resolve_extensions(self, fields: Mapping[str, Any]) -> Dict[str, str]:
|
|
@@ -10,6 +10,7 @@ from multiprocessing import Pool, cpu_count
|
|
|
10
10
|
from pathlib import Path
|
|
11
11
|
from typing import Any, Iterable, Iterator, Optional
|
|
12
12
|
|
|
13
|
+
from ._version import __author__, __copyright__, __email__, __license__, __version__
|
|
13
14
|
from .api import StreamConverter, convert_line
|
|
14
15
|
|
|
15
16
|
logger = logging.getLogger(__name__)
|
|
@@ -381,8 +382,27 @@ def _optional_path(value: str) -> Optional[Path]:
|
|
|
381
382
|
return Path(value) if value else None
|
|
382
383
|
|
|
383
384
|
|
|
385
|
+
# GNU-style --version text: name and version first (scripts parse it),
|
|
386
|
+
# then copyright, license, and authorship.
|
|
387
|
+
VERSION_TEXT = (
|
|
388
|
+
f"syslogcef {__version__}\n"
|
|
389
|
+
f"{__copyright__}\n"
|
|
390
|
+
f"License {__license__}: <https://opensource.org/licenses/{__license__}>\n"
|
|
391
|
+
"This is free software: you are free to change and redistribute it.\n"
|
|
392
|
+
"There is NO WARRANTY, to the extent permitted by law.\n"
|
|
393
|
+
"\n"
|
|
394
|
+
f"Written by {__author__} <{__email__}>."
|
|
395
|
+
)
|
|
396
|
+
|
|
397
|
+
|
|
384
398
|
def main(argv: Optional[list[str]] = None) -> int:
|
|
385
|
-
|
|
399
|
+
# RawDescriptionHelpFormatter keeps VERSION_TEXT's line breaks; argparse
|
|
400
|
+
# otherwise reflows the version string onto one line. Option help is
|
|
401
|
+
# still wrapped as before.
|
|
402
|
+
parser = argparse.ArgumentParser(
|
|
403
|
+
description="Convert syslog lines to ArcSight CEF",
|
|
404
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
405
|
+
)
|
|
386
406
|
parser.add_argument("paths", nargs="*", type=Path, help="Input files (defaults to stdin)")
|
|
387
407
|
parser.add_argument("-o", "--output", type=_optional_path, help="Output file; strftime codes such as %%Y-%%m-%%d or %%H start a new file when the rendered path changes (templated paths always append and parent directories are created). Use %%%% for a literal percent; unsupported codes are rejected. An empty value means stdout.")
|
|
388
408
|
parser.add_argument("-a", "--append", action="store_true", help="Append to --output instead of truncating it (implied for templated paths)")
|
|
@@ -398,6 +418,12 @@ def main(argv: Optional[list[str]] = None) -> int:
|
|
|
398
418
|
parser.add_argument("--validate", action="store_true", help="Validate CEF extensions against the ArcSight dictionary; violations are logged as warnings")
|
|
399
419
|
parser.add_argument("--strict", action="store_true", help="Like --validate, but exit with an error on type/length violations")
|
|
400
420
|
parser.add_argument("--log-level", default="WARNING", help="Logging level")
|
|
421
|
+
parser.add_argument(
|
|
422
|
+
"--version",
|
|
423
|
+
action="version",
|
|
424
|
+
version=VERSION_TEXT,
|
|
425
|
+
help="Show the version, license and author, then exit",
|
|
426
|
+
)
|
|
401
427
|
|
|
402
428
|
args = parser.parse_args(argv)
|
|
403
429
|
logging.basicConfig(level=getattr(logging, args.log_level.upper(), logging.WARNING))
|
|
@@ -40,12 +40,12 @@ from __future__ import annotations
|
|
|
40
40
|
import json
|
|
41
41
|
import re
|
|
42
42
|
from dataclasses import dataclass
|
|
43
|
-
from datetime import datetime,
|
|
43
|
+
from datetime import datetime, timezone
|
|
44
44
|
from pathlib import Path
|
|
45
45
|
from typing import Callable, Dict, List, Optional, Union
|
|
46
46
|
|
|
47
47
|
from .parsers import PARSERS, ParsedEvent
|
|
48
|
-
from .utils import convert_pri, parse_iso8601
|
|
48
|
+
from .utils import convert_pri, latest_past_occurrence, parse_iso8601
|
|
49
49
|
|
|
50
50
|
_EVENT_GROUPS = {"pri", "host", "app", "pid", "msgid", "msg"}
|
|
51
51
|
_TS_SPECIAL_FORMATS = {"iso8601", "epoch"}
|
|
@@ -126,10 +126,10 @@ def _validate_strptime_format(fmt: str) -> bool:
|
|
|
126
126
|
|
|
127
127
|
|
|
128
128
|
def _year_with_rollover(parsed: datetime, ref: datetime) -> datetime:
|
|
129
|
-
"""Assign the
|
|
129
|
+
"""Assign the year to a yearless timestamp, as the built-in parsers do.
|
|
130
130
|
|
|
131
|
-
Keeps every parsed component (tz offset, microseconds);
|
|
132
|
-
|
|
131
|
+
Keeps every parsed component (tz offset, microseconds); see
|
|
132
|
+
:func:`syslogcef.utils.latest_past_occurrence` for the rule.
|
|
133
133
|
"""
|
|
134
134
|
|
|
135
135
|
def with_year(year: int) -> Optional[datetime]:
|
|
@@ -138,19 +138,7 @@ def _year_with_rollover(parsed: datetime, ref: datetime) -> datetime:
|
|
|
138
138
|
except ValueError: # Feb 29 in a non-leap year
|
|
139
139
|
return None
|
|
140
140
|
|
|
141
|
-
|
|
142
|
-
if candidate is None:
|
|
143
|
-
# Feb 29: pick the occurrence closest in time, not the first
|
|
144
|
-
# leap year found in either direction.
|
|
145
|
-
candidates = [c for off in range(-4, 5) if off and (c := with_year(ref.year + off))]
|
|
146
|
-
if not candidates:
|
|
147
|
-
return parsed
|
|
148
|
-
return min(candidates, key=lambda c: abs(c - ref))
|
|
149
|
-
if candidate - ref > timedelta(days=180):
|
|
150
|
-
return with_year(ref.year - 1) or candidate
|
|
151
|
-
if ref - candidate > timedelta(days=180):
|
|
152
|
-
return with_year(ref.year + 1) or candidate
|
|
153
|
-
return candidate
|
|
141
|
+
return latest_past_occurrence(with_year, ref) or parsed
|
|
154
142
|
|
|
155
143
|
|
|
156
144
|
def _parse_timestamp(text: str, fmt: str, *, now: Optional[datetime] = None) -> datetime:
|
|
@@ -12,6 +12,7 @@ from .utils import (
|
|
|
12
12
|
JournalJSONMapping,
|
|
13
13
|
convert_pri,
|
|
14
14
|
guess_hostname,
|
|
15
|
+
latest_past_occurrence,
|
|
15
16
|
month_abbr_to_int,
|
|
16
17
|
parse_iso8601,
|
|
17
18
|
parse_key_value_pairs,
|
|
@@ -151,20 +152,9 @@ def _infer_timestamp(month: int, day: int, time_str: str, *, now: Optional[datet
|
|
|
151
152
|
except ValueError: # Feb 29 in a non-leap year
|
|
152
153
|
return None
|
|
153
154
|
|
|
154
|
-
candidate = build
|
|
155
|
+
candidate = latest_past_occurrence(build, now)
|
|
155
156
|
if candidate is None:
|
|
156
|
-
|
|
157
|
-
# crash the parser: pick the occurrence closest in time (not
|
|
158
|
-
# merely the first leap year found in either direction).
|
|
159
|
-
candidates = [c for off in range(-4, 5) if off and (c := build(now.year + off))]
|
|
160
|
-
if not candidates:
|
|
161
|
-
raise ValueError(f"impossible calendar date: month={month} day={day}")
|
|
162
|
-
return min(candidates, key=lambda c: abs(c - now))
|
|
163
|
-
# Handle rollover into previous year.
|
|
164
|
-
if candidate - now > timedelta(days=180):
|
|
165
|
-
candidate = build(now.year - 1) or candidate
|
|
166
|
-
elif now - candidate > timedelta(days=180):
|
|
167
|
-
candidate = build(now.year + 1) or candidate
|
|
157
|
+
raise ValueError(f"impossible calendar date: month={month} day={day}")
|
|
168
158
|
return candidate
|
|
169
159
|
|
|
170
160
|
|
|
@@ -5,8 +5,8 @@ import logging
|
|
|
5
5
|
import re
|
|
6
6
|
import socket
|
|
7
7
|
from dataclasses import dataclass
|
|
8
|
-
from datetime import datetime, timezone
|
|
9
|
-
from typing import Any, Dict, Mapping, Optional
|
|
8
|
+
from datetime import datetime, timedelta, timezone
|
|
9
|
+
from typing import Any, Callable, Dict, Mapping, Optional
|
|
10
10
|
|
|
11
11
|
logger = logging.getLogger(__name__)
|
|
12
12
|
|
|
@@ -104,6 +104,32 @@ def parse_iso8601(value: str) -> datetime:
|
|
|
104
104
|
raise ValueError(f"Invalid ISO8601 timestamp: {value!r}") from exc
|
|
105
105
|
|
|
106
106
|
|
|
107
|
+
# How far ahead of "now" a yearless timestamp may sit and still count as
|
|
108
|
+
# the current occurrence: covers senders in timezones ahead of UTC (up to
|
|
109
|
+
# +14h, since yearless local times are read as UTC) plus modest clock skew.
|
|
110
|
+
YEARLESS_FUTURE_TOLERANCE = timedelta(days=1)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def latest_past_occurrence(
|
|
114
|
+
build: Callable[[int], Optional[datetime]], now: datetime
|
|
115
|
+
) -> Optional[datetime]:
|
|
116
|
+
"""Pick the year for a timestamp that carries none.
|
|
117
|
+
|
|
118
|
+
``build(year)`` returns the timestamp in that year, or ``None`` when the
|
|
119
|
+
date does not exist then (Feb 29). Logs describe events that already
|
|
120
|
+
happened, so the answer is the most recent occurrence no later than
|
|
121
|
+
``now`` plus :data:`YEARLESS_FUTURE_TOLERANCE`. Next year only wins at
|
|
122
|
+
the year boundary (a Jan 1 00:00:05 line received at Dec 31 23:59:58).
|
|
123
|
+
Leap days recur at most eight years apart, which bounds the search.
|
|
124
|
+
"""
|
|
125
|
+
|
|
126
|
+
for year in range(now.year + 1, now.year - 9, -1):
|
|
127
|
+
candidate = build(year)
|
|
128
|
+
if candidate is not None and candidate - now <= YEARLESS_FUTURE_TOLERANCE:
|
|
129
|
+
return candidate
|
|
130
|
+
return None
|
|
131
|
+
|
|
132
|
+
|
|
107
133
|
def ensure_tzaware(ts: Optional[datetime]) -> Optional[datetime]:
|
|
108
134
|
if ts is None:
|
|
109
135
|
return None
|
|
@@ -184,3 +184,47 @@ def test_full_conversion_syntax_is_accepted(template):
|
|
|
184
184
|
# Flags, width, precision and length modifiers are all valid syntax;
|
|
185
185
|
# whether a given value satisfies the conversion is data-dependent.
|
|
186
186
|
assert convert_line("plain log", mapping={"name": template}).startswith("CEF:0|")
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _extensions(cef: str) -> str:
|
|
190
|
+
return cef.split("|", 7)[7]
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def test_asa_line_without_kv_keeps_raw_event_value():
|
|
194
|
+
# Issue #111: the ASA mapping took cs1 from raw_kv, which is empty for
|
|
195
|
+
# most ASA messages, so cs1 was dropped but cs1Label=rawEvent remained.
|
|
196
|
+
line = "<166>Jan 1 12:34:56 fw1 %ASA-6-302013: Built inbound TCP connection"
|
|
197
|
+
ext = _extensions(convert_line(line))
|
|
198
|
+
|
|
199
|
+
assert "cs1Label=rawEvent" in ext
|
|
200
|
+
assert "cs1=<166>Jan 1 12:34:56 fw1 %ASA-6-302013: Built inbound TCP connection" in ext
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
@pytest.mark.parametrize("name", ["cisco_asa", "cisco_ios", "f5", "fortinet", "linux", "sophos", "vmware"])
|
|
204
|
+
def test_bundled_mappings_carry_the_raw_line_in_cs1(name):
|
|
205
|
+
assert load_mapping(name)["extensions"]["cs1"] == "%(raw)s"
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
@pytest.mark.parametrize(
|
|
209
|
+
"label, value",
|
|
210
|
+
[("cs2Label", "cs2"), ("cn1Label", "cn1"), ("c6a1Label", "c6a1"),
|
|
211
|
+
("deviceCustomDate1Label", "deviceCustomDate1"), ("flexString1Label", "flexString1")],
|
|
212
|
+
)
|
|
213
|
+
def test_label_is_dropped_when_its_value_is_empty(label, value):
|
|
214
|
+
mapping = {"extensions": {label: "Session", value: "%(session)s"}}
|
|
215
|
+
|
|
216
|
+
without = _extensions(convert_line("srcip=1.2.3.4 action=x", mapping=mapping))
|
|
217
|
+
assert f"{label}=" not in without
|
|
218
|
+
assert f" {value}=" not in f" {without}"
|
|
219
|
+
|
|
220
|
+
with_value = _extensions(convert_line("srcip=1.2.3.4 session=abc", mapping=mapping))
|
|
221
|
+
assert f"{label}=Session" in with_value
|
|
222
|
+
assert f"{value}=abc" in with_value
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def test_custom_key_ending_in_label_is_not_treated_as_a_label():
|
|
226
|
+
# Only the dictionary's custom-field labels pair with a value; an
|
|
227
|
+
# independent custom key that merely ends in "Label" keeps its data.
|
|
228
|
+
ext = _extensions(convert_line("plain log", mapping={"extensions": {"classificationLabel": "Confidential"}}))
|
|
229
|
+
|
|
230
|
+
assert "classificationLabel=Confidential" in ext
|
|
@@ -176,3 +176,46 @@ def test_cli_output_template_end_to_end(tmp_path: Path):
|
|
|
176
176
|
files = list((tmp_path / "cef").rglob("events.cef"))
|
|
177
177
|
assert len(files) == 1
|
|
178
178
|
assert "CEF:0" in files[0].read_text(encoding="utf-8")
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _run_cli(*args: str) -> subprocess.CompletedProcess:
|
|
182
|
+
return subprocess.run(
|
|
183
|
+
[sys.executable, "-m", "syslogcef", *args],
|
|
184
|
+
capture_output=True,
|
|
185
|
+
text=True,
|
|
186
|
+
check=True,
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def test_version_shows_version_and_author():
|
|
191
|
+
out = _run_cli("--version").stdout
|
|
192
|
+
|
|
193
|
+
from syslogcef import __version__
|
|
194
|
+
|
|
195
|
+
lines = out.splitlines()
|
|
196
|
+
# First line is "name version" so scripts can parse it.
|
|
197
|
+
assert lines[0] == f"syslogcef {__version__}"
|
|
198
|
+
assert "Copyright (c) 2023 Tamir Suliman" in lines
|
|
199
|
+
assert lines[-1] == "Written by Tamir Suliman <allamiro@gmail.com>."
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def test_conversion_output_has_no_author_line():
|
|
203
|
+
# stdout carries CEF records for SIEMs; nothing else may be printed.
|
|
204
|
+
result = subprocess.run(
|
|
205
|
+
[sys.executable, "-m", "syslogcef"],
|
|
206
|
+
input="<166>Jan 1 12:34:56 router1 %ASA-6-302013: Built inbound TCP connection\n",
|
|
207
|
+
capture_output=True,
|
|
208
|
+
text=True,
|
|
209
|
+
check=True,
|
|
210
|
+
)
|
|
211
|
+
assert result.stdout.splitlines()[0].startswith("CEF:0|")
|
|
212
|
+
assert len(result.stdout.splitlines()) == 1
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def test_package_version_matches_pyproject():
|
|
216
|
+
import re
|
|
217
|
+
|
|
218
|
+
from syslogcef import __version__
|
|
219
|
+
|
|
220
|
+
pyproject = (Path(__file__).resolve().parent.parent / "pyproject.toml").read_text(encoding="utf-8")
|
|
221
|
+
assert re.search(r'^version = "([^"]+)"', pyproject, re.M).group(1) == __version__
|
|
@@ -287,4 +287,21 @@ def test_cli_rejects_unknown_mode(tmp_path: Path):
|
|
|
287
287
|
from syslogcef.cli import main
|
|
288
288
|
|
|
289
289
|
with pytest.raises(SystemExit):
|
|
290
|
-
main(["--mode", "nonexistent"])
|
|
290
|
+
main(["--mode", "nonexistent"])
|
|
291
|
+
|
|
292
|
+
def test_yearless_pattern_timestamp_is_dated_in_the_past(tmp_path: Path):
|
|
293
|
+
# Issue #112: custom patterns share the built-in year inference.
|
|
294
|
+
load_patterns(
|
|
295
|
+
write_patterns(
|
|
296
|
+
tmp_path,
|
|
297
|
+
{"name": "past", "regex": r"^P (?P<ts>\w{3} \d{2} \d{2}:\d{2}:\d{2}) (?P<msg>.*)$", "timestamp_format": "%b %d %H:%M:%S"},
|
|
298
|
+
)
|
|
299
|
+
)
|
|
300
|
+
parse = custom.get_registered("past")
|
|
301
|
+
|
|
302
|
+
late_november = datetime(2026, 11, 20, tzinfo=timezone.utc)
|
|
303
|
+
assert parse("P Jan 01 12:34:56 x", now=late_november).ts.year == 2026
|
|
304
|
+
assert parse("P Nov 25 10:00:00 x", now=late_november).ts.year == 2025
|
|
305
|
+
|
|
306
|
+
new_years_eve = datetime(2026, 12, 31, 23, 59, 58, tzinfo=timezone.utc)
|
|
307
|
+
assert parse("P Jan 01 00:00:05 x", now=new_years_eve).ts.year == 2027
|
|
@@ -73,3 +73,47 @@ def test_rfc3164_year_rollover_previous_year():
|
|
|
73
73
|
line = "<14>Dec 31 23:59:59 host1 app: year rollover"
|
|
74
74
|
result = autodetect_and_parse(line, now=datetime(2024, 1, 2, tzinfo=timezone.utc))
|
|
75
75
|
assert result.ts.year == 2023
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
# Issue #112: a yearless timestamp describes an event that already
|
|
79
|
+
# happened, so it takes the most recent occurrence that is not later than
|
|
80
|
+
# now plus a small clock-skew/timezone tolerance — never a date months
|
|
81
|
+
# ahead just because it is nearer than last year's.
|
|
82
|
+
|
|
83
|
+
@pytest.mark.parametrize(
|
|
84
|
+
"line, now, year",
|
|
85
|
+
[
|
|
86
|
+
# Jan 1 seen in late November: earlier this year, not next year.
|
|
87
|
+
("<14>Jan 1 12:34:56 h1 sshd[22]: x", datetime(2026, 11, 20, tzinfo=timezone.utc), 2026),
|
|
88
|
+
# Year boundary: a line stamped just after midnight by a clock
|
|
89
|
+
# slightly ahead lands in the year that is about to begin.
|
|
90
|
+
("<14>Jan 1 00:00:05 h1 app: x", datetime(2026, 12, 31, 23, 59, 58, tzinfo=timezone.utc), 2027),
|
|
91
|
+
# Later today (sender in a timezone ahead of UTC) stays this year.
|
|
92
|
+
("<14>Nov 20 22:00:00 h1 app: x", datetime(2026, 11, 20, 9, 0, tzinfo=timezone.utc), 2026),
|
|
93
|
+
# Several days in the future can only be last year's date.
|
|
94
|
+
("<14>Nov 25 10:00:00 h1 app: x", datetime(2026, 11, 20, tzinfo=timezone.utc), 2025),
|
|
95
|
+
# Six-plus months back is still the past, not next year.
|
|
96
|
+
("<14>Mar 3 10:00:00 h1 app: x", datetime(2026, 11, 20, tzinfo=timezone.utc), 2026),
|
|
97
|
+
],
|
|
98
|
+
)
|
|
99
|
+
def test_yearless_timestamp_is_dated_in_the_past(line, now, year):
|
|
100
|
+
assert autodetect_and_parse(line, now=now).ts.year == year
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def test_yearless_feb29_takes_the_last_leap_day_in_the_past():
|
|
104
|
+
from syslogcef.parsers import _infer_timestamp
|
|
105
|
+
|
|
106
|
+
# 2028-02-29 is nearer to 2027-03-01 than 2024-02-29, but it has not
|
|
107
|
+
# happened yet.
|
|
108
|
+
ts = _infer_timestamp(2, 29, "12:00:00", now=datetime(2027, 3, 1, tzinfo=timezone.utc))
|
|
109
|
+
assert ts.year == 2024
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def test_adaptive_mon_day_is_dated_in_the_past():
|
|
113
|
+
import re
|
|
114
|
+
|
|
115
|
+
from syslogcef.adaptive import TIMESTAMP_LIB
|
|
116
|
+
|
|
117
|
+
src, conv = next((s, c) for n, s, c in TIMESTAMP_LIB if n == "mon_day")
|
|
118
|
+
ts = conv(re.compile(src).match("Jan 1 12:34:56"), datetime(2026, 11, 20, tzinfo=timezone.utc))
|
|
119
|
+
assert ts.year == 2026
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|