syslog2cef 0.3.1__tar.gz → 0.3.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {syslog2cef-0.3.1/syslog2cef.egg-info → syslog2cef-0.3.3}/PKG-INFO +10 -4
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/README.md +9 -3
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/pyproject.toml +1 -1
- {syslog2cef-0.3.1 → syslog2cef-0.3.3/syslog2cef.egg-info}/PKG-INFO +10 -4
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/adaptive.py +69 -19
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/api.py +122 -11
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/cef.py +22 -2
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/mappings/linux.json +1 -1
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/normalizer.py +46 -5
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/parsers.py +47 -20
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/utils.py +10 -1
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_adaptive.py +129 -0
- syslog2cef-0.3.3/tests/test_cef.py +186 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_macos_logs.py +50 -10
- syslog2cef-0.3.3/tests/test_normalizer.py +123 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_pipeline.py +19 -2
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_real_world.py +19 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_utils.py +36 -0
- syslog2cef-0.3.1/tests/test_cef.py +0 -64
- syslog2cef-0.3.1/tests/test_normalizer.py +0 -49
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/LICENSE +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/setup.cfg +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslog2cef.egg-info/SOURCES.txt +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslog2cef.egg-info/dependency_links.txt +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslog2cef.egg-info/entry_points.txt +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslog2cef.egg-info/requires.txt +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslog2cef.egg-info/top_level.txt +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/__init__.py +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/__main__.py +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/cli.py +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/custom.py +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/dictionary.json +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/dictionary.py +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/mappings/__init__.py +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/mappings/cisco_asa.json +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/mappings/cisco_ios.json +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/mappings/f5.json +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/mappings/fortinet.json +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/mappings/sophos.json +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/mappings/vmware.json +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/net.py +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/validation.py +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_cli.py +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_cli_validation.py +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_custom_parsers.py +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_dictionary.py +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_fuzz_grammar.py +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_fuzz_properties.py +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_net.py +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_parsers.py +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_parsers_extra.py +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_rfc5424_epoch.py +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_security_hardening.py +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_tail_rotation.py +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_validation.py +0 -0
- {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_zipapp_safe.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: syslog2cef
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.3
|
|
4
4
|
Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
|
|
5
5
|
Author-email: Tamir Suliman <allamiro@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/allamiro/syslogcef
|
|
@@ -198,8 +198,11 @@ normalized event's fields:
|
|
|
198
198
|
the message and any RFC5424 structured-data or journald fields.
|
|
199
199
|
|
|
200
200
|
Field templates that reference missing keys resolve to an empty string
|
|
201
|
-
rather than failing the event.
|
|
202
|
-
|
|
201
|
+
rather than failing the event. Mapping files and Python mapping dictionaries
|
|
202
|
+
are validated before rendering; malformed templates, severity maps, and CEF
|
|
203
|
+
extension keys fail with a configuration error instead of corrupting a record.
|
|
204
|
+
See [docs/cef_fields.md](docs/cef_fields.md) for the full CEF extension
|
|
205
|
+
dictionary.
|
|
203
206
|
|
|
204
207
|
## Field Dictionary
|
|
205
208
|
|
|
@@ -217,7 +220,10 @@ knowledge, derived from the ArcSight Extension Dictionary (see
|
|
|
217
220
|
are applied during normalization for **every** event — including
|
|
218
221
|
key=value pairs extracted from adaptively-parsed unknown formats — so
|
|
219
222
|
mappings and validation always see canonical names. Original keys are
|
|
220
|
-
preserved and an explicit canonical key is never overwritten.
|
|
223
|
+
preserved and an explicit canonical key is never overwritten. Source key
|
|
224
|
+
casing is normalized as well (`SRCIP`, `SrcIp`, and `srcip` all make `src`
|
|
225
|
+
available), while parsed syslog envelope fields such as host and severity
|
|
226
|
+
remain authoritative over same-named text in the message.
|
|
221
227
|
|
|
222
228
|
This means a Fortinet-style `srcip=10.1.1.1 dstport=443` and an unknown
|
|
223
229
|
device emitting the same pairs behind an unrecognized prefix both end up
|
|
@@ -175,8 +175,11 @@ normalized event's fields:
|
|
|
175
175
|
the message and any RFC5424 structured-data or journald fields.
|
|
176
176
|
|
|
177
177
|
Field templates that reference missing keys resolve to an empty string
|
|
178
|
-
rather than failing the event.
|
|
179
|
-
|
|
178
|
+
rather than failing the event. Mapping files and Python mapping dictionaries
|
|
179
|
+
are validated before rendering; malformed templates, severity maps, and CEF
|
|
180
|
+
extension keys fail with a configuration error instead of corrupting a record.
|
|
181
|
+
See [docs/cef_fields.md](docs/cef_fields.md) for the full CEF extension
|
|
182
|
+
dictionary.
|
|
180
183
|
|
|
181
184
|
## Field Dictionary
|
|
182
185
|
|
|
@@ -194,7 +197,10 @@ knowledge, derived from the ArcSight Extension Dictionary (see
|
|
|
194
197
|
are applied during normalization for **every** event — including
|
|
195
198
|
key=value pairs extracted from adaptively-parsed unknown formats — so
|
|
196
199
|
mappings and validation always see canonical names. Original keys are
|
|
197
|
-
preserved and an explicit canonical key is never overwritten.
|
|
200
|
+
preserved and an explicit canonical key is never overwritten. Source key
|
|
201
|
+
casing is normalized as well (`SRCIP`, `SrcIp`, and `srcip` all make `src`
|
|
202
|
+
available), while parsed syslog envelope fields such as host and severity
|
|
203
|
+
remain authoritative over same-named text in the message.
|
|
198
204
|
|
|
199
205
|
This means a Fortinet-style `srcip=10.1.1.1 dstport=443` and an unknown
|
|
200
206
|
device emitting the same pairs behind an unrecognized prefix both end up
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "syslog2cef"
|
|
7
|
-
version = "0.3.
|
|
7
|
+
version = "0.3.3"
|
|
8
8
|
description = "Convert RFC3164/RFC5424 syslog lines into ArcSight CEF"
|
|
9
9
|
authors = [{name = "Tamir Suliman", email = "allamiro@gmail.com"}]
|
|
10
10
|
readme = "README.md"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: syslog2cef
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.3
|
|
4
4
|
Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
|
|
5
5
|
Author-email: Tamir Suliman <allamiro@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/allamiro/syslogcef
|
|
@@ -198,8 +198,11 @@ normalized event's fields:
|
|
|
198
198
|
the message and any RFC5424 structured-data or journald fields.
|
|
199
199
|
|
|
200
200
|
Field templates that reference missing keys resolve to an empty string
|
|
201
|
-
rather than failing the event.
|
|
202
|
-
|
|
201
|
+
rather than failing the event. Mapping files and Python mapping dictionaries
|
|
202
|
+
are validated before rendering; malformed templates, severity maps, and CEF
|
|
203
|
+
extension keys fail with a configuration error instead of corrupting a record.
|
|
204
|
+
See [docs/cef_fields.md](docs/cef_fields.md) for the full CEF extension
|
|
205
|
+
dictionary.
|
|
203
206
|
|
|
204
207
|
## Field Dictionary
|
|
205
208
|
|
|
@@ -217,7 +220,10 @@ knowledge, derived from the ArcSight Extension Dictionary (see
|
|
|
217
220
|
are applied during normalization for **every** event — including
|
|
218
221
|
key=value pairs extracted from adaptively-parsed unknown formats — so
|
|
219
222
|
mappings and validation always see canonical names. Original keys are
|
|
220
|
-
preserved and an explicit canonical key is never overwritten.
|
|
223
|
+
preserved and an explicit canonical key is never overwritten. Source key
|
|
224
|
+
casing is normalized as well (`SRCIP`, `SrcIp`, and `srcip` all make `src`
|
|
225
|
+
available), while parsed syslog envelope fields such as host and severity
|
|
226
|
+
remain authoritative over same-named text in the message.
|
|
221
227
|
|
|
222
228
|
This means a Fortinet-style `srcip=10.1.1.1 dstport=443` and an unknown
|
|
223
229
|
device emitting the same pairs behind an unrecognized prefix both end up
|
|
@@ -15,7 +15,7 @@ import re
|
|
|
15
15
|
from datetime import datetime, timezone
|
|
16
16
|
from typing import Callable, Dict, Optional, Pattern, Tuple
|
|
17
17
|
|
|
18
|
-
from .utils import convert_pri, month_abbr_to_int, parse_iso8601, sanitize_message
|
|
18
|
+
from .utils import convert_pri, is_ip, month_abbr_to_int, parse_iso8601, sanitize_message
|
|
19
19
|
|
|
20
20
|
PRI_RE = re.compile(r"^<(\d{1,3})>")
|
|
21
21
|
|
|
@@ -69,7 +69,20 @@ TIMESTAMP_LIB: Tuple[Tuple[str, str, Callable], ...] = (
|
|
|
69
69
|
)
|
|
70
70
|
|
|
71
71
|
HOST_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
|
|
72
|
-
_SKIP_HOST_TOKENS = {
|
|
72
|
+
_SKIP_HOST_TOKENS = {
|
|
73
|
+
"alert", "crit", "critical", "debug", "emerg", "emergency", "err",
|
|
74
|
+
"error", "info", "information", "informational", "kernel", "notice",
|
|
75
|
+
"panic", "warn", "warning",
|
|
76
|
+
# Common named timezone remnants. The timestamp shapes deliberately do
|
|
77
|
+
# not pretend to resolve named zones, but these must not become hosts.
|
|
78
|
+
# Abbreviations that double as ordinary English words (WEST, CAT, EAT,
|
|
79
|
+
# ART) are deliberately excluded: rejecting them would drop genuine
|
|
80
|
+
# short hostnames, which is the worse failure of the two.
|
|
81
|
+
"utc", "gmt", "cet", "cest", "eet", "eest", "est", "edt", "cst",
|
|
82
|
+
"cdt", "mst", "mdt", "pst", "pdt", "bst", "ist", "idt", "jst",
|
|
83
|
+
"kst", "aest", "aedt", "acst", "acdt", "awst", "nzst", "nzdt",
|
|
84
|
+
"msk", "hst", "akst", "akdt", "sast", "brt", "clt",
|
|
85
|
+
}
|
|
73
86
|
|
|
74
87
|
# Syslog-style program tag following the host: "app:", "app[pid]:".
|
|
75
88
|
TAG_RE = re.compile(r"^(?P<app>[\w\-/\.]+)(?:\[(?P<pid>[^\]]+)\])?:$")
|
|
@@ -78,13 +91,38 @@ _TAG_SRC = r"(?:(?P<adaptive_app>[\w\-/\.]+)(?:\[(?P<adaptive_pid>[^\]]+)\])?:\s
|
|
|
78
91
|
|
|
79
92
|
|
|
80
93
|
def _valid_host(candidate: Optional[str]) -> bool:
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
)
|
|
94
|
+
if not candidate or "=" in candidate or candidate.lower() in _SKIP_HOST_TOKENS:
|
|
95
|
+
return False
|
|
96
|
+
# Adaptive layouts sometimes use a literal IP in the hostname slot.
|
|
97
|
+
# Check it before rejecting all-numeric tokens.
|
|
98
|
+
if is_ip(candidate):
|
|
99
|
+
return True
|
|
100
|
+
return bool(HOST_RE.fullmatch(candidate) and not candidate.isdigit())
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _normalize_host_token(token: str) -> str:
|
|
104
|
+
"""Strip trailing layout delimiters from a host token.
|
|
105
|
+
|
|
106
|
+
A token that is already a valid address is returned untouched: a
|
|
107
|
+
compressed IPv6 literal may legitimately end in "::" ("2001:db8::",
|
|
108
|
+
"fe80::"), and stripping that would silently corrupt the host. Anything
|
|
109
|
+
else has its whole trailing ":," run removed, so multi-character layout
|
|
110
|
+
delimiters ("host::", "host:,") work as the analysis path always did.
|
|
111
|
+
"""
|
|
112
|
+
if is_ip(token):
|
|
113
|
+
return token
|
|
114
|
+
# A layout delimiter may follow a compressed literal that itself ends in
|
|
115
|
+
# "::" ("2001:db8::,"), where neither the whole token nor a full strip
|
|
116
|
+
# yields the address. Remove one trailing separator at a time and keep
|
|
117
|
+
# the first candidate that parses, falling back to the fully stripped
|
|
118
|
+
# token for ordinary hostnames ("host::", "host:,").
|
|
119
|
+
candidate = token
|
|
120
|
+
while candidate and candidate[-1] in ":,":
|
|
121
|
+
candidate = candidate[:-1]
|
|
122
|
+
if is_ip(candidate):
|
|
123
|
+
return candidate
|
|
124
|
+
return candidate
|
|
125
|
+
|
|
88
126
|
|
|
89
127
|
_CACHE: Dict[str, Tuple[Pattern[str], str, Callable]] = {}
|
|
90
128
|
_CACHE_MAX = 256
|
|
@@ -114,7 +152,7 @@ def cache_size() -> int:
|
|
|
114
152
|
return len(_CACHE)
|
|
115
153
|
|
|
116
154
|
|
|
117
|
-
def _build_event(line, pri, ts, host, msg, app=None, pid=None):
|
|
155
|
+
def _build_event(line, pri, ts, host, msg, app=None, pid=None, ts_orig=""):
|
|
118
156
|
from .parsers import ParsedEvent
|
|
119
157
|
|
|
120
158
|
facility, severity = convert_pri(pri)
|
|
@@ -123,7 +161,7 @@ def _build_event(line, pri, ts, host, msg, app=None, pid=None):
|
|
|
123
161
|
facility=facility,
|
|
124
162
|
severity=severity,
|
|
125
163
|
ts=ts,
|
|
126
|
-
ts_orig=
|
|
164
|
+
ts_orig=ts_orig,
|
|
127
165
|
host=host,
|
|
128
166
|
app=app,
|
|
129
167
|
pid=pid,
|
|
@@ -156,18 +194,22 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
|
|
|
156
194
|
except (ValueError, KeyError):
|
|
157
195
|
ts = None
|
|
158
196
|
gd = match.groupdict()
|
|
159
|
-
|
|
197
|
+
ts_orig = match.group("adaptive_ts")
|
|
198
|
+
host_token = gd.get("adaptive_host_token")
|
|
199
|
+
host = _normalize_host_token(host_token) if host_token else None
|
|
160
200
|
app = gd.get("adaptive_app")
|
|
161
201
|
pid = gd.get("adaptive_pid")
|
|
162
202
|
msg = gd.get("adaptive_msg")
|
|
163
203
|
msg = line if msg is None else msg
|
|
164
204
|
# The shape signature reduces all words to the same class, so a
|
|
165
205
|
# cached host slot may capture a non-host token; revalidate.
|
|
166
|
-
if
|
|
206
|
+
if host_token is not None and not _valid_host(host):
|
|
167
207
|
tag = f"{app}[{pid}]: " if app and pid else (f"{app}: " if app else "")
|
|
168
|
-
msg = f"{
|
|
208
|
+
msg = f"{host_token} {tag}{msg}".strip()
|
|
169
209
|
host = app = pid = None
|
|
170
|
-
return _build_event(
|
|
210
|
+
return _build_event(
|
|
211
|
+
raw, pri, ts, host, msg, app=app, pid=pid, ts_orig=ts_orig
|
|
212
|
+
)
|
|
171
213
|
|
|
172
214
|
# Analysis pass: find the earliest timestamp in the line head.
|
|
173
215
|
best = None
|
|
@@ -194,7 +236,7 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
|
|
|
194
236
|
msg = rest
|
|
195
237
|
tokens = rest.split(None, 1)
|
|
196
238
|
if tokens:
|
|
197
|
-
candidate = tokens[0]
|
|
239
|
+
candidate = _normalize_host_token(tokens[0])
|
|
198
240
|
if _valid_host(candidate):
|
|
199
241
|
host = candidate
|
|
200
242
|
msg = tokens[1] if len(tokens) > 1 else ""
|
|
@@ -213,19 +255,27 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
|
|
|
213
255
|
prefix = line[: ts_match.start()]
|
|
214
256
|
parts = [re.escape(prefix), f"(?P<adaptive_ts>{src})", r"[\s:,]*"]
|
|
215
257
|
if host is not None:
|
|
216
|
-
|
|
258
|
+
# Capture the complete token. Normalizing punctuation after the
|
|
259
|
+
# match keeps first-sight and cached results identical and avoids
|
|
260
|
+
# treating the final colon of an IPv6 address as a separator.
|
|
261
|
+
parts.append(r"(?P<adaptive_host_token>\S+)(?:\s+|$)")
|
|
217
262
|
parts.append(_TAG_SRC)
|
|
218
263
|
parts.append(r"(?P<adaptive_msg>.*)$")
|
|
219
264
|
try:
|
|
220
265
|
pattern = re.compile("".join(parts))
|
|
221
266
|
except re.error:
|
|
222
267
|
pattern = None
|
|
223
|
-
|
|
268
|
+
# A hostless pattern is not stable enough to reuse: severity/timezone
|
|
269
|
+
# words and plain alphabetic hostnames collapse to the same signature.
|
|
270
|
+
# Caching it would suppress a valid host on a later line of that shape.
|
|
271
|
+
if host is not None and pattern is not None and pattern.match(line):
|
|
224
272
|
if len(_CACHE) >= _CACHE_MAX:
|
|
225
273
|
_CACHE.pop(next(iter(_CACHE)))
|
|
226
274
|
_CACHE[sig] = (pattern, name, conv)
|
|
227
275
|
|
|
228
|
-
return _build_event(
|
|
276
|
+
return _build_event(
|
|
277
|
+
raw, pri, ts, host, msg, app=app, pid=pid, ts_orig=ts_match.group(0)
|
|
278
|
+
)
|
|
229
279
|
|
|
230
280
|
|
|
231
281
|
def pattern_ts_match(pattern: Pattern[str], match: "re.Match[str]"):
|
|
@@ -2,6 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
4
|
import logging
|
|
5
|
+
import re
|
|
5
6
|
from dataclasses import dataclass
|
|
6
7
|
from datetime import datetime
|
|
7
8
|
from pathlib import Path
|
|
@@ -11,10 +12,72 @@ from .parsers import ParsedEvent, ParserError, autodetect_and_parse
|
|
|
11
12
|
from .normalizer import NormalizedEvent, normalize
|
|
12
13
|
from .utils import sanitize_message
|
|
13
14
|
from .mappings import CISCO_ASA, CISCO_IOS, F5, FORTINET, LINUX, SOPHOS, VMWARE
|
|
14
|
-
from .cef import CEFEvent, build_cef
|
|
15
|
+
from .cef import HEADER_KEYS, CEFEvent, build_cef
|
|
15
16
|
|
|
16
17
|
logger = logging.getLogger(__name__)
|
|
17
18
|
|
|
19
|
+
_EXTENSION_KEY_RE = re.compile(r"^[A-Za-z0-9_.]+$")
|
|
20
|
+
|
|
21
|
+
# What may follow a mapping key in ``template % fields``: flags, width,
|
|
22
|
+
# precision, length modifier, conversion. A width/precision of "*" is
|
|
23
|
+
# deliberately excluded because it consumes a positional argument a mapping
|
|
24
|
+
# cannot supply. A bare "." precision is legal Python (it means zero).
|
|
25
|
+
_FORMAT_TAIL_RE = re.compile(r"[#0\- +]*\d*(?:\.\d*)?[hlL]?[diouxXeEfFgGcrsa]")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _scan_format_key(template: str, start: int) -> int:
|
|
29
|
+
"""Return the index just past a mapping key's closing paren, or -1.
|
|
30
|
+
|
|
31
|
+
Mirrors CPython's own scan, which counts nesting rather than stopping at
|
|
32
|
+
the first ")": "%((a))s" is a valid reference to the key "(a)", while
|
|
33
|
+
"%(a(b)s" is an incomplete key that raises at render time.
|
|
34
|
+
"""
|
|
35
|
+
depth = 1
|
|
36
|
+
for index in range(start, len(template)):
|
|
37
|
+
if template[index] == "(":
|
|
38
|
+
depth += 1
|
|
39
|
+
elif template[index] == ")":
|
|
40
|
+
depth -= 1
|
|
41
|
+
if depth == 0:
|
|
42
|
+
return index + 1
|
|
43
|
+
return -1
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _validate_template(template: str, where: str, what: str) -> None:
|
|
47
|
+
"""Reject a template that cannot render, before any record is produced.
|
|
48
|
+
|
|
49
|
+
``MappingResolver._format`` swallows a rendering failure and returns "",
|
|
50
|
+
which silently falls a header back to its default or drops an extension.
|
|
51
|
+
A malformed template is a configuration error, so surface it eagerly.
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
def fail(problem: str) -> None:
|
|
55
|
+
raise ValueError(f"{where}: {what} template {template!r} {problem}")
|
|
56
|
+
|
|
57
|
+
position = 0
|
|
58
|
+
while True:
|
|
59
|
+
start = template.find("%", position)
|
|
60
|
+
if start == -1:
|
|
61
|
+
return
|
|
62
|
+
cursor = start + 1
|
|
63
|
+
if cursor < len(template) and template[cursor] == "%":
|
|
64
|
+
position = cursor + 1
|
|
65
|
+
continue
|
|
66
|
+
if cursor >= len(template) or template[cursor] != "(":
|
|
67
|
+
fail(
|
|
68
|
+
f"has an invalid format specifier at index {start} "
|
|
69
|
+
"(use '%%' for a literal percent)"
|
|
70
|
+
)
|
|
71
|
+
key_end = _scan_format_key(template, cursor + 1)
|
|
72
|
+
if key_end == -1:
|
|
73
|
+
fail(f"has an incomplete format key at index {start}")
|
|
74
|
+
if key_end - 1 == cursor + 1:
|
|
75
|
+
fail(f"has an empty format key at index {start}")
|
|
76
|
+
tail = _FORMAT_TAIL_RE.match(template, key_end)
|
|
77
|
+
if tail is None:
|
|
78
|
+
fail(f"has an invalid conversion at index {key_end}")
|
|
79
|
+
position = tail.end()
|
|
80
|
+
|
|
18
81
|
|
|
19
82
|
@dataclass
|
|
20
83
|
class ParseResult:
|
|
@@ -62,23 +125,69 @@ def normalize_event(event: ParsedEvent | NormalizedEvent) -> NormalizedEvent:
|
|
|
62
125
|
return normalize(event)
|
|
63
126
|
|
|
64
127
|
|
|
128
|
+
class _ValidatedMapping(dict):
|
|
129
|
+
"""A mapping that has already passed structural validation.
|
|
130
|
+
|
|
131
|
+
``to_cef`` loads its mapping per event, so a long stream would otherwise
|
|
132
|
+
re-run template and severity-map validation for every record. Callers
|
|
133
|
+
that convert many events (``StreamConverter``, the CLI) validate once and
|
|
134
|
+
pass the result back in; this marker lets the second load short-circuit.
|
|
135
|
+
"""
|
|
136
|
+
|
|
137
|
+
|
|
65
138
|
def _load_mapping(mapping: Mapping[str, Any] | Path | str | None) -> Mapping[str, Any]:
|
|
66
139
|
if mapping is None:
|
|
67
140
|
return {}
|
|
68
|
-
if isinstance(mapping,
|
|
141
|
+
if isinstance(mapping, _ValidatedMapping):
|
|
69
142
|
return mapping
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
143
|
+
if isinstance(mapping, Mapping):
|
|
144
|
+
data = mapping
|
|
145
|
+
where = "mapping"
|
|
146
|
+
else:
|
|
147
|
+
path = Path(mapping)
|
|
148
|
+
with path.open("r", encoding="utf-8") as fp:
|
|
149
|
+
data = json.load(fp)
|
|
150
|
+
where = str(path)
|
|
73
151
|
# Structural validation with clear messages: a top-level array would
|
|
74
152
|
# otherwise TypeError deep inside the renderer, and a wrong-shaped
|
|
75
153
|
# extensions/severity_map would fail per event instead of up front.
|
|
76
|
-
if not isinstance(data,
|
|
77
|
-
raise ValueError(f"{
|
|
154
|
+
if not isinstance(data, Mapping):
|
|
155
|
+
raise ValueError(f"{where}: mapping must be a JSON object")
|
|
78
156
|
for key in ("extensions", "severity_map"):
|
|
79
|
-
if key in data and not isinstance(data[key],
|
|
80
|
-
raise ValueError(f"{
|
|
81
|
-
|
|
157
|
+
if key in data and not isinstance(data[key], Mapping):
|
|
158
|
+
raise ValueError(f"{where}: '{key}' must be an object")
|
|
159
|
+
for key in HEADER_KEYS:
|
|
160
|
+
if key in data:
|
|
161
|
+
if not isinstance(data[key], str):
|
|
162
|
+
raise ValueError(f"{where}: '{key}' must be a string")
|
|
163
|
+
_validate_template(data[key], where, f"header {key!r}")
|
|
164
|
+
for key, template in data.get("extensions", {}).items():
|
|
165
|
+
if not isinstance(key, str) or not _EXTENSION_KEY_RE.fullmatch(key):
|
|
166
|
+
raise ValueError(f"{where}: invalid extension key {key!r}")
|
|
167
|
+
if not isinstance(template, str):
|
|
168
|
+
raise ValueError(
|
|
169
|
+
f"{where}: extension template for {key!r} must be a string"
|
|
170
|
+
)
|
|
171
|
+
_validate_template(template, where, f"extension {key!r}")
|
|
172
|
+
for source, target in data.get("severity_map", {}).items():
|
|
173
|
+
source_text, target_text = str(source), str(target)
|
|
174
|
+
if not (
|
|
175
|
+
source_text.isascii() and source_text.isdigit()
|
|
176
|
+
and len(source_text) == 1
|
|
177
|
+
and 0 <= int(source_text) <= 7
|
|
178
|
+
):
|
|
179
|
+
raise ValueError(
|
|
180
|
+
f"{where}: severity_map key {source!r} must be an integer from 0 to 7"
|
|
181
|
+
)
|
|
182
|
+
if not (
|
|
183
|
+
target_text.isascii() and target_text.isdigit()
|
|
184
|
+
and len(target_text) <= 2
|
|
185
|
+
and 0 <= int(target_text) <= 10
|
|
186
|
+
):
|
|
187
|
+
raise ValueError(
|
|
188
|
+
f"{where}: severity_map value {target!r} must be an integer from 0 to 10"
|
|
189
|
+
)
|
|
190
|
+
return _ValidatedMapping(data)
|
|
82
191
|
|
|
83
192
|
|
|
84
193
|
def to_cef(
|
|
@@ -149,7 +258,9 @@ class StreamConverter:
|
|
|
149
258
|
strict: bool = False,
|
|
150
259
|
) -> None:
|
|
151
260
|
self.mode = mode
|
|
152
|
-
|
|
261
|
+
# Validate the mapping once here rather than per converted line, and
|
|
262
|
+
# surface a malformed mapping before any record is emitted.
|
|
263
|
+
self.mapping = None if mapping is None else _load_mapping(mapping)
|
|
153
264
|
self.validate = validate
|
|
154
265
|
self.strict = strict
|
|
155
266
|
self._contexts: dict[str, ParsedEvent] = {}
|
|
@@ -9,6 +9,12 @@ from .utils import cef_escape, cef_escape_header
|
|
|
9
9
|
|
|
10
10
|
logger = logging.getLogger(__name__)
|
|
11
11
|
|
|
12
|
+
# The CEF:0 header slots a mapping may supply a template for, in wire order.
|
|
13
|
+
# Single source of truth: mapping validation in api.py checks exactly these.
|
|
14
|
+
HEADER_KEYS = (
|
|
15
|
+
"deviceVendor", "deviceProduct", "deviceVersion", "eventClassId", "name",
|
|
16
|
+
)
|
|
17
|
+
|
|
12
18
|
|
|
13
19
|
DEFAULT_MAPPING = {
|
|
14
20
|
"deviceVendor": "Generic",
|
|
@@ -28,6 +34,14 @@ DEFAULT_MAPPING = {
|
|
|
28
34
|
},
|
|
29
35
|
"extensions": {
|
|
30
36
|
"msg": "%(msg)s",
|
|
37
|
+
# The syslog HOSTNAME identifies the device that GENERATED the
|
|
38
|
+
# event, so it is the source host (shost) — never the destination
|
|
39
|
+
# (dhost), which is reserved for network-specific logs whose
|
|
40
|
+
# payload names an actual destination. shost (max 1023) rather
|
|
41
|
+
# than dvchost (max 100): RFC 5424 hostnames may reach 255 chars,
|
|
42
|
+
# which would fail --strict against dvchost's dictionary limit.
|
|
43
|
+
# Empty values are dropped, so this is a no-op for a hostless line.
|
|
44
|
+
"shost": "%(host)s",
|
|
31
45
|
# Event time (epoch milliseconds) parsed from the line, or the
|
|
32
46
|
# processing time when the line carried no usable timestamp.
|
|
33
47
|
"rt": "%(rt)s",
|
|
@@ -60,7 +74,7 @@ class MappingResolver:
|
|
|
60
74
|
|
|
61
75
|
def resolve_header(self, fields: Mapping[str, Any]) -> Dict[str, str]:
|
|
62
76
|
header = {}
|
|
63
|
-
for key in
|
|
77
|
+
for key in HEADER_KEYS:
|
|
64
78
|
template = self.mapping.get(key, DEFAULT_MAPPING[key])
|
|
65
79
|
value = self._format(template, fields)
|
|
66
80
|
if not value:
|
|
@@ -73,7 +87,13 @@ class MappingResolver:
|
|
|
73
87
|
return header
|
|
74
88
|
|
|
75
89
|
def _resolve_severity(self, fields: Mapping[str, Any]) -> str:
|
|
76
|
-
severity_map =
|
|
90
|
+
severity_map = DEFAULT_MAPPING["severity_map"].copy()
|
|
91
|
+
# Programmatic mappings may naturally use integer keys/values while
|
|
92
|
+
# JSON mappings always contain strings. Normalize both forms so a
|
|
93
|
+
# validated {6: 9} override behaves like {"6": "9"}.
|
|
94
|
+
severity_map.update(
|
|
95
|
+
{str(key): str(value) for key, value in self.mapping.get("severity_map", {}).items()}
|
|
96
|
+
)
|
|
77
97
|
value = fields.get("severity")
|
|
78
98
|
if value is None and "pri" in fields:
|
|
79
99
|
# "pri" may come from an untrusted kv pair (pri=zzz), not just
|
|
@@ -38,10 +38,29 @@ class NormalizedEvent(ParsedEvent):
|
|
|
38
38
|
"source_hint": self.source_hint,
|
|
39
39
|
"msg": self.msg,
|
|
40
40
|
}
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
41
|
+
fields: Dict[str, Any] = {}
|
|
42
|
+
fields.update(self.sd)
|
|
43
|
+
fields.update(self.kv)
|
|
44
|
+
fields.update(self.extras)
|
|
45
|
+
|
|
46
|
+
# Parsed envelope metadata is more trustworthy than same-named
|
|
47
|
+
# key=value text inside the message. Reapply it last when present,
|
|
48
|
+
# while still allowing a kv field to fill metadata absent from the
|
|
49
|
+
# envelope. ``msg`` intentionally retains the established kv
|
|
50
|
+
# behavior for formats that carry an inner msg="..." value.
|
|
51
|
+
authoritative = {
|
|
52
|
+
"pri", "facility", "severity", "timestamp", "ts", "ts_orig",
|
|
53
|
+
"host", "app", "pid", "msgid", "source_hint",
|
|
54
|
+
}
|
|
55
|
+
for key in authoritative:
|
|
56
|
+
# An absent original timestamp is represented as "" rather than
|
|
57
|
+
# None, so emptiness must count as absent too: otherwise a
|
|
58
|
+
# record with no recognized envelope timestamp would reapply ""
|
|
59
|
+
# over a payload ts_orig= value and hide it from mappings.
|
|
60
|
+
if base[key] is not None and base[key] != "":
|
|
61
|
+
fields[key] = base[key]
|
|
62
|
+
fields.setdefault("msg", base["msg"])
|
|
63
|
+
return {k: v for k, v in fields.items() if v is not None}
|
|
45
64
|
|
|
46
65
|
|
|
47
66
|
def normalize(event: ParsedEvent) -> NormalizedEvent:
|
|
@@ -98,7 +117,29 @@ def _derive_common_fields(event: NormalizedEvent) -> None:
|
|
|
98
117
|
# including kv pairs extracted from adaptive-parsed lines — so
|
|
99
118
|
# mappings and validation always see canonical CEF keys. Original
|
|
100
119
|
# keys are kept; an existing canonical key is never overwritten.
|
|
101
|
-
from .dictionary import field_aliases
|
|
120
|
+
from .dictionary import cef_keys, field_aliases
|
|
121
|
+
|
|
122
|
+
# Source formats frequently vary only in field-name casing (SRCIP,
|
|
123
|
+
# SrcIp, srcip). Preserve every original key, add a lowercase spelling,
|
|
124
|
+
# and prefer an explicitly lowercase value when both forms are present.
|
|
125
|
+
original_items = list(event.kv.items())
|
|
126
|
+
for key, value in original_items:
|
|
127
|
+
lower = key.lower()
|
|
128
|
+
if lower not in event.kv:
|
|
129
|
+
event.kv[lower] = value
|
|
130
|
+
|
|
131
|
+
# Restore the dictionary's exact spelling for canonical camelCase CEF
|
|
132
|
+
# keys (DeviceExternalId -> deviceExternalId), again without overwrite.
|
|
133
|
+
canonical_by_lower = {key.lower(): key for key in cef_keys()}
|
|
134
|
+
for key in list(event.kv):
|
|
135
|
+
lower = key.lower()
|
|
136
|
+
canonical = canonical_by_lower.get(lower)
|
|
137
|
+
if canonical is not None and canonical not in event.kv:
|
|
138
|
+
# Take the folded lowercase value rather than this spelling.
|
|
139
|
+
# The loop above already applied lowercase-wins precedence, so
|
|
140
|
+
# an explicit deviceexternalid= beats DEVICEEXTERNALID=
|
|
141
|
+
# regardless of which appeared first in the message.
|
|
142
|
+
event.kv[canonical] = event.kv.get(lower, event.kv[key])
|
|
102
143
|
|
|
103
144
|
for alias, canonical in field_aliases().items():
|
|
104
145
|
if canonical not in event.kv and alias in event.kv:
|