syslog2cef 0.3.1__tar.gz → 0.3.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. {syslog2cef-0.3.1/syslog2cef.egg-info → syslog2cef-0.3.3}/PKG-INFO +10 -4
  2. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/README.md +9 -3
  3. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/pyproject.toml +1 -1
  4. {syslog2cef-0.3.1 → syslog2cef-0.3.3/syslog2cef.egg-info}/PKG-INFO +10 -4
  5. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/adaptive.py +69 -19
  6. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/api.py +122 -11
  7. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/cef.py +22 -2
  8. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/mappings/linux.json +1 -1
  9. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/normalizer.py +46 -5
  10. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/parsers.py +47 -20
  11. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/utils.py +10 -1
  12. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_adaptive.py +129 -0
  13. syslog2cef-0.3.3/tests/test_cef.py +186 -0
  14. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_macos_logs.py +50 -10
  15. syslog2cef-0.3.3/tests/test_normalizer.py +123 -0
  16. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_pipeline.py +19 -2
  17. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_real_world.py +19 -0
  18. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_utils.py +36 -0
  19. syslog2cef-0.3.1/tests/test_cef.py +0 -64
  20. syslog2cef-0.3.1/tests/test_normalizer.py +0 -49
  21. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/LICENSE +0 -0
  22. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/setup.cfg +0 -0
  23. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslog2cef.egg-info/SOURCES.txt +0 -0
  24. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslog2cef.egg-info/dependency_links.txt +0 -0
  25. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslog2cef.egg-info/entry_points.txt +0 -0
  26. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslog2cef.egg-info/requires.txt +0 -0
  27. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslog2cef.egg-info/top_level.txt +0 -0
  28. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/__init__.py +0 -0
  29. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/__main__.py +0 -0
  30. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/cli.py +0 -0
  31. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/custom.py +0 -0
  32. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/dictionary.json +0 -0
  33. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/dictionary.py +0 -0
  34. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/mappings/__init__.py +0 -0
  35. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/mappings/cisco_asa.json +0 -0
  36. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/mappings/cisco_ios.json +0 -0
  37. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/mappings/f5.json +0 -0
  38. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/mappings/fortinet.json +0 -0
  39. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/mappings/sophos.json +0 -0
  40. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/mappings/vmware.json +0 -0
  41. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/net.py +0 -0
  42. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/syslogcef/validation.py +0 -0
  43. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_cli.py +0 -0
  44. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_cli_validation.py +0 -0
  45. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_custom_parsers.py +0 -0
  46. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_dictionary.py +0 -0
  47. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_fuzz_grammar.py +0 -0
  48. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_fuzz_properties.py +0 -0
  49. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_net.py +0 -0
  50. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_parsers.py +0 -0
  51. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_parsers_extra.py +0 -0
  52. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_rfc5424_epoch.py +0 -0
  53. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_security_hardening.py +0 -0
  54. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_tail_rotation.py +0 -0
  55. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_validation.py +0 -0
  56. {syslog2cef-0.3.1 → syslog2cef-0.3.3}/tests/test_zipapp_safe.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: syslog2cef
3
- Version: 0.3.1
3
+ Version: 0.3.3
4
4
  Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
5
  Author-email: Tamir Suliman <allamiro@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/allamiro/syslogcef
@@ -198,8 +198,11 @@ normalized event's fields:
198
198
  the message and any RFC5424 structured-data or journald fields.
199
199
 
200
200
  Field templates that reference missing keys resolve to an empty string
201
- rather than failing the event. See [docs/cef_fields.md](docs/cef_fields.md)
202
- for the full CEF extension dictionary.
201
+ rather than failing the event. Mapping files and Python mapping dictionaries
202
+ are validated before rendering; malformed templates, severity maps, and CEF
203
+ extension keys fail with a configuration error instead of corrupting a record.
204
+ See [docs/cef_fields.md](docs/cef_fields.md) for the full CEF extension
205
+ dictionary.
203
206
 
204
207
  ## Field Dictionary
205
208
 
@@ -217,7 +220,10 @@ knowledge, derived from the ArcSight Extension Dictionary (see
217
220
  are applied during normalization for **every** event — including
218
221
  key=value pairs extracted from adaptively-parsed unknown formats — so
219
222
  mappings and validation always see canonical names. Original keys are
220
- preserved and an explicit canonical key is never overwritten.
223
+ preserved and an explicit canonical key is never overwritten. Source key
224
+ casing is normalized as well (`SRCIP`, `SrcIp`, and `srcip` all make `src`
225
+ available), while parsed syslog envelope fields such as host and severity
226
+ remain authoritative over same-named text in the message.
221
227
 
222
228
  This means a Fortinet-style `srcip=10.1.1.1 dstport=443` and an unknown
223
229
  device emitting the same pairs behind an unrecognized prefix both end up
@@ -175,8 +175,11 @@ normalized event's fields:
175
175
  the message and any RFC5424 structured-data or journald fields.
176
176
 
177
177
  Field templates that reference missing keys resolve to an empty string
178
- rather than failing the event. See [docs/cef_fields.md](docs/cef_fields.md)
179
- for the full CEF extension dictionary.
178
+ rather than failing the event. Mapping files and Python mapping dictionaries
179
+ are validated before rendering; malformed templates, severity maps, and CEF
180
+ extension keys fail with a configuration error instead of corrupting a record.
181
+ See [docs/cef_fields.md](docs/cef_fields.md) for the full CEF extension
182
+ dictionary.
180
183
 
181
184
  ## Field Dictionary
182
185
 
@@ -194,7 +197,10 @@ knowledge, derived from the ArcSight Extension Dictionary (see
194
197
  are applied during normalization for **every** event — including
195
198
  key=value pairs extracted from adaptively-parsed unknown formats — so
196
199
  mappings and validation always see canonical names. Original keys are
197
- preserved and an explicit canonical key is never overwritten.
200
+ preserved and an explicit canonical key is never overwritten. Source key
201
+ casing is normalized as well (`SRCIP`, `SrcIp`, and `srcip` all make `src`
202
+ available), while parsed syslog envelope fields such as host and severity
203
+ remain authoritative over same-named text in the message.
198
204
 
199
205
  This means a Fortinet-style `srcip=10.1.1.1 dstport=443` and an unknown
200
206
  device emitting the same pairs behind an unrecognized prefix both end up
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "syslog2cef"
7
- version = "0.3.1"
7
+ version = "0.3.3"
8
8
  description = "Convert RFC3164/RFC5424 syslog lines into ArcSight CEF"
9
9
  authors = [{name = "Tamir Suliman", email = "allamiro@gmail.com"}]
10
10
  readme = "README.md"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: syslog2cef
3
- Version: 0.3.1
3
+ Version: 0.3.3
4
4
  Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
5
  Author-email: Tamir Suliman <allamiro@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/allamiro/syslogcef
@@ -198,8 +198,11 @@ normalized event's fields:
198
198
  the message and any RFC5424 structured-data or journald fields.
199
199
 
200
200
  Field templates that reference missing keys resolve to an empty string
201
- rather than failing the event. See [docs/cef_fields.md](docs/cef_fields.md)
202
- for the full CEF extension dictionary.
201
+ rather than failing the event. Mapping files and Python mapping dictionaries
202
+ are validated before rendering; malformed templates, severity maps, and CEF
203
+ extension keys fail with a configuration error instead of corrupting a record.
204
+ See [docs/cef_fields.md](docs/cef_fields.md) for the full CEF extension
205
+ dictionary.
203
206
 
204
207
  ## Field Dictionary
205
208
 
@@ -217,7 +220,10 @@ knowledge, derived from the ArcSight Extension Dictionary (see
217
220
  are applied during normalization for **every** event — including
218
221
  key=value pairs extracted from adaptively-parsed unknown formats — so
219
222
  mappings and validation always see canonical names. Original keys are
220
- preserved and an explicit canonical key is never overwritten.
223
+ preserved and an explicit canonical key is never overwritten. Source key
224
+ casing is normalized as well (`SRCIP`, `SrcIp`, and `srcip` all make `src`
225
+ available), while parsed syslog envelope fields such as host and severity
226
+ remain authoritative over same-named text in the message.
221
227
 
222
228
  This means a Fortinet-style `srcip=10.1.1.1 dstport=443` and an unknown
223
229
  device emitting the same pairs behind an unrecognized prefix both end up
@@ -15,7 +15,7 @@ import re
15
15
  from datetime import datetime, timezone
16
16
  from typing import Callable, Dict, Optional, Pattern, Tuple
17
17
 
18
- from .utils import convert_pri, month_abbr_to_int, parse_iso8601, sanitize_message
18
+ from .utils import convert_pri, is_ip, month_abbr_to_int, parse_iso8601, sanitize_message
19
19
 
20
20
  PRI_RE = re.compile(r"^<(\d{1,3})>")
21
21
 
@@ -69,7 +69,20 @@ TIMESTAMP_LIB: Tuple[Tuple[str, str, Callable], ...] = (
69
69
  )
70
70
 
71
71
  HOST_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
72
- _SKIP_HOST_TOKENS = {"error", "warning", "info", "debug", "notice", "kernel"}
72
+ _SKIP_HOST_TOKENS = {
73
+ "alert", "crit", "critical", "debug", "emerg", "emergency", "err",
74
+ "error", "info", "information", "informational", "kernel", "notice",
75
+ "panic", "warn", "warning",
76
+ # Common named timezone remnants. The timestamp shapes deliberately do
77
+ # not pretend to resolve named zones, but these must not become hosts.
78
+ # Abbreviations that double as ordinary English words (WEST, CAT, EAT,
79
+ # ART) are deliberately excluded: rejecting them would drop genuine
80
+ # short hostnames, which is the worse failure of the two.
81
+ "utc", "gmt", "cet", "cest", "eet", "eest", "est", "edt", "cst",
82
+ "cdt", "mst", "mdt", "pst", "pdt", "bst", "ist", "idt", "jst",
83
+ "kst", "aest", "aedt", "acst", "acdt", "awst", "nzst", "nzdt",
84
+ "msk", "hst", "akst", "akdt", "sast", "brt", "clt",
85
+ }
73
86
 
74
87
  # Syslog-style program tag following the host: "app:", "app[pid]:".
75
88
  TAG_RE = re.compile(r"^(?P<app>[\w\-/\.]+)(?:\[(?P<pid>[^\]]+)\])?:$")
@@ -78,13 +91,38 @@ _TAG_SRC = r"(?:(?P<adaptive_app>[\w\-/\.]+)(?:\[(?P<adaptive_pid>[^\]]+)\])?:\s
78
91
 
79
92
 
80
93
  def _valid_host(candidate: Optional[str]) -> bool:
81
- return bool(
82
- candidate
83
- and "=" not in candidate
84
- and HOST_RE.match(candidate)
85
- and candidate.lower() not in _SKIP_HOST_TOKENS
86
- and not candidate.isdigit()
87
- )
94
+ if not candidate or "=" in candidate or candidate.lower() in _SKIP_HOST_TOKENS:
95
+ return False
96
+ # Adaptive layouts sometimes use a literal IP in the hostname slot.
97
+ # Check it before rejecting all-numeric tokens.
98
+ if is_ip(candidate):
99
+ return True
100
+ return bool(HOST_RE.fullmatch(candidate) and not candidate.isdigit())
101
+
102
+
103
+ def _normalize_host_token(token: str) -> str:
104
+ """Strip trailing layout delimiters from a host token.
105
+
106
+ A token that is already a valid address is returned untouched: a
107
+ compressed IPv6 literal may legitimately end in "::" ("2001:db8::",
108
+ "fe80::"), and stripping that would silently corrupt the host. Anything
109
+ else has its whole trailing ":," run removed, so multi-character layout
110
+ delimiters ("host::", "host:,") work as the analysis path always did.
111
+ """
112
+ if is_ip(token):
113
+ return token
114
+ # A layout delimiter may follow a compressed literal that itself ends in
115
+ # "::" ("2001:db8::,"), where neither the whole token nor a full strip
116
+ # yields the address. Remove one trailing separator at a time and keep
117
+ # the first candidate that parses, falling back to the fully stripped
118
+ # token for ordinary hostnames ("host::", "host:,").
119
+ candidate = token
120
+ while candidate and candidate[-1] in ":,":
121
+ candidate = candidate[:-1]
122
+ if is_ip(candidate):
123
+ return candidate
124
+ return candidate
125
+
88
126
 
89
127
  _CACHE: Dict[str, Tuple[Pattern[str], str, Callable]] = {}
90
128
  _CACHE_MAX = 256
@@ -114,7 +152,7 @@ def cache_size() -> int:
114
152
  return len(_CACHE)
115
153
 
116
154
 
117
- def _build_event(line, pri, ts, host, msg, app=None, pid=None):
155
+ def _build_event(line, pri, ts, host, msg, app=None, pid=None, ts_orig=""):
118
156
  from .parsers import ParsedEvent
119
157
 
120
158
  facility, severity = convert_pri(pri)
@@ -123,7 +161,7 @@ def _build_event(line, pri, ts, host, msg, app=None, pid=None):
123
161
  facility=facility,
124
162
  severity=severity,
125
163
  ts=ts,
126
- ts_orig="",
164
+ ts_orig=ts_orig,
127
165
  host=host,
128
166
  app=app,
129
167
  pid=pid,
@@ -156,18 +194,22 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
156
194
  except (ValueError, KeyError):
157
195
  ts = None
158
196
  gd = match.groupdict()
159
- host = gd.get("adaptive_host")
197
+ ts_orig = match.group("adaptive_ts")
198
+ host_token = gd.get("adaptive_host_token")
199
+ host = _normalize_host_token(host_token) if host_token else None
160
200
  app = gd.get("adaptive_app")
161
201
  pid = gd.get("adaptive_pid")
162
202
  msg = gd.get("adaptive_msg")
163
203
  msg = line if msg is None else msg
164
204
  # The shape signature reduces all words to the same class, so a
165
205
  # cached host slot may capture a non-host token; revalidate.
166
- if host is not None and not _valid_host(host.rstrip(":,")):
206
+ if host_token is not None and not _valid_host(host):
167
207
  tag = f"{app}[{pid}]: " if app and pid else (f"{app}: " if app else "")
168
- msg = f"{host} {tag}{msg}".strip()
208
+ msg = f"{host_token} {tag}{msg}".strip()
169
209
  host = app = pid = None
170
- return _build_event(raw, pri, ts, host, msg, app=app, pid=pid)
210
+ return _build_event(
211
+ raw, pri, ts, host, msg, app=app, pid=pid, ts_orig=ts_orig
212
+ )
171
213
 
172
214
  # Analysis pass: find the earliest timestamp in the line head.
173
215
  best = None
@@ -194,7 +236,7 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
194
236
  msg = rest
195
237
  tokens = rest.split(None, 1)
196
238
  if tokens:
197
- candidate = tokens[0].rstrip(":,")
239
+ candidate = _normalize_host_token(tokens[0])
198
240
  if _valid_host(candidate):
199
241
  host = candidate
200
242
  msg = tokens[1] if len(tokens) > 1 else ""
@@ -213,19 +255,27 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
213
255
  prefix = line[: ts_match.start()]
214
256
  parts = [re.escape(prefix), f"(?P<adaptive_ts>{src})", r"[\s:,]*"]
215
257
  if host is not None:
216
- parts.append(r"(?P<adaptive_host>\S+?):?(?:\s+|$)")
258
+ # Capture the complete token. Normalizing punctuation after the
259
+ # match keeps first-sight and cached results identical and avoids
260
+ # treating the final colon of an IPv6 address as a separator.
261
+ parts.append(r"(?P<adaptive_host_token>\S+)(?:\s+|$)")
217
262
  parts.append(_TAG_SRC)
218
263
  parts.append(r"(?P<adaptive_msg>.*)$")
219
264
  try:
220
265
  pattern = re.compile("".join(parts))
221
266
  except re.error:
222
267
  pattern = None
223
- if pattern is not None and pattern.match(line):
268
+ # A hostless pattern is not stable enough to reuse: severity/timezone
269
+ # words and plain alphabetic hostnames collapse to the same signature.
270
+ # Caching it would suppress a valid host on a later line of that shape.
271
+ if host is not None and pattern is not None and pattern.match(line):
224
272
  if len(_CACHE) >= _CACHE_MAX:
225
273
  _CACHE.pop(next(iter(_CACHE)))
226
274
  _CACHE[sig] = (pattern, name, conv)
227
275
 
228
- return _build_event(raw, pri, ts, host, msg, app=app, pid=pid)
276
+ return _build_event(
277
+ raw, pri, ts, host, msg, app=app, pid=pid, ts_orig=ts_match.group(0)
278
+ )
229
279
 
230
280
 
231
281
  def pattern_ts_match(pattern: Pattern[str], match: "re.Match[str]"):
@@ -2,6 +2,7 @@ from __future__ import annotations
2
2
 
3
3
  import json
4
4
  import logging
5
+ import re
5
6
  from dataclasses import dataclass
6
7
  from datetime import datetime
7
8
  from pathlib import Path
@@ -11,10 +12,72 @@ from .parsers import ParsedEvent, ParserError, autodetect_and_parse
11
12
  from .normalizer import NormalizedEvent, normalize
12
13
  from .utils import sanitize_message
13
14
  from .mappings import CISCO_ASA, CISCO_IOS, F5, FORTINET, LINUX, SOPHOS, VMWARE
14
- from .cef import CEFEvent, build_cef
15
+ from .cef import HEADER_KEYS, CEFEvent, build_cef
15
16
 
16
17
  logger = logging.getLogger(__name__)
17
18
 
19
+ _EXTENSION_KEY_RE = re.compile(r"^[A-Za-z0-9_.]+$")
20
+
21
+ # What may follow a mapping key in ``template % fields``: flags, width,
22
+ # precision, length modifier, conversion. A width/precision of "*" is
23
+ # deliberately excluded because it consumes a positional argument a mapping
24
+ # cannot supply. A bare "." precision is legal Python (it means zero).
25
+ _FORMAT_TAIL_RE = re.compile(r"[#0\- +]*\d*(?:\.\d*)?[hlL]?[diouxXeEfFgGcrsa]")
26
+
27
+
28
+ def _scan_format_key(template: str, start: int) -> int:
29
+ """Return the index just past a mapping key's closing paren, or -1.
30
+
31
+ Mirrors CPython's own scan, which counts nesting rather than stopping at
32
+ the first ")": "%((a))s" is a valid reference to the key "(a)", while
33
+ "%(a(b)s" is an incomplete key that raises at render time.
34
+ """
35
+ depth = 1
36
+ for index in range(start, len(template)):
37
+ if template[index] == "(":
38
+ depth += 1
39
+ elif template[index] == ")":
40
+ depth -= 1
41
+ if depth == 0:
42
+ return index + 1
43
+ return -1
44
+
45
+
46
+ def _validate_template(template: str, where: str, what: str) -> None:
47
+ """Reject a template that cannot render, before any record is produced.
48
+
49
+ ``MappingResolver._format`` swallows a rendering failure and returns "",
50
+ which silently falls a header back to its default or drops an extension.
51
+ A malformed template is a configuration error, so surface it eagerly.
52
+ """
53
+
54
+ def fail(problem: str) -> None:
55
+ raise ValueError(f"{where}: {what} template {template!r} {problem}")
56
+
57
+ position = 0
58
+ while True:
59
+ start = template.find("%", position)
60
+ if start == -1:
61
+ return
62
+ cursor = start + 1
63
+ if cursor < len(template) and template[cursor] == "%":
64
+ position = cursor + 1
65
+ continue
66
+ if cursor >= len(template) or template[cursor] != "(":
67
+ fail(
68
+ f"has an invalid format specifier at index {start} "
69
+ "(use '%%' for a literal percent)"
70
+ )
71
+ key_end = _scan_format_key(template, cursor + 1)
72
+ if key_end == -1:
73
+ fail(f"has an incomplete format key at index {start}")
74
+ if key_end - 1 == cursor + 1:
75
+ fail(f"has an empty format key at index {start}")
76
+ tail = _FORMAT_TAIL_RE.match(template, key_end)
77
+ if tail is None:
78
+ fail(f"has an invalid conversion at index {key_end}")
79
+ position = tail.end()
80
+
18
81
 
19
82
  @dataclass
20
83
  class ParseResult:
@@ -62,23 +125,69 @@ def normalize_event(event: ParsedEvent | NormalizedEvent) -> NormalizedEvent:
62
125
  return normalize(event)
63
126
 
64
127
 
128
+ class _ValidatedMapping(dict):
129
+ """A mapping that has already passed structural validation.
130
+
131
+ ``to_cef`` loads its mapping per event, so a long stream would otherwise
132
+ re-run template and severity-map validation for every record. Callers
133
+ that convert many events (``StreamConverter``, the CLI) validate once and
134
+ pass the result back in; this marker lets the second load short-circuit.
135
+ """
136
+
137
+
65
138
  def _load_mapping(mapping: Mapping[str, Any] | Path | str | None) -> Mapping[str, Any]:
66
139
  if mapping is None:
67
140
  return {}
68
- if isinstance(mapping, Mapping):
141
+ if isinstance(mapping, _ValidatedMapping):
69
142
  return mapping
70
- path = Path(mapping)
71
- with path.open("r", encoding="utf-8") as fp:
72
- data = json.load(fp)
143
+ if isinstance(mapping, Mapping):
144
+ data = mapping
145
+ where = "mapping"
146
+ else:
147
+ path = Path(mapping)
148
+ with path.open("r", encoding="utf-8") as fp:
149
+ data = json.load(fp)
150
+ where = str(path)
73
151
  # Structural validation with clear messages: a top-level array would
74
152
  # otherwise TypeError deep inside the renderer, and a wrong-shaped
75
153
  # extensions/severity_map would fail per event instead of up front.
76
- if not isinstance(data, dict):
77
- raise ValueError(f"{path}: mapping must be a JSON object")
154
+ if not isinstance(data, Mapping):
155
+ raise ValueError(f"{where}: mapping must be a JSON object")
78
156
  for key in ("extensions", "severity_map"):
79
- if key in data and not isinstance(data[key], dict):
80
- raise ValueError(f"{path}: '{key}' must be a JSON object")
81
- return data
157
+ if key in data and not isinstance(data[key], Mapping):
158
+ raise ValueError(f"{where}: '{key}' must be an object")
159
+ for key in HEADER_KEYS:
160
+ if key in data:
161
+ if not isinstance(data[key], str):
162
+ raise ValueError(f"{where}: '{key}' must be a string")
163
+ _validate_template(data[key], where, f"header {key!r}")
164
+ for key, template in data.get("extensions", {}).items():
165
+ if not isinstance(key, str) or not _EXTENSION_KEY_RE.fullmatch(key):
166
+ raise ValueError(f"{where}: invalid extension key {key!r}")
167
+ if not isinstance(template, str):
168
+ raise ValueError(
169
+ f"{where}: extension template for {key!r} must be a string"
170
+ )
171
+ _validate_template(template, where, f"extension {key!r}")
172
+ for source, target in data.get("severity_map", {}).items():
173
+ source_text, target_text = str(source), str(target)
174
+ if not (
175
+ source_text.isascii() and source_text.isdigit()
176
+ and len(source_text) == 1
177
+ and 0 <= int(source_text) <= 7
178
+ ):
179
+ raise ValueError(
180
+ f"{where}: severity_map key {source!r} must be an integer from 0 to 7"
181
+ )
182
+ if not (
183
+ target_text.isascii() and target_text.isdigit()
184
+ and len(target_text) <= 2
185
+ and 0 <= int(target_text) <= 10
186
+ ):
187
+ raise ValueError(
188
+ f"{where}: severity_map value {target!r} must be an integer from 0 to 10"
189
+ )
190
+ return _ValidatedMapping(data)
82
191
 
83
192
 
84
193
  def to_cef(
@@ -149,7 +258,9 @@ class StreamConverter:
149
258
  strict: bool = False,
150
259
  ) -> None:
151
260
  self.mode = mode
152
- self.mapping = mapping
261
+ # Validate the mapping once here rather than per converted line, and
262
+ # surface a malformed mapping before any record is emitted.
263
+ self.mapping = None if mapping is None else _load_mapping(mapping)
153
264
  self.validate = validate
154
265
  self.strict = strict
155
266
  self._contexts: dict[str, ParsedEvent] = {}
@@ -9,6 +9,12 @@ from .utils import cef_escape, cef_escape_header
9
9
 
10
10
  logger = logging.getLogger(__name__)
11
11
 
12
+ # The CEF:0 header slots a mapping may supply a template for, in wire order.
13
+ # Single source of truth: mapping validation in api.py checks exactly these.
14
+ HEADER_KEYS = (
15
+ "deviceVendor", "deviceProduct", "deviceVersion", "eventClassId", "name",
16
+ )
17
+
12
18
 
13
19
  DEFAULT_MAPPING = {
14
20
  "deviceVendor": "Generic",
@@ -28,6 +34,14 @@ DEFAULT_MAPPING = {
28
34
  },
29
35
  "extensions": {
30
36
  "msg": "%(msg)s",
37
+ # The syslog HOSTNAME identifies the device that GENERATED the
38
+ # event, so it is the source host (shost) — never the destination
39
+ # (dhost), which is reserved for network-specific logs whose
40
+ # payload names an actual destination. shost (max 1023) rather
41
+ # than dvchost (max 100): RFC 5424 hostnames may reach 255 chars,
42
+ # which would fail --strict against dvchost's dictionary limit.
43
+ # Empty values are dropped, so this is a no-op for a hostless line.
44
+ "shost": "%(host)s",
31
45
  # Event time (epoch milliseconds) parsed from the line, or the
32
46
  # processing time when the line carried no usable timestamp.
33
47
  "rt": "%(rt)s",
@@ -60,7 +74,7 @@ class MappingResolver:
60
74
 
61
75
  def resolve_header(self, fields: Mapping[str, Any]) -> Dict[str, str]:
62
76
  header = {}
63
- for key in ["deviceVendor", "deviceProduct", "deviceVersion", "eventClassId", "name"]:
77
+ for key in HEADER_KEYS:
64
78
  template = self.mapping.get(key, DEFAULT_MAPPING[key])
65
79
  value = self._format(template, fields)
66
80
  if not value:
@@ -73,7 +87,13 @@ class MappingResolver:
73
87
  return header
74
88
 
75
89
  def _resolve_severity(self, fields: Mapping[str, Any]) -> str:
76
- severity_map = {**DEFAULT_MAPPING["severity_map"], **self.mapping.get("severity_map", {})}
90
+ severity_map = DEFAULT_MAPPING["severity_map"].copy()
91
+ # Programmatic mappings may naturally use integer keys/values while
92
+ # JSON mappings always contain strings. Normalize both forms so a
93
+ # validated {6: 9} override behaves like {"6": "9"}.
94
+ severity_map.update(
95
+ {str(key): str(value) for key, value in self.mapping.get("severity_map", {}).items()}
96
+ )
77
97
  value = fields.get("severity")
78
98
  if value is None and "pri" in fields:
79
99
  # "pri" may come from an untrusted kv pair (pri=zzz), not just
@@ -5,7 +5,7 @@
5
5
  "eventClassId": "linux.syslog",
6
6
  "name": "%(message_short)s",
7
7
  "extensions": {
8
- "dhost": "%(host)s",
8
+ "shost": "%(host)s",
9
9
  "sproc": "%(app)s",
10
10
  "dvcpid": "%(dvcpid)s",
11
11
  "cs1Label": "rawEvent",
@@ -38,10 +38,29 @@ class NormalizedEvent(ParsedEvent):
38
38
  "source_hint": self.source_hint,
39
39
  "msg": self.msg,
40
40
  }
41
- base.update(self.sd)
42
- base.update(self.kv)
43
- base.update(self.extras)
44
- return {k: v for k, v in base.items() if v is not None}
41
+ fields: Dict[str, Any] = {}
42
+ fields.update(self.sd)
43
+ fields.update(self.kv)
44
+ fields.update(self.extras)
45
+
46
+ # Parsed envelope metadata is more trustworthy than same-named
47
+ # key=value text inside the message. Reapply it last when present,
48
+ # while still allowing a kv field to fill metadata absent from the
49
+ # envelope. ``msg`` intentionally retains the established kv
50
+ # behavior for formats that carry an inner msg="..." value.
51
+ authoritative = {
52
+ "pri", "facility", "severity", "timestamp", "ts", "ts_orig",
53
+ "host", "app", "pid", "msgid", "source_hint",
54
+ }
55
+ for key in authoritative:
56
+ # An absent original timestamp is represented as "" rather than
57
+ # None, so emptiness must count as absent too: otherwise a
58
+ # record with no recognized envelope timestamp would reapply ""
59
+ # over a payload ts_orig= value and hide it from mappings.
60
+ if base[key] is not None and base[key] != "":
61
+ fields[key] = base[key]
62
+ fields.setdefault("msg", base["msg"])
63
+ return {k: v for k, v in fields.items() if v is not None}
45
64
 
46
65
 
47
66
  def normalize(event: ParsedEvent) -> NormalizedEvent:
@@ -98,7 +117,29 @@ def _derive_common_fields(event: NormalizedEvent) -> None:
98
117
  # including kv pairs extracted from adaptive-parsed lines — so
99
118
  # mappings and validation always see canonical CEF keys. Original
100
119
  # keys are kept; an existing canonical key is never overwritten.
101
- from .dictionary import field_aliases
120
+ from .dictionary import cef_keys, field_aliases
121
+
122
+ # Source formats frequently vary only in field-name casing (SRCIP,
123
+ # SrcIp, srcip). Preserve every original key, add a lowercase spelling,
124
+ # and prefer an explicitly lowercase value when both forms are present.
125
+ original_items = list(event.kv.items())
126
+ for key, value in original_items:
127
+ lower = key.lower()
128
+ if lower not in event.kv:
129
+ event.kv[lower] = value
130
+
131
+ # Restore the dictionary's exact spelling for canonical camelCase CEF
132
+ # keys (DeviceExternalId -> deviceExternalId), again without overwrite.
133
+ canonical_by_lower = {key.lower(): key for key in cef_keys()}
134
+ for key in list(event.kv):
135
+ lower = key.lower()
136
+ canonical = canonical_by_lower.get(lower)
137
+ if canonical is not None and canonical not in event.kv:
138
+ # Take the folded lowercase value rather than this spelling.
139
+ # The loop above already applied lowercase-wins precedence, so
140
+ # an explicit deviceexternalid= beats DEVICEEXTERNALID=
141
+ # regardless of which appeared first in the message.
142
+ event.kv[canonical] = event.kv.get(lower, event.kv[key])
102
143
 
103
144
  for alias, canonical in field_aliases().items():
104
145
  if canonical not in event.kv and alias in event.kv: