syslog2cef 0.3.2__tar.gz → 0.3.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. {syslog2cef-0.3.2/syslog2cef.egg-info → syslog2cef-0.3.3}/PKG-INFO +10 -4
  2. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/README.md +9 -3
  3. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/pyproject.toml +1 -1
  4. {syslog2cef-0.3.2 → syslog2cef-0.3.3/syslog2cef.egg-info}/PKG-INFO +10 -4
  5. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/syslogcef/adaptive.py +69 -19
  6. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/syslogcef/api.py +122 -11
  7. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/syslogcef/cef.py +14 -2
  8. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/syslogcef/normalizer.py +46 -5
  9. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/syslogcef/parsers.py +40 -19
  10. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/syslogcef/utils.py +10 -1
  11. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/tests/test_adaptive.py +129 -0
  12. syslog2cef-0.3.3/tests/test_cef.py +186 -0
  13. syslog2cef-0.3.3/tests/test_normalizer.py +123 -0
  14. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/tests/test_pipeline.py +19 -2
  15. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/tests/test_utils.py +36 -0
  16. syslog2cef-0.3.2/tests/test_cef.py +0 -64
  17. syslog2cef-0.3.2/tests/test_normalizer.py +0 -49
  18. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/LICENSE +0 -0
  19. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/setup.cfg +0 -0
  20. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/syslog2cef.egg-info/SOURCES.txt +0 -0
  21. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/syslog2cef.egg-info/dependency_links.txt +0 -0
  22. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/syslog2cef.egg-info/entry_points.txt +0 -0
  23. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/syslog2cef.egg-info/requires.txt +0 -0
  24. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/syslog2cef.egg-info/top_level.txt +0 -0
  25. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/syslogcef/__init__.py +0 -0
  26. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/syslogcef/__main__.py +0 -0
  27. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/syslogcef/cli.py +0 -0
  28. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/syslogcef/custom.py +0 -0
  29. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/syslogcef/dictionary.json +0 -0
  30. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/syslogcef/dictionary.py +0 -0
  31. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/syslogcef/mappings/__init__.py +0 -0
  32. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/syslogcef/mappings/cisco_asa.json +0 -0
  33. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/syslogcef/mappings/cisco_ios.json +0 -0
  34. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/syslogcef/mappings/f5.json +0 -0
  35. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/syslogcef/mappings/fortinet.json +0 -0
  36. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/syslogcef/mappings/linux.json +0 -0
  37. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/syslogcef/mappings/sophos.json +0 -0
  38. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/syslogcef/mappings/vmware.json +0 -0
  39. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/syslogcef/net.py +0 -0
  40. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/syslogcef/validation.py +0 -0
  41. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/tests/test_cli.py +0 -0
  42. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/tests/test_cli_validation.py +0 -0
  43. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/tests/test_custom_parsers.py +0 -0
  44. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/tests/test_dictionary.py +0 -0
  45. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/tests/test_fuzz_grammar.py +0 -0
  46. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/tests/test_fuzz_properties.py +0 -0
  47. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/tests/test_macos_logs.py +0 -0
  48. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/tests/test_net.py +0 -0
  49. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/tests/test_parsers.py +0 -0
  50. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/tests/test_parsers_extra.py +0 -0
  51. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/tests/test_real_world.py +0 -0
  52. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/tests/test_rfc5424_epoch.py +0 -0
  53. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/tests/test_security_hardening.py +0 -0
  54. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/tests/test_tail_rotation.py +0 -0
  55. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/tests/test_validation.py +0 -0
  56. {syslog2cef-0.3.2 → syslog2cef-0.3.3}/tests/test_zipapp_safe.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: syslog2cef
3
- Version: 0.3.2
3
+ Version: 0.3.3
4
4
  Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
5
  Author-email: Tamir Suliman <allamiro@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/allamiro/syslogcef
@@ -198,8 +198,11 @@ normalized event's fields:
198
198
  the message and any RFC5424 structured-data or journald fields.
199
199
 
200
200
  Field templates that reference missing keys resolve to an empty string
201
- rather than failing the event. See [docs/cef_fields.md](docs/cef_fields.md)
202
- for the full CEF extension dictionary.
201
+ rather than failing the event. Mapping files and Python mapping dictionaries
202
+ are validated before rendering; malformed templates, severity maps, and CEF
203
+ extension keys fail with a configuration error instead of corrupting a record.
204
+ See [docs/cef_fields.md](docs/cef_fields.md) for the full CEF extension
205
+ dictionary.
203
206
 
204
207
  ## Field Dictionary
205
208
 
@@ -217,7 +220,10 @@ knowledge, derived from the ArcSight Extension Dictionary (see
217
220
  are applied during normalization for **every** event — including
218
221
  key=value pairs extracted from adaptively-parsed unknown formats — so
219
222
  mappings and validation always see canonical names. Original keys are
220
- preserved and an explicit canonical key is never overwritten.
223
+ preserved and an explicit canonical key is never overwritten. Source key
224
+ casing is normalized as well (`SRCIP`, `SrcIp`, and `srcip` all make `src`
225
+ available), while parsed syslog envelope fields such as host and severity
226
+ remain authoritative over same-named text in the message.
221
227
 
222
228
  This means a Fortinet-style `srcip=10.1.1.1 dstport=443` and an unknown
223
229
  device emitting the same pairs behind an unrecognized prefix both end up
@@ -175,8 +175,11 @@ normalized event's fields:
175
175
  the message and any RFC5424 structured-data or journald fields.
176
176
 
177
177
  Field templates that reference missing keys resolve to an empty string
178
- rather than failing the event. See [docs/cef_fields.md](docs/cef_fields.md)
179
- for the full CEF extension dictionary.
178
+ rather than failing the event. Mapping files and Python mapping dictionaries
179
+ are validated before rendering; malformed templates, severity maps, and CEF
180
+ extension keys fail with a configuration error instead of corrupting a record.
181
+ See [docs/cef_fields.md](docs/cef_fields.md) for the full CEF extension
182
+ dictionary.
180
183
 
181
184
  ## Field Dictionary
182
185
 
@@ -194,7 +197,10 @@ knowledge, derived from the ArcSight Extension Dictionary (see
194
197
  are applied during normalization for **every** event — including
195
198
  key=value pairs extracted from adaptively-parsed unknown formats — so
196
199
  mappings and validation always see canonical names. Original keys are
197
- preserved and an explicit canonical key is never overwritten.
200
+ preserved and an explicit canonical key is never overwritten. Source key
201
+ casing is normalized as well (`SRCIP`, `SrcIp`, and `srcip` all make `src`
202
+ available), while parsed syslog envelope fields such as host and severity
203
+ remain authoritative over same-named text in the message.
198
204
 
199
205
  This means a Fortinet-style `srcip=10.1.1.1 dstport=443` and an unknown
200
206
  device emitting the same pairs behind an unrecognized prefix both end up
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "syslog2cef"
7
- version = "0.3.2"
7
+ version = "0.3.3"
8
8
  description = "Convert RFC3164/RFC5424 syslog lines into ArcSight CEF"
9
9
  authors = [{name = "Tamir Suliman", email = "allamiro@gmail.com"}]
10
10
  readme = "README.md"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: syslog2cef
3
- Version: 0.3.2
3
+ Version: 0.3.3
4
4
  Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
5
  Author-email: Tamir Suliman <allamiro@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/allamiro/syslogcef
@@ -198,8 +198,11 @@ normalized event's fields:
198
198
  the message and any RFC5424 structured-data or journald fields.
199
199
 
200
200
  Field templates that reference missing keys resolve to an empty string
201
- rather than failing the event. See [docs/cef_fields.md](docs/cef_fields.md)
202
- for the full CEF extension dictionary.
201
+ rather than failing the event. Mapping files and Python mapping dictionaries
202
+ are validated before rendering; malformed templates, severity maps, and CEF
203
+ extension keys fail with a configuration error instead of corrupting a record.
204
+ See [docs/cef_fields.md](docs/cef_fields.md) for the full CEF extension
205
+ dictionary.
203
206
 
204
207
  ## Field Dictionary
205
208
 
@@ -217,7 +220,10 @@ knowledge, derived from the ArcSight Extension Dictionary (see
217
220
  are applied during normalization for **every** event — including
218
221
  key=value pairs extracted from adaptively-parsed unknown formats — so
219
222
  mappings and validation always see canonical names. Original keys are
220
- preserved and an explicit canonical key is never overwritten.
223
+ preserved and an explicit canonical key is never overwritten. Source key
224
+ casing is normalized as well (`SRCIP`, `SrcIp`, and `srcip` all make `src`
225
+ available), while parsed syslog envelope fields such as host and severity
226
+ remain authoritative over same-named text in the message.
221
227
 
222
228
  This means a Fortinet-style `srcip=10.1.1.1 dstport=443` and an unknown
223
229
  device emitting the same pairs behind an unrecognized prefix both end up
@@ -15,7 +15,7 @@ import re
15
15
  from datetime import datetime, timezone
16
16
  from typing import Callable, Dict, Optional, Pattern, Tuple
17
17
 
18
- from .utils import convert_pri, month_abbr_to_int, parse_iso8601, sanitize_message
18
+ from .utils import convert_pri, is_ip, month_abbr_to_int, parse_iso8601, sanitize_message
19
19
 
20
20
  PRI_RE = re.compile(r"^<(\d{1,3})>")
21
21
 
@@ -69,7 +69,20 @@ TIMESTAMP_LIB: Tuple[Tuple[str, str, Callable], ...] = (
69
69
  )
70
70
 
71
71
  HOST_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
72
- _SKIP_HOST_TOKENS = {"error", "warning", "info", "debug", "notice", "kernel"}
72
+ _SKIP_HOST_TOKENS = {
73
+ "alert", "crit", "critical", "debug", "emerg", "emergency", "err",
74
+ "error", "info", "information", "informational", "kernel", "notice",
75
+ "panic", "warn", "warning",
76
+ # Common named timezone remnants. The timestamp shapes deliberately do
77
+ # not pretend to resolve named zones, but these must not become hosts.
78
+ # Abbreviations that double as ordinary English words (WEST, CAT, EAT,
79
+ # ART) are deliberately excluded: rejecting them would drop genuine
80
+ # short hostnames, which is the worse failure of the two.
81
+ "utc", "gmt", "cet", "cest", "eet", "eest", "est", "edt", "cst",
82
+ "cdt", "mst", "mdt", "pst", "pdt", "bst", "ist", "idt", "jst",
83
+ "kst", "aest", "aedt", "acst", "acdt", "awst", "nzst", "nzdt",
84
+ "msk", "hst", "akst", "akdt", "sast", "brt", "clt",
85
+ }
73
86
 
74
87
  # Syslog-style program tag following the host: "app:", "app[pid]:".
75
88
  TAG_RE = re.compile(r"^(?P<app>[\w\-/\.]+)(?:\[(?P<pid>[^\]]+)\])?:$")
@@ -78,13 +91,38 @@ _TAG_SRC = r"(?:(?P<adaptive_app>[\w\-/\.]+)(?:\[(?P<adaptive_pid>[^\]]+)\])?:\s
78
91
 
79
92
 
80
93
  def _valid_host(candidate: Optional[str]) -> bool:
81
- return bool(
82
- candidate
83
- and "=" not in candidate
84
- and HOST_RE.match(candidate)
85
- and candidate.lower() not in _SKIP_HOST_TOKENS
86
- and not candidate.isdigit()
87
- )
94
+ if not candidate or "=" in candidate or candidate.lower() in _SKIP_HOST_TOKENS:
95
+ return False
96
+ # Adaptive layouts sometimes use a literal IP in the hostname slot.
97
+ # Check it before rejecting all-numeric tokens.
98
+ if is_ip(candidate):
99
+ return True
100
+ return bool(HOST_RE.fullmatch(candidate) and not candidate.isdigit())
101
+
102
+
103
+ def _normalize_host_token(token: str) -> str:
104
+ """Strip trailing layout delimiters from a host token.
105
+
106
+ A token that is already a valid address is returned untouched: a
107
+ compressed IPv6 literal may legitimately end in "::" ("2001:db8::",
108
+ "fe80::"), and stripping that would silently corrupt the host. Anything
109
+ else has its whole trailing ":," run removed, so multi-character layout
110
+ delimiters ("host::", "host:,") work as the analysis path always did.
111
+ """
112
+ if is_ip(token):
113
+ return token
114
+ # A layout delimiter may follow a compressed literal that itself ends in
115
+ # "::" ("2001:db8::,"), where neither the whole token nor a full strip
116
+ # yields the address. Remove one trailing separator at a time and keep
117
+ # the first candidate that parses, falling back to the fully stripped
118
+ # token for ordinary hostnames ("host::", "host:,").
119
+ candidate = token
120
+ while candidate and candidate[-1] in ":,":
121
+ candidate = candidate[:-1]
122
+ if is_ip(candidate):
123
+ return candidate
124
+ return candidate
125
+
88
126
 
89
127
  _CACHE: Dict[str, Tuple[Pattern[str], str, Callable]] = {}
90
128
  _CACHE_MAX = 256
@@ -114,7 +152,7 @@ def cache_size() -> int:
114
152
  return len(_CACHE)
115
153
 
116
154
 
117
- def _build_event(line, pri, ts, host, msg, app=None, pid=None):
155
+ def _build_event(line, pri, ts, host, msg, app=None, pid=None, ts_orig=""):
118
156
  from .parsers import ParsedEvent
119
157
 
120
158
  facility, severity = convert_pri(pri)
@@ -123,7 +161,7 @@ def _build_event(line, pri, ts, host, msg, app=None, pid=None):
123
161
  facility=facility,
124
162
  severity=severity,
125
163
  ts=ts,
126
- ts_orig="",
164
+ ts_orig=ts_orig,
127
165
  host=host,
128
166
  app=app,
129
167
  pid=pid,
@@ -156,18 +194,22 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
156
194
  except (ValueError, KeyError):
157
195
  ts = None
158
196
  gd = match.groupdict()
159
- host = gd.get("adaptive_host")
197
+ ts_orig = match.group("adaptive_ts")
198
+ host_token = gd.get("adaptive_host_token")
199
+ host = _normalize_host_token(host_token) if host_token else None
160
200
  app = gd.get("adaptive_app")
161
201
  pid = gd.get("adaptive_pid")
162
202
  msg = gd.get("adaptive_msg")
163
203
  msg = line if msg is None else msg
164
204
  # The shape signature reduces all words to the same class, so a
165
205
  # cached host slot may capture a non-host token; revalidate.
166
- if host is not None and not _valid_host(host.rstrip(":,")):
206
+ if host_token is not None and not _valid_host(host):
167
207
  tag = f"{app}[{pid}]: " if app and pid else (f"{app}: " if app else "")
168
- msg = f"{host} {tag}{msg}".strip()
208
+ msg = f"{host_token} {tag}{msg}".strip()
169
209
  host = app = pid = None
170
- return _build_event(raw, pri, ts, host, msg, app=app, pid=pid)
210
+ return _build_event(
211
+ raw, pri, ts, host, msg, app=app, pid=pid, ts_orig=ts_orig
212
+ )
171
213
 
172
214
  # Analysis pass: find the earliest timestamp in the line head.
173
215
  best = None
@@ -194,7 +236,7 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
194
236
  msg = rest
195
237
  tokens = rest.split(None, 1)
196
238
  if tokens:
197
- candidate = tokens[0].rstrip(":,")
239
+ candidate = _normalize_host_token(tokens[0])
198
240
  if _valid_host(candidate):
199
241
  host = candidate
200
242
  msg = tokens[1] if len(tokens) > 1 else ""
@@ -213,19 +255,27 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
213
255
  prefix = line[: ts_match.start()]
214
256
  parts = [re.escape(prefix), f"(?P<adaptive_ts>{src})", r"[\s:,]*"]
215
257
  if host is not None:
216
- parts.append(r"(?P<adaptive_host>\S+?):?(?:\s+|$)")
258
+ # Capture the complete token. Normalizing punctuation after the
259
+ # match keeps first-sight and cached results identical and avoids
260
+ # treating the final colon of an IPv6 address as a separator.
261
+ parts.append(r"(?P<adaptive_host_token>\S+)(?:\s+|$)")
217
262
  parts.append(_TAG_SRC)
218
263
  parts.append(r"(?P<adaptive_msg>.*)$")
219
264
  try:
220
265
  pattern = re.compile("".join(parts))
221
266
  except re.error:
222
267
  pattern = None
223
- if pattern is not None and pattern.match(line):
268
+ # A hostless pattern is not stable enough to reuse: severity/timezone
269
+ # words and plain alphabetic hostnames collapse to the same signature.
270
+ # Caching it would suppress a valid host on a later line of that shape.
271
+ if host is not None and pattern is not None and pattern.match(line):
224
272
  if len(_CACHE) >= _CACHE_MAX:
225
273
  _CACHE.pop(next(iter(_CACHE)))
226
274
  _CACHE[sig] = (pattern, name, conv)
227
275
 
228
- return _build_event(raw, pri, ts, host, msg, app=app, pid=pid)
276
+ return _build_event(
277
+ raw, pri, ts, host, msg, app=app, pid=pid, ts_orig=ts_match.group(0)
278
+ )
229
279
 
230
280
 
231
281
  def pattern_ts_match(pattern: Pattern[str], match: "re.Match[str]"):
@@ -2,6 +2,7 @@ from __future__ import annotations
2
2
 
3
3
  import json
4
4
  import logging
5
+ import re
5
6
  from dataclasses import dataclass
6
7
  from datetime import datetime
7
8
  from pathlib import Path
@@ -11,10 +12,72 @@ from .parsers import ParsedEvent, ParserError, autodetect_and_parse
11
12
  from .normalizer import NormalizedEvent, normalize
12
13
  from .utils import sanitize_message
13
14
  from .mappings import CISCO_ASA, CISCO_IOS, F5, FORTINET, LINUX, SOPHOS, VMWARE
14
- from .cef import CEFEvent, build_cef
15
+ from .cef import HEADER_KEYS, CEFEvent, build_cef
15
16
 
16
17
  logger = logging.getLogger(__name__)
17
18
 
19
+ _EXTENSION_KEY_RE = re.compile(r"^[A-Za-z0-9_.]+$")
20
+
21
+ # What may follow a mapping key in ``template % fields``: flags, width,
22
+ # precision, length modifier, conversion. A width/precision of "*" is
23
+ # deliberately excluded because it consumes a positional argument a mapping
24
+ # cannot supply. A bare "." precision is legal Python (it means zero).
25
+ _FORMAT_TAIL_RE = re.compile(r"[#0\- +]*\d*(?:\.\d*)?[hlL]?[diouxXeEfFgGcrsa]")
26
+
27
+
28
+ def _scan_format_key(template: str, start: int) -> int:
29
+ """Return the index just past a mapping key's closing paren, or -1.
30
+
31
+ Mirrors CPython's own scan, which counts nesting rather than stopping at
32
+ the first ")": "%((a))s" is a valid reference to the key "(a)", while
33
+ "%(a(b)s" is an incomplete key that raises at render time.
34
+ """
35
+ depth = 1
36
+ for index in range(start, len(template)):
37
+ if template[index] == "(":
38
+ depth += 1
39
+ elif template[index] == ")":
40
+ depth -= 1
41
+ if depth == 0:
42
+ return index + 1
43
+ return -1
44
+
45
+
46
+ def _validate_template(template: str, where: str, what: str) -> None:
47
+ """Reject a template that cannot render, before any record is produced.
48
+
49
+ ``MappingResolver._format`` swallows a rendering failure and returns "",
50
+ which silently falls a header back to its default or drops an extension.
51
+ A malformed template is a configuration error, so surface it eagerly.
52
+ """
53
+
54
+ def fail(problem: str) -> None:
55
+ raise ValueError(f"{where}: {what} template {template!r} {problem}")
56
+
57
+ position = 0
58
+ while True:
59
+ start = template.find("%", position)
60
+ if start == -1:
61
+ return
62
+ cursor = start + 1
63
+ if cursor < len(template) and template[cursor] == "%":
64
+ position = cursor + 1
65
+ continue
66
+ if cursor >= len(template) or template[cursor] != "(":
67
+ fail(
68
+ f"has an invalid format specifier at index {start} "
69
+ "(use '%%' for a literal percent)"
70
+ )
71
+ key_end = _scan_format_key(template, cursor + 1)
72
+ if key_end == -1:
73
+ fail(f"has an incomplete format key at index {start}")
74
+ if key_end - 1 == cursor + 1:
75
+ fail(f"has an empty format key at index {start}")
76
+ tail = _FORMAT_TAIL_RE.match(template, key_end)
77
+ if tail is None:
78
+ fail(f"has an invalid conversion at index {key_end}")
79
+ position = tail.end()
80
+
18
81
 
19
82
  @dataclass
20
83
  class ParseResult:
@@ -62,23 +125,69 @@ def normalize_event(event: ParsedEvent | NormalizedEvent) -> NormalizedEvent:
62
125
  return normalize(event)
63
126
 
64
127
 
128
+ class _ValidatedMapping(dict):
129
+ """A mapping that has already passed structural validation.
130
+
131
+ ``to_cef`` loads its mapping per event, so a long stream would otherwise
132
+ re-run template and severity-map validation for every record. Callers
133
+ that convert many events (``StreamConverter``, the CLI) validate once and
134
+ pass the result back in; this marker lets the second load short-circuit.
135
+ """
136
+
137
+
65
138
  def _load_mapping(mapping: Mapping[str, Any] | Path | str | None) -> Mapping[str, Any]:
66
139
  if mapping is None:
67
140
  return {}
68
- if isinstance(mapping, Mapping):
141
+ if isinstance(mapping, _ValidatedMapping):
69
142
  return mapping
70
- path = Path(mapping)
71
- with path.open("r", encoding="utf-8") as fp:
72
- data = json.load(fp)
143
+ if isinstance(mapping, Mapping):
144
+ data = mapping
145
+ where = "mapping"
146
+ else:
147
+ path = Path(mapping)
148
+ with path.open("r", encoding="utf-8") as fp:
149
+ data = json.load(fp)
150
+ where = str(path)
73
151
  # Structural validation with clear messages: a top-level array would
74
152
  # otherwise TypeError deep inside the renderer, and a wrong-shaped
75
153
  # extensions/severity_map would fail per event instead of up front.
76
- if not isinstance(data, dict):
77
- raise ValueError(f"{path}: mapping must be a JSON object")
154
+ if not isinstance(data, Mapping):
155
+ raise ValueError(f"{where}: mapping must be a JSON object")
78
156
  for key in ("extensions", "severity_map"):
79
- if key in data and not isinstance(data[key], dict):
80
- raise ValueError(f"{path}: '{key}' must be a JSON object")
81
- return data
157
+ if key in data and not isinstance(data[key], Mapping):
158
+ raise ValueError(f"{where}: '{key}' must be an object")
159
+ for key in HEADER_KEYS:
160
+ if key in data:
161
+ if not isinstance(data[key], str):
162
+ raise ValueError(f"{where}: '{key}' must be a string")
163
+ _validate_template(data[key], where, f"header {key!r}")
164
+ for key, template in data.get("extensions", {}).items():
165
+ if not isinstance(key, str) or not _EXTENSION_KEY_RE.fullmatch(key):
166
+ raise ValueError(f"{where}: invalid extension key {key!r}")
167
+ if not isinstance(template, str):
168
+ raise ValueError(
169
+ f"{where}: extension template for {key!r} must be a string"
170
+ )
171
+ _validate_template(template, where, f"extension {key!r}")
172
+ for source, target in data.get("severity_map", {}).items():
173
+ source_text, target_text = str(source), str(target)
174
+ if not (
175
+ source_text.isascii() and source_text.isdigit()
176
+ and len(source_text) == 1
177
+ and 0 <= int(source_text) <= 7
178
+ ):
179
+ raise ValueError(
180
+ f"{where}: severity_map key {source!r} must be an integer from 0 to 7"
181
+ )
182
+ if not (
183
+ target_text.isascii() and target_text.isdigit()
184
+ and len(target_text) <= 2
185
+ and 0 <= int(target_text) <= 10
186
+ ):
187
+ raise ValueError(
188
+ f"{where}: severity_map value {target!r} must be an integer from 0 to 10"
189
+ )
190
+ return _ValidatedMapping(data)
82
191
 
83
192
 
84
193
  def to_cef(
@@ -149,7 +258,9 @@ class StreamConverter:
149
258
  strict: bool = False,
150
259
  ) -> None:
151
260
  self.mode = mode
152
- self.mapping = mapping
261
+ # Validate the mapping once here rather than per converted line, and
262
+ # surface a malformed mapping before any record is emitted.
263
+ self.mapping = None if mapping is None else _load_mapping(mapping)
153
264
  self.validate = validate
154
265
  self.strict = strict
155
266
  self._contexts: dict[str, ParsedEvent] = {}
@@ -9,6 +9,12 @@ from .utils import cef_escape, cef_escape_header
9
9
 
10
10
  logger = logging.getLogger(__name__)
11
11
 
12
+ # The CEF:0 header slots a mapping may supply a template for, in wire order.
13
+ # Single source of truth: mapping validation in api.py checks exactly these.
14
+ HEADER_KEYS = (
15
+ "deviceVendor", "deviceProduct", "deviceVersion", "eventClassId", "name",
16
+ )
17
+
12
18
 
13
19
  DEFAULT_MAPPING = {
14
20
  "deviceVendor": "Generic",
@@ -68,7 +74,7 @@ class MappingResolver:
68
74
 
69
75
  def resolve_header(self, fields: Mapping[str, Any]) -> Dict[str, str]:
70
76
  header = {}
71
- for key in ["deviceVendor", "deviceProduct", "deviceVersion", "eventClassId", "name"]:
77
+ for key in HEADER_KEYS:
72
78
  template = self.mapping.get(key, DEFAULT_MAPPING[key])
73
79
  value = self._format(template, fields)
74
80
  if not value:
@@ -81,7 +87,13 @@ class MappingResolver:
81
87
  return header
82
88
 
83
89
  def _resolve_severity(self, fields: Mapping[str, Any]) -> str:
84
- severity_map = {**DEFAULT_MAPPING["severity_map"], **self.mapping.get("severity_map", {})}
90
+ severity_map = DEFAULT_MAPPING["severity_map"].copy()
91
+ # Programmatic mappings may naturally use integer keys/values while
92
+ # JSON mappings always contain strings. Normalize both forms so a
93
+ # validated {6: 9} override behaves like {"6": "9"}.
94
+ severity_map.update(
95
+ {str(key): str(value) for key, value in self.mapping.get("severity_map", {}).items()}
96
+ )
85
97
  value = fields.get("severity")
86
98
  if value is None and "pri" in fields:
87
99
  # "pri" may come from an untrusted kv pair (pri=zzz), not just
@@ -38,10 +38,29 @@ class NormalizedEvent(ParsedEvent):
38
38
  "source_hint": self.source_hint,
39
39
  "msg": self.msg,
40
40
  }
41
- base.update(self.sd)
42
- base.update(self.kv)
43
- base.update(self.extras)
44
- return {k: v for k, v in base.items() if v is not None}
41
+ fields: Dict[str, Any] = {}
42
+ fields.update(self.sd)
43
+ fields.update(self.kv)
44
+ fields.update(self.extras)
45
+
46
+ # Parsed envelope metadata is more trustworthy than same-named
47
+ # key=value text inside the message. Reapply it last when present,
48
+ # while still allowing a kv field to fill metadata absent from the
49
+ # envelope. ``msg`` intentionally retains the established kv
50
+ # behavior for formats that carry an inner msg="..." value.
51
+ authoritative = {
52
+ "pri", "facility", "severity", "timestamp", "ts", "ts_orig",
53
+ "host", "app", "pid", "msgid", "source_hint",
54
+ }
55
+ for key in authoritative:
56
+ # An absent original timestamp is represented as "" rather than
57
+ # None, so emptiness must count as absent too: otherwise a
58
+ # record with no recognized envelope timestamp would reapply ""
59
+ # over a payload ts_orig= value and hide it from mappings.
60
+ if base[key] is not None and base[key] != "":
61
+ fields[key] = base[key]
62
+ fields.setdefault("msg", base["msg"])
63
+ return {k: v for k, v in fields.items() if v is not None}
45
64
 
46
65
 
47
66
  def normalize(event: ParsedEvent) -> NormalizedEvent:
@@ -98,7 +117,29 @@ def _derive_common_fields(event: NormalizedEvent) -> None:
98
117
  # including kv pairs extracted from adaptive-parsed lines — so
99
118
  # mappings and validation always see canonical CEF keys. Original
100
119
  # keys are kept; an existing canonical key is never overwritten.
101
- from .dictionary import field_aliases
120
+ from .dictionary import cef_keys, field_aliases
121
+
122
+ # Source formats frequently vary only in field-name casing (SRCIP,
123
+ # SrcIp, srcip). Preserve every original key, add a lowercase spelling,
124
+ # and prefer an explicitly lowercase value when both forms are present.
125
+ original_items = list(event.kv.items())
126
+ for key, value in original_items:
127
+ lower = key.lower()
128
+ if lower not in event.kv:
129
+ event.kv[lower] = value
130
+
131
+ # Restore the dictionary's exact spelling for canonical camelCase CEF
132
+ # keys (DeviceExternalId -> deviceExternalId), again without overwrite.
133
+ canonical_by_lower = {key.lower(): key for key in cef_keys()}
134
+ for key in list(event.kv):
135
+ lower = key.lower()
136
+ canonical = canonical_by_lower.get(lower)
137
+ if canonical is not None and canonical not in event.kv:
138
+ # Take the folded lowercase value rather than this spelling.
139
+ # The loop above already applied lowercase-wins precedence, so
140
+ # an explicit deviceexternalid= beats DEVICEEXTERNALID=
141
+ # regardless of which appeared first in the message.
142
+ event.kv[canonical] = event.kv.get(lower, event.kv[key])
102
143
 
103
144
  for alias, canonical in field_aliases().items():
104
145
  if canonical not in event.kv and alias in event.kv:
@@ -124,6 +124,16 @@ def _looks_like_kv_line(body: str) -> bool:
124
124
  if len(matches) < 3:
125
125
  return False
126
126
  covered = sum(m.end() - m.start() for m in matches)
127
+ # KEY_VALUE_RE excludes a trailing comma from a value, where it used to
128
+ # be captured as part of it. Credit exactly those commas back so a
129
+ # comma-delimited stream of short values ("a=1, b=2, c=3, d=4") scores
130
+ # as it always did. Only commas are credited, and only in a gap that is
131
+ # nothing but separators: crediting whitespace too would newly qualify
132
+ # lines of prose that merely contain a few pairs.
133
+ for current, following in zip(matches, matches[1:]):
134
+ gap = body[current.end():following.start()]
135
+ if gap and not gap.strip(", \t"):
136
+ covered += gap.count(",")
127
137
  return covered >= 0.7 * len(body.strip())
128
138
 
129
139
 
@@ -609,6 +619,16 @@ def parse_kv_stream(line: str) -> ParsedEvent | None:
609
619
  if len(pairs) < 3:
610
620
  return None
611
621
 
622
+ # Device key names are commonly emitted in uppercase. Use a folded
623
+ # metadata view for parsing while preserving the exact originals in the
624
+ # message/normalizer. An explicitly lowercase spelling wins collisions.
625
+ pairs_ci: Dict[str, str] = {}
626
+ for key, value in pairs.items():
627
+ pairs_ci.setdefault(key.lower(), value)
628
+ for key, value in pairs.items():
629
+ if key == key.lower():
630
+ pairs_ci[key] = value
631
+
612
632
  # Timestamp candidates in preference order, each falling through to
613
633
  # the next when absent or unparseable (an invalid ts= must not mask
614
634
  # a valid eventtime=). Overflow/OSError cover out-of-range epoch
@@ -616,17 +636,17 @@ def parse_kv_stream(line: str) -> ParsedEvent | None:
616
636
  # ts_orig records the candidate that actually parsed.
617
637
  ts = None
618
638
  ts_orig = ""
619
- if pairs.get("date") and pairs.get("time"):
639
+ if pairs_ci.get("date") and pairs_ci.get("time"):
620
640
  try:
621
- ts = parse_iso8601(f"{pairs['date']}T{pairs['time']}")
622
- ts_orig = f"{pairs['date']} {pairs['time']}"
641
+ ts = parse_iso8601(f"{pairs_ci['date']}T{pairs_ci['time']}")
642
+ ts_orig = f"{pairs_ci['date']} {pairs_ci['time']}"
623
643
  except (ValueError, OverflowError, OSError):
624
644
  ts = None
625
645
  if ts is None:
626
646
  # Each ISO alias tried independently: an invalid ts= must not
627
647
  # mask a valid timestamp= or datetime=.
628
648
  for iso_key in ("ts", "timestamp", "datetime"):
629
- raw_iso = pairs.get(iso_key)
649
+ raw_iso = pairs_ci.get(iso_key)
630
650
  if not raw_iso:
631
651
  continue
632
652
  try:
@@ -635,28 +655,29 @@ def parse_kv_stream(line: str) -> ParsedEvent | None:
635
655
  break
636
656
  except (ValueError, OverflowError, OSError):
637
657
  continue
638
- if ts is None and pairs.get("eventtime", "").isdigit():
658
+ if ts is None and pairs_ci.get("eventtime", "").isdigit():
639
659
  try:
640
660
  # Full value: parse_iso8601 dispatches on digit count
641
661
  # (10/13/16/19), so Fortinet's 19-digit nanoseconds no
642
662
  # longer need lossy [:16] truncation.
643
- ts = parse_iso8601(pairs["eventtime"])
644
- ts_orig = pairs["eventtime"]
663
+ ts = parse_iso8601(pairs_ci["eventtime"])
664
+ ts_orig = pairs_ci["eventtime"]
645
665
  except (ValueError, OverflowError, OSError):
646
666
  ts = None
647
667
  if not ts_orig:
648
668
  ts_orig = (
649
- f"{pairs.get('date', '')} {pairs.get('time', '')}".strip()
650
- or pairs.get("ts")
651
- or pairs.get("timestamp")
652
- or pairs.get("datetime")
669
+ f"{pairs_ci.get('date', '')} {pairs_ci.get('time', '')}".strip()
670
+ or pairs_ci.get("ts")
671
+ or pairs_ci.get("timestamp")
672
+ or pairs_ci.get("datetime")
653
673
  or ""
654
674
  )
655
675
  if ts is not None and ts.tzinfo is None:
656
676
  # Apply a numeric offset supplied in the record (tz="-0500");
657
677
  # named zones (timezone="CEST") cannot be resolved portably.
658
678
  offset_match = re.fullmatch(
659
- r"([+-])(\d{2}):?(\d{2})", pairs.get("tz") or pairs.get("timezone") or ""
679
+ r"([+-])(\d{2}):?(\d{2})",
680
+ pairs_ci.get("tz") or pairs_ci.get("timezone") or "",
660
681
  )
661
682
  if offset_match:
662
683
  sign = 1 if offset_match.group(1) == "+" else -1
@@ -671,11 +692,11 @@ def parse_kv_stream(line: str) -> ParsedEvent | None:
671
692
  pass
672
693
 
673
694
  host = (
674
- pairs.get("devname")
675
- or pairs.get("device_name")
676
- or pairs.get("dvchost")
677
- or pairs.get("devid")
678
- or pairs.get("device_id")
695
+ pairs_ci.get("devname")
696
+ or pairs_ci.get("device_name")
697
+ or pairs_ci.get("dvchost")
698
+ or pairs_ci.get("devid")
699
+ or pairs_ci.get("device_id")
679
700
  )
680
701
  return ParsedEvent(
681
702
  pri=pri,
@@ -684,9 +705,9 @@ def parse_kv_stream(line: str) -> ParsedEvent | None:
684
705
  ts=ts,
685
706
  ts_orig=ts_orig,
686
707
  host=host,
687
- app=pairs.get("type") or pairs.get("log_type"),
708
+ app=pairs_ci.get("type") or pairs_ci.get("log_type"),
688
709
  pid=None,
689
- msgid=pairs.get("logid") or pairs.get("log_id"),
710
+ msgid=pairs_ci.get("logid") or pairs_ci.get("log_id"),
690
711
  sd={},
691
712
  msg=body.strip(),
692
713
  raw=line,