syslog2cef 0.3.0__tar.gz → 0.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {syslog2cef-0.3.0/syslog2cef.egg-info → syslog2cef-0.3.2}/PKG-INFO +1 -1
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/pyproject.toml +1 -1
- {syslog2cef-0.3.0 → syslog2cef-0.3.2/syslog2cef.egg-info}/PKG-INFO +1 -1
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/syslogcef/api.py +36 -8
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/syslogcef/cef.py +8 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/syslogcef/cli.py +38 -12
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/syslogcef/mappings/linux.json +2 -2
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/syslogcef/normalizer.py +9 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/syslogcef/parsers.py +7 -1
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/tests/test_macos_logs.py +135 -3
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/tests/test_real_world.py +19 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/LICENSE +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/README.md +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/setup.cfg +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/syslog2cef.egg-info/SOURCES.txt +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/syslog2cef.egg-info/dependency_links.txt +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/syslog2cef.egg-info/entry_points.txt +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/syslog2cef.egg-info/requires.txt +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/syslog2cef.egg-info/top_level.txt +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/syslogcef/__init__.py +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/syslogcef/__main__.py +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/syslogcef/adaptive.py +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/syslogcef/custom.py +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/syslogcef/dictionary.json +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/syslogcef/dictionary.py +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/syslogcef/mappings/__init__.py +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/syslogcef/mappings/cisco_asa.json +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/syslogcef/mappings/cisco_ios.json +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/syslogcef/mappings/f5.json +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/syslogcef/mappings/fortinet.json +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/syslogcef/mappings/sophos.json +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/syslogcef/mappings/vmware.json +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/syslogcef/net.py +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/syslogcef/utils.py +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/syslogcef/validation.py +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/tests/test_adaptive.py +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/tests/test_cef.py +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/tests/test_cli.py +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/tests/test_cli_validation.py +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/tests/test_custom_parsers.py +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/tests/test_dictionary.py +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/tests/test_fuzz_grammar.py +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/tests/test_fuzz_properties.py +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/tests/test_net.py +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/tests/test_normalizer.py +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/tests/test_parsers.py +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/tests/test_parsers_extra.py +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/tests/test_pipeline.py +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/tests/test_rfc5424_epoch.py +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/tests/test_security_hardening.py +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/tests/test_tail_rotation.py +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/tests/test_utils.py +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/tests/test_validation.py +0 -0
- {syslog2cef-0.3.0 → syslog2cef-0.3.2}/tests/test_zipapp_safe.py +0 -0
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "syslog2cef"
|
|
7
|
-
version = "0.3.
|
|
7
|
+
version = "0.3.2"
|
|
8
8
|
description = "Convert RFC3164/RFC5424 syslog lines into ArcSight CEF"
|
|
9
9
|
authors = [{name = "Tamir Suliman", email = "allamiro@gmail.com"}]
|
|
10
10
|
readme = "README.md"
|
|
@@ -7,8 +7,9 @@ from datetime import datetime
|
|
|
7
7
|
from pathlib import Path
|
|
8
8
|
from typing import Any, Mapping, Optional
|
|
9
9
|
|
|
10
|
-
from .parsers import ParsedEvent, autodetect_and_parse
|
|
10
|
+
from .parsers import ParsedEvent, ParserError, autodetect_and_parse
|
|
11
11
|
from .normalizer import NormalizedEvent, normalize
|
|
12
|
+
from .utils import sanitize_message
|
|
12
13
|
from .mappings import CISCO_ASA, CISCO_IOS, F5, FORTINET, LINUX, SOPHOS, VMWARE
|
|
13
14
|
from .cef import CEFEvent, build_cef
|
|
14
15
|
|
|
@@ -132,6 +133,11 @@ class StreamConverter:
|
|
|
132
133
|
inherits host, app, pid, PRI, and timestamp from the most recent
|
|
133
134
|
fully-parsed event and is tagged ``source_hint="continuation"``. One
|
|
134
135
|
CEF record is still emitted per input line.
|
|
136
|
+
|
|
137
|
+
Context is tracked per ``source`` (e.g. one entry per input file), so
|
|
138
|
+
interleaved streams — several files tailed at once — never inherit
|
|
139
|
+
another file's host or timestamp. Callers with a single stream can
|
|
140
|
+
ignore the parameter.
|
|
135
141
|
"""
|
|
136
142
|
|
|
137
143
|
def __init__(
|
|
@@ -146,12 +152,34 @@ class StreamConverter:
|
|
|
146
152
|
self.mapping = mapping
|
|
147
153
|
self.validate = validate
|
|
148
154
|
self.strict = strict
|
|
149
|
-
self.
|
|
150
|
-
|
|
151
|
-
def convert(self, line: str, *, now: Optional[datetime] = None) -> str:
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
+
self._contexts: dict[str, ParsedEvent] = {}
|
|
156
|
+
|
|
157
|
+
def convert(self, line: str, *, now: Optional[datetime] = None, source: str = "") -> str:
|
|
158
|
+
ctx = self._contexts.get(source)
|
|
159
|
+
is_continuation = line[:1] in ("\t", " ") and ctx is not None
|
|
160
|
+
try:
|
|
161
|
+
parsed = parse_syslog(line, mode=self.mode, now=now)
|
|
162
|
+
except ParserError:
|
|
163
|
+
if not is_continuation:
|
|
164
|
+
raise
|
|
165
|
+
# A forced --mode rejects headerless continuation lines; a
|
|
166
|
+
# continuation with context must inherit, not abort the run.
|
|
167
|
+
parsed = ParsedEvent(
|
|
168
|
+
pri=None,
|
|
169
|
+
facility=None,
|
|
170
|
+
severity=None,
|
|
171
|
+
ts=None,
|
|
172
|
+
ts_orig="",
|
|
173
|
+
host=None,
|
|
174
|
+
app=None,
|
|
175
|
+
pid=None,
|
|
176
|
+
msgid=None,
|
|
177
|
+
sd={},
|
|
178
|
+
msg=sanitize_message(line),
|
|
179
|
+
raw=line,
|
|
180
|
+
source_hint="unknown",
|
|
181
|
+
)
|
|
182
|
+
if is_continuation:
|
|
155
183
|
parsed.host = ctx.host
|
|
156
184
|
parsed.app = parsed.app or ctx.app
|
|
157
185
|
parsed.pid = parsed.pid or ctx.pid
|
|
@@ -166,7 +194,7 @@ class StreamConverter:
|
|
|
166
194
|
elif parsed.host is not None and parsed.source_hint != "unknown":
|
|
167
195
|
# Only a line whose host was actually parsed (not the
|
|
168
196
|
# guessed fallback) may become continuation context.
|
|
169
|
-
self.
|
|
197
|
+
self._contexts[source] = parsed
|
|
170
198
|
normalized = normalize_event(parsed)
|
|
171
199
|
return to_cef(normalized, self.mapping, validate=self.validate, strict=self.strict)
|
|
172
200
|
|
|
@@ -28,6 +28,14 @@ DEFAULT_MAPPING = {
|
|
|
28
28
|
},
|
|
29
29
|
"extensions": {
|
|
30
30
|
"msg": "%(msg)s",
|
|
31
|
+
# The syslog HOSTNAME identifies the device that GENERATED the
|
|
32
|
+
# event, so it is the source host (shost) — never the destination
|
|
33
|
+
# (dhost), which is reserved for network-specific logs whose
|
|
34
|
+
# payload names an actual destination. shost (max 1023) rather
|
|
35
|
+
# than dvchost (max 100): RFC 5424 hostnames may reach 255 chars,
|
|
36
|
+
# which would fail --strict against dvchost's dictionary limit.
|
|
37
|
+
# Empty values are dropped, so this is a no-op for a hostless line.
|
|
38
|
+
"shost": "%(host)s",
|
|
31
39
|
# Event time (epoch milliseconds) parsed from the line, or the
|
|
32
40
|
# processing time when the line carried no usable timestamp.
|
|
33
41
|
"rt": "%(rt)s",
|
|
@@ -15,16 +15,26 @@ from .api import StreamConverter, convert_line
|
|
|
15
15
|
logger = logging.getLogger(__name__)
|
|
16
16
|
|
|
17
17
|
|
|
18
|
-
def iter_lines_from_sources(paths: Iterable[Path] | None) -> Iterator[
|
|
18
|
+
def iter_lines_from_sources(paths: Iterable[Path] | None, *, tag_sources: bool = False) -> Iterator[Any]:
|
|
19
|
+
"""Yield lines from stdin or the given files.
|
|
20
|
+
|
|
21
|
+
With ``tag_sources`` each item is ``(source, line)`` so downstream
|
|
22
|
+
continuation handling can keep per-file context instead of letting
|
|
23
|
+
an indented line at the top of one file inherit the previous file's
|
|
24
|
+
host and timestamp.
|
|
25
|
+
"""
|
|
19
26
|
if not paths:
|
|
20
27
|
for line in sys.stdin:
|
|
21
|
-
|
|
28
|
+
line = line.rstrip("\n")
|
|
29
|
+
yield ("stdin", line) if tag_sources else line
|
|
22
30
|
return
|
|
23
31
|
|
|
24
32
|
for path in paths:
|
|
33
|
+
source = str(path)
|
|
25
34
|
with path.open("r", encoding="utf-8", errors="replace") as handle:
|
|
26
35
|
for line in handle:
|
|
27
|
-
|
|
36
|
+
line = line.rstrip("\n")
|
|
37
|
+
yield (source, line) if tag_sources else line
|
|
28
38
|
|
|
29
39
|
|
|
30
40
|
def tail_file(path: Path, *, follow: bool = False) -> Iterator[str]:
|
|
@@ -165,7 +175,8 @@ def follow_files(
|
|
|
165
175
|
*,
|
|
166
176
|
poll_interval: float = 0.5,
|
|
167
177
|
max_idle_polls: Optional[int] = None,
|
|
168
|
-
|
|
178
|
+
tag_sources: bool = False,
|
|
179
|
+
) -> Iterator[Any]:
|
|
169
180
|
"""Tail several files at once, round-robin, yielding lines as they appear.
|
|
170
181
|
|
|
171
182
|
Survives log rotation on every path independently: renamed-and-
|
|
@@ -175,6 +186,10 @@ def follow_files(
|
|
|
175
186
|
|
|
176
187
|
``max_idle_polls`` bounds how many consecutive empty polls are allowed
|
|
177
188
|
before the generator stops; ``None`` follows forever.
|
|
189
|
+
|
|
190
|
+
With ``tag_sources`` each item is ``(path, line)``: the round-robin
|
|
191
|
+
interleaves files, so continuation context downstream must be keyed
|
|
192
|
+
per file rather than shared across them.
|
|
178
193
|
"""
|
|
179
194
|
|
|
180
195
|
followers = [_Follower(path) for path in paths]
|
|
@@ -183,9 +198,10 @@ def follow_files(
|
|
|
183
198
|
while True:
|
|
184
199
|
any_line = False
|
|
185
200
|
for follower in followers:
|
|
201
|
+
source = str(follower.path)
|
|
186
202
|
for line in follower.read_lines():
|
|
187
203
|
any_line = True
|
|
188
|
-
yield line
|
|
204
|
+
yield (source, line) if tag_sources else line
|
|
189
205
|
# Rotation is checked for every follower on every pass —
|
|
190
206
|
# including busy ones — so a file renamed away while still
|
|
191
207
|
# being written cannot pin this follower to the stale
|
|
@@ -323,8 +339,16 @@ def _mp_load_patterns(pattern_files: list) -> None:
|
|
|
323
339
|
load_patterns(path)
|
|
324
340
|
|
|
325
341
|
|
|
342
|
+
def _split_item(item: Any) -> tuple:
|
|
343
|
+
# Line iterators may tag items as (source, line); untagged plain
|
|
344
|
+
# strings share the "" source.
|
|
345
|
+
if isinstance(item, tuple):
|
|
346
|
+
return item
|
|
347
|
+
return ("", item)
|
|
348
|
+
|
|
349
|
+
|
|
326
350
|
def process_lines(
|
|
327
|
-
lines: Iterable[
|
|
351
|
+
lines: Iterable[Any],
|
|
328
352
|
*,
|
|
329
353
|
mode: Optional[str],
|
|
330
354
|
mapping: Any,
|
|
@@ -338,15 +362,17 @@ def process_lines(
|
|
|
338
362
|
size = pool_size or max(1, cpu_count() - 1)
|
|
339
363
|
with Pool(size, initializer=_mp_load_patterns, initargs=(pattern_files or [],)) as pool:
|
|
340
364
|
worker = partial(_convert_line_mp, mode=mode, mapping=mapping, validate=validate, strict=strict)
|
|
341
|
-
for cef in pool.imap(worker, lines):
|
|
365
|
+
for cef in pool.imap(worker, (_split_item(item)[1] for item in lines)):
|
|
342
366
|
yield cef
|
|
343
367
|
else:
|
|
344
368
|
# Stateful conversion so whitespace-indented continuation lines
|
|
345
|
-
# inherit host/app/timestamp from the preceding event
|
|
369
|
+
# inherit host/app/timestamp from the preceding event, keyed by
|
|
370
|
+
# source so interleaved files never cross-inherit. Unavailable
|
|
346
371
|
# under --multiprocess, where workers cannot share line order.
|
|
347
372
|
converter = StreamConverter(mode=mode, mapping=mapping, validate=validate, strict=strict)
|
|
348
|
-
for
|
|
349
|
-
|
|
373
|
+
for item in lines:
|
|
374
|
+
source, line = _split_item(item)
|
|
375
|
+
yield converter.convert(line, source=source)
|
|
350
376
|
|
|
351
377
|
|
|
352
378
|
def _optional_path(value: str) -> Optional[Path]:
|
|
@@ -443,9 +469,9 @@ def main(argv: Optional[list[str]] = None) -> int:
|
|
|
443
469
|
lines = listen_lines(proto, host, port)
|
|
444
470
|
outputs = process_lines(lines, mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size, validate=args.validate or args.strict, strict=args.strict, pattern_files=args.patterns)
|
|
445
471
|
elif args.tail and args.paths:
|
|
446
|
-
outputs = process_lines(follow_files(args.paths), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size, validate=args.validate or args.strict, strict=args.strict, pattern_files=args.patterns)
|
|
472
|
+
outputs = process_lines(follow_files(args.paths, tag_sources=True), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size, validate=args.validate or args.strict, strict=args.strict, pattern_files=args.patterns)
|
|
447
473
|
else:
|
|
448
|
-
outputs = process_lines(iter_lines_from_sources(args.paths), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size, validate=args.validate or args.strict, strict=args.strict, pattern_files=args.patterns)
|
|
474
|
+
outputs = process_lines(iter_lines_from_sources(args.paths, tag_sources=True), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size, validate=args.validate or args.strict, strict=args.strict, pattern_files=args.patterns)
|
|
449
475
|
|
|
450
476
|
sender = None
|
|
451
477
|
if args.send:
|
|
@@ -58,6 +58,15 @@ def normalize(event: ParsedEvent) -> NormalizedEvent:
|
|
|
58
58
|
"rt": str(int(ts.timestamp() * 1000)),
|
|
59
59
|
}
|
|
60
60
|
|
|
61
|
+
# dvcpid is an integer field in the CEF dictionary, but RFC 5424
|
|
62
|
+
# PROCID (and kv pids) may be arbitrary strings ("worker-A"); expose
|
|
63
|
+
# a numeric-only alias so mappings can emit dvcpid without tripping
|
|
64
|
+
# strict validation. ASCII check: isdigit() alone accepts Unicode
|
|
65
|
+
# digits that int() consumers reject.
|
|
66
|
+
pid = str(event.pid) if event.pid is not None else ""
|
|
67
|
+
if pid.isascii() and pid.isdigit():
|
|
68
|
+
extras["dvcpid"] = pid
|
|
69
|
+
|
|
61
70
|
if event.source_hint in {"rfc3164", "rsyslog", "journald"} and event.host is None:
|
|
62
71
|
extras["host"] = "localhost"
|
|
63
72
|
|
|
@@ -728,10 +728,16 @@ def autodetect_and_parse(line: str, *, mode: Optional[str] = None, now: Optional
|
|
|
728
728
|
("rsyslog_file", lambda s: bool(RSYSLOG_FILE_RE.match(s))),
|
|
729
729
|
("journald_iso", lambda s: bool(JOURNALCTL_ISO_RE.match(s))),
|
|
730
730
|
("iso_syslog", lambda s: bool(ISO_SYSLOG_RE.match(s))),
|
|
731
|
-
|
|
731
|
+
# Header-based parsers run BEFORE kv: a normal RFC3164/journald
|
|
732
|
+
# line whose *message* carries key=value pairs (containerd,
|
|
733
|
+
# dockerd, NetworkManager, ...) must be claimed by its syslog
|
|
734
|
+
# parser, not the kv format. Genuine kv streams (Fortinet, Sophos)
|
|
735
|
+
# start with date=/device= and match none of these headers, so
|
|
736
|
+
# they still fall through to kv.
|
|
732
737
|
("cisco_seq", lambda s: bool(CISCO_SEQ_RE.match(s))),
|
|
733
738
|
("journald_short", lambda s: bool(JOURNALCTL_SHORT_RE.match(s))),
|
|
734
739
|
("rfc3164", lambda s: bool(RFC3164_RE.match(s))),
|
|
740
|
+
("kv", lambda s: "=" in s[:80]),
|
|
735
741
|
]
|
|
736
742
|
|
|
737
743
|
from . import custom
|
|
@@ -101,9 +101,9 @@ class TestContinuationLines:
|
|
|
101
101
|
)
|
|
102
102
|
cont = conv.convert('\t "dsAttrTypeStandard:OperatingSystemVersion"')
|
|
103
103
|
closing = conv.convert("\t)")
|
|
104
|
-
assert "
|
|
104
|
+
assert "shost=MacBookPro" in first
|
|
105
105
|
for cef in (cont, closing):
|
|
106
|
-
assert "
|
|
106
|
+
assert "shost=MacBookPro" in cef
|
|
107
107
|
assert "sproc=opendirectoryd" in cef
|
|
108
108
|
assert "dvcpid=69" in cef
|
|
109
109
|
# Continuation lines carry the parent's event time.
|
|
@@ -123,7 +123,7 @@ class TestContinuationLines:
|
|
|
123
123
|
cont = conv.convert("\tindented continuation")
|
|
124
124
|
# Context still comes from the last *parsed* event, not the
|
|
125
125
|
# freeform fallback line.
|
|
126
|
-
assert "
|
|
126
|
+
assert "shost=MacBookPro" in cont
|
|
127
127
|
|
|
128
128
|
def test_indented_message_preserved(self):
|
|
129
129
|
conv = StreamConverter()
|
|
@@ -140,3 +140,135 @@ class TestEventTimeInOutput:
|
|
|
140
140
|
datetime(2026, 7, 20, 3, 23, 5, tzinfo=timezone(timedelta(hours=2))).timestamp() * 1000
|
|
141
141
|
)
|
|
142
142
|
assert f"rt={expected_ms}" in cef
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class TestPerSourceContext:
|
|
146
|
+
"""Continuation context must not leak across input sources (files)."""
|
|
147
|
+
|
|
148
|
+
def test_continuation_does_not_inherit_across_sources(self):
|
|
149
|
+
conv = StreamConverter()
|
|
150
|
+
conv.convert("May 28 16:04:52 MacBookPro opendirectoryd[69]: parent", source="a.log")
|
|
151
|
+
cef = conv.convert("\tindented start of another file", source="b.log")
|
|
152
|
+
assert "shost=MacBookPro" not in cef
|
|
153
|
+
|
|
154
|
+
def test_each_source_keeps_its_own_context(self):
|
|
155
|
+
conv = StreamConverter()
|
|
156
|
+
conv.convert("May 28 16:04:52 host-a appa[1]: parent A", source="a.log")
|
|
157
|
+
conv.convert("May 28 16:04:53 host-b appb[2]: parent B", source="b.log")
|
|
158
|
+
# Interleaved continuations inherit from their own file.
|
|
159
|
+
cef_a = conv.convert("\tcontinuation for A", source="a.log")
|
|
160
|
+
cef_b = conv.convert("\tcontinuation for B", source="b.log")
|
|
161
|
+
assert "shost=host-a" in cef_a and "sproc=appa" in cef_a
|
|
162
|
+
assert "shost=host-b" in cef_b and "sproc=appb" in cef_b
|
|
163
|
+
|
|
164
|
+
def test_process_lines_tags_sources(self):
|
|
165
|
+
from syslogcef.cli import process_lines
|
|
166
|
+
|
|
167
|
+
items = [
|
|
168
|
+
("a.log", "May 28 16:04:52 host-a appa[1]: parent A"),
|
|
169
|
+
("b.log", "\tindented first line of file B"),
|
|
170
|
+
]
|
|
171
|
+
out = list(
|
|
172
|
+
process_lines(items, mode=None, mapping=None, use_multiprocessing=False, pool_size=None)
|
|
173
|
+
)
|
|
174
|
+
assert "shost=host-a" in out[0]
|
|
175
|
+
assert "shost=host-a" not in out[1]
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
class TestForcedModeContinuation:
|
|
179
|
+
"""A forced --mode must inherit continuation lines, not abort on them."""
|
|
180
|
+
|
|
181
|
+
def test_forced_mode_continuation_inherits(self):
|
|
182
|
+
conv = StreamConverter(mode="journald_short")
|
|
183
|
+
parent = conv.convert("May 28 16:04:52 MacBookPro opendirectoryd[69]: parent event")
|
|
184
|
+
cont = conv.convert("\t wrapped payload line")
|
|
185
|
+
assert "shost=MacBookPro" in parent
|
|
186
|
+
assert "shost=MacBookPro" in cont
|
|
187
|
+
assert "sproc=opendirectoryd" in cont
|
|
188
|
+
|
|
189
|
+
def test_forced_mode_still_rejects_non_continuations(self):
|
|
190
|
+
from syslogcef.parsers import ParserError
|
|
191
|
+
|
|
192
|
+
conv = StreamConverter(mode="journald_short")
|
|
193
|
+
with pytest.raises(ParserError):
|
|
194
|
+
conv.convert("not a journald line at all")
|
|
195
|
+
|
|
196
|
+
def test_forced_mode_indented_without_context_still_raises(self):
|
|
197
|
+
from syslogcef.parsers import ParserError
|
|
198
|
+
|
|
199
|
+
conv = StreamConverter(mode="journald_short")
|
|
200
|
+
with pytest.raises(ParserError):
|
|
201
|
+
conv.convert("\tindented but no preceding event")
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
class TestDvcpidNumericOnly:
|
|
205
|
+
"""dvcpid is an integer CEF field; non-numeric PROCIDs must not land there."""
|
|
206
|
+
|
|
207
|
+
def test_numeric_pid_emits_dvcpid(self):
|
|
208
|
+
from syslogcef import convert_line
|
|
209
|
+
from syslogcef.mappings import LINUX
|
|
210
|
+
|
|
211
|
+
cef = convert_line(
|
|
212
|
+
"<34>1 2026-07-20T03:23:05Z host1 myapp 4711 ID47 - message", mapping=LINUX
|
|
213
|
+
)
|
|
214
|
+
assert "dvcpid=4711" in cef
|
|
215
|
+
|
|
216
|
+
def test_nonnumeric_procid_omits_dvcpid(self):
|
|
217
|
+
from syslogcef import convert_line
|
|
218
|
+
from syslogcef.mappings import LINUX
|
|
219
|
+
|
|
220
|
+
cef = convert_line(
|
|
221
|
+
"<34>1 2026-07-20T03:23:05Z host1 myapp worker-A ID47 - message", mapping=LINUX
|
|
222
|
+
)
|
|
223
|
+
assert "dvcpid=" not in cef
|
|
224
|
+
|
|
225
|
+
def test_nonnumeric_procid_passes_strict_validation(self):
|
|
226
|
+
from syslogcef import convert_line
|
|
227
|
+
from syslogcef.mappings import LINUX
|
|
228
|
+
|
|
229
|
+
cef = convert_line(
|
|
230
|
+
"<34>1 2026-07-20T03:23:05Z host1 myapp worker-A ID47 - message",
|
|
231
|
+
mapping=LINUX,
|
|
232
|
+
strict=True,
|
|
233
|
+
)
|
|
234
|
+
assert cef.startswith("CEF:0|")
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
class TestSyslogHostIsSource:
|
|
238
|
+
"""The syslog HOSTNAME is the device that generated the event, so it
|
|
239
|
+
maps to shost (source host), never dhost (destination). shost, not
|
|
240
|
+
dvchost: RFC 5424 hostnames can exceed dvchost's 100-char limit."""
|
|
241
|
+
|
|
242
|
+
def test_linux_mapping_host_is_shost_not_dhost(self):
|
|
243
|
+
from syslogcef import convert_line
|
|
244
|
+
|
|
245
|
+
cef = convert_line(
|
|
246
|
+
"<13>Aug 10 13:39:25 clay-wks-linux-15 kernel: docker0: port 1 entered blocking state"
|
|
247
|
+
)
|
|
248
|
+
assert "shost=clay-wks-linux-15" in cef
|
|
249
|
+
assert "dhost=" not in cef
|
|
250
|
+
assert "dvchost=" not in cef
|
|
251
|
+
|
|
252
|
+
def test_default_mapping_surfaces_host_as_source(self):
|
|
253
|
+
# Exercise DEFAULT_MAPPING directly: a minimal custom mapping with
|
|
254
|
+
# no extensions falls through to the default extension set, so the
|
|
255
|
+
# default's shost is what surfaces (mapping=None would route via
|
|
256
|
+
# _guess_mapping to the bundled LINUX mapping instead).
|
|
257
|
+
from syslogcef import convert_line
|
|
258
|
+
|
|
259
|
+
cef = convert_line(
|
|
260
|
+
"<13>Aug 10 13:39:25 gw-01 weirdproc: payload zzz",
|
|
261
|
+
mapping={"deviceVendor": "Acme", "deviceProduct": "Thing"},
|
|
262
|
+
)
|
|
263
|
+
assert "|Acme|Thing|" in cef # confirms the custom (default-based) mapping ran
|
|
264
|
+
assert "shost=gw-01" in cef
|
|
265
|
+
assert "dhost=" not in cef
|
|
266
|
+
|
|
267
|
+
def test_long_hostname_passes_strict(self):
|
|
268
|
+
# A 200-char hostname (valid per RFC 5424, > dvchost's 100 limit)
|
|
269
|
+
# must not fail --strict — which is why host maps to shost (1023).
|
|
270
|
+
from syslogcef import convert_line
|
|
271
|
+
|
|
272
|
+
host = "h" * 200
|
|
273
|
+
cef = convert_line(f"<13>Aug 10 13:39:25 {host} app: msg", strict=True)
|
|
274
|
+
assert f"shost={host}" in cef
|
|
@@ -75,6 +75,25 @@ def test_pri_survives_sophos_kv_line():
|
|
|
75
75
|
assert fields(cef)[6] == "2"
|
|
76
76
|
|
|
77
77
|
|
|
78
|
+
def test_rfc3164_with_kv_message_body_is_not_claimed_by_kv():
|
|
79
|
+
# An ordinary RFC3164/journald-short line whose message carries many
|
|
80
|
+
# key=value pairs (containerd, dockerd, ...) must be parsed by its
|
|
81
|
+
# syslog parser — host/app extracted, header stripped — not misread
|
|
82
|
+
# as the kv format (regression for the detector ordering, #102).
|
|
83
|
+
line = (
|
|
84
|
+
"Aug 10 13:44:31 clay-wks-linux-15 containerd[3749511]: "
|
|
85
|
+
'time="2026-08-10T13:44:31.216507070Z" level=info '
|
|
86
|
+
'msg="connecting to shim f6f0" address="unix:///run/x" '
|
|
87
|
+
"namespace=moby protocol=ttrpc version=3"
|
|
88
|
+
)
|
|
89
|
+
ev = parse_syslog(line)
|
|
90
|
+
assert ev.source_hint in {"rfc3164", "journald"}
|
|
91
|
+
assert ev.host == "clay-wks-linux-15"
|
|
92
|
+
assert ev.app == "containerd"
|
|
93
|
+
assert ev.pid == "3749511"
|
|
94
|
+
assert ev.msg.startswith("time=")
|
|
95
|
+
|
|
96
|
+
|
|
78
97
|
def test_pri_survives_raw_fallback():
|
|
79
98
|
line = "<30>completely nonstandard payload without structure"
|
|
80
99
|
ev = parse_syslog(line)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|