syslog2cef 0.1.5__tar.gz → 0.1.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {syslog2cef-0.1.5/syslog2cef.egg-info → syslog2cef-0.1.6}/PKG-INFO +1 -1
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/pyproject.toml +1 -1
- {syslog2cef-0.1.5 → syslog2cef-0.1.6/syslog2cef.egg-info}/PKG-INFO +1 -1
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/syslog2cef.egg-info/SOURCES.txt +1 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/syslogcef/cef.py +18 -3
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/syslogcef/custom.py +6 -5
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/syslogcef/parsers.py +124 -12
- syslog2cef-0.1.6/tests/test_fuzz_grammar.py +293 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/tests/test_fuzz_properties.py +30 -31
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/LICENSE +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/README.md +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/setup.cfg +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/syslog2cef.egg-info/dependency_links.txt +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/syslog2cef.egg-info/entry_points.txt +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/syslog2cef.egg-info/requires.txt +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/syslog2cef.egg-info/top_level.txt +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/syslogcef/__init__.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/syslogcef/__main__.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/syslogcef/adaptive.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/syslogcef/api.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/syslogcef/cli.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/syslogcef/mappings/__init__.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/syslogcef/mappings/cisco_asa.json +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/syslogcef/mappings/cisco_ios.json +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/syslogcef/mappings/f5.json +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/syslogcef/mappings/fortinet.json +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/syslogcef/mappings/linux.json +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/syslogcef/mappings/sophos.json +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/syslogcef/mappings/vmware.json +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/syslogcef/net.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/syslogcef/normalizer.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/syslogcef/utils.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/syslogcef/validation.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/tests/test_adaptive.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/tests/test_cef.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/tests/test_cli.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/tests/test_custom_parsers.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/tests/test_net.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/tests/test_normalizer.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/tests/test_parsers.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/tests/test_parsers_extra.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/tests/test_pipeline.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/tests/test_real_world.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/tests/test_utils.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.1.6}/tests/test_validation.py +0 -0
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "syslog2cef"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.6"
|
|
8
8
|
description = "Convert RFC3164/RFC5424 syslog lines into ArcSight CEF"
|
|
9
9
|
authors = [{name = "Tamir Suliman", email = "allamiro@gmail.com"}]
|
|
10
10
|
readme = "README.md"
|
|
@@ -69,11 +69,26 @@ class MappingResolver:
|
|
|
69
69
|
severity_map = {**DEFAULT_MAPPING["severity_map"], **self.mapping.get("severity_map", {})}
|
|
70
70
|
value = fields.get("severity")
|
|
71
71
|
if value is None and "pri" in fields:
|
|
72
|
-
|
|
73
|
-
|
|
72
|
+
# "pri" may come from an untrusted kv pair (pri=zzz), not just
|
|
73
|
+
# a parsed <PRI> header; garbage must not abort the pipeline.
|
|
74
|
+
try:
|
|
75
|
+
_, value = divmod(int(fields.get("pri", 0)), 8)
|
|
76
|
+
except (TypeError, ValueError):
|
|
77
|
+
value = None
|
|
74
78
|
if value is None:
|
|
75
79
|
return "3"
|
|
76
|
-
|
|
80
|
+
mapped = str(severity_map.get(str(value), value))
|
|
81
|
+
# The CEF:0 header severity slot must be numeric 0-10; anything
|
|
82
|
+
# else (non-numeric kv severity, bad severity_map value) falls
|
|
83
|
+
# back to the default rather than corrupting the header. ASCII
|
|
84
|
+
# check required: str.isdigit() accepts Unicode digits like "²"
|
|
85
|
+
# that int() rejects.
|
|
86
|
+
# Length guard first: int() on a multi-thousand-digit string hits
|
|
87
|
+
# Python 3.11+'s integer-conversion limit and raises. Valid CEF
|
|
88
|
+
# severities are at most two characters ("10").
|
|
89
|
+
if not (mapped.isascii() and mapped.isdigit() and len(mapped) <= 2) or int(mapped) > 10:
|
|
90
|
+
return "3"
|
|
91
|
+
return mapped
|
|
77
92
|
|
|
78
93
|
def resolve_raw_extensions(self, fields: Mapping[str, Any]) -> Dict[str, str]:
|
|
79
94
|
raw = {}
|
|
@@ -140,11 +140,12 @@ def _year_with_rollover(parsed: datetime, ref: datetime) -> datetime:
|
|
|
140
140
|
|
|
141
141
|
candidate = with_year(ref.year)
|
|
142
142
|
if candidate is None:
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
143
|
+
# Feb 29: pick the occurrence closest in time, not the first
|
|
144
|
+
# leap year found in either direction.
|
|
145
|
+
candidates = [c for off in range(-4, 5) if off and (c := with_year(ref.year + off))]
|
|
146
|
+
if not candidates:
|
|
147
|
+
return parsed
|
|
148
|
+
return min(candidates, key=lambda c: abs(c - ref))
|
|
148
149
|
if candidate - ref > timedelta(days=180):
|
|
149
150
|
return with_year(ref.year - 1) or candidate
|
|
150
151
|
if ref - candidate > timedelta(days=180):
|
|
@@ -129,12 +129,30 @@ def _infer_timestamp(month: int, day: int, time_str: str, *, now: Optional[datet
|
|
|
129
129
|
if now is None:
|
|
130
130
|
now = datetime.now(timezone.utc)
|
|
131
131
|
hour, minute, second = map(int, time_str.split(":"))
|
|
132
|
-
|
|
132
|
+
|
|
133
|
+
def build(year: int) -> Optional[datetime]:
|
|
134
|
+
try:
|
|
135
|
+
return now.replace(
|
|
136
|
+
year=year, month=month, day=day,
|
|
137
|
+
hour=hour, minute=minute, second=second, microsecond=0,
|
|
138
|
+
)
|
|
139
|
+
except ValueError: # Feb 29 in a non-leap year
|
|
140
|
+
return None
|
|
141
|
+
|
|
142
|
+
candidate = build(now.year)
|
|
143
|
+
if candidate is None:
|
|
144
|
+
# A yearless Feb 29 line in a non-leap runtime year must not
|
|
145
|
+
# crash the parser: pick the occurrence closest in time (not
|
|
146
|
+
# merely the first leap year found in either direction).
|
|
147
|
+
candidates = [c for off in range(-4, 5) if off and (c := build(now.year + off))]
|
|
148
|
+
if not candidates:
|
|
149
|
+
raise ValueError(f"impossible calendar date: month={month} day={day}")
|
|
150
|
+
return min(candidates, key=lambda c: abs(c - now))
|
|
133
151
|
# Handle rollover into previous year.
|
|
134
152
|
if candidate - now > timedelta(days=180):
|
|
135
|
-
candidate =
|
|
153
|
+
candidate = build(now.year - 1) or candidate
|
|
136
154
|
elif now - candidate > timedelta(days=180):
|
|
137
|
-
candidate =
|
|
155
|
+
candidate = build(now.year + 1) or candidate
|
|
138
156
|
return candidate
|
|
139
157
|
|
|
140
158
|
|
|
@@ -232,12 +250,33 @@ def parse_rsyslog_json(line: str) -> ParsedEvent | None:
|
|
|
232
250
|
pri = data.get("pri") or data.get("syslogpriority")
|
|
233
251
|
pri = int(pri) if pri is not None else None
|
|
234
252
|
facility, severity = convert_pri(pri)
|
|
235
|
-
|
|
236
|
-
|
|
253
|
+
# Each alias is tried independently: an invalid value in an earlier
|
|
254
|
+
# key must not mask a valid one in a later key.
|
|
255
|
+
ts = None
|
|
256
|
+
timestamp = ""
|
|
257
|
+
for ts_key in ("timestamp", "timegenerated", "@timestamp", "timereported"):
|
|
258
|
+
raw_ts = data.get(ts_key)
|
|
259
|
+
if not raw_ts:
|
|
260
|
+
continue
|
|
261
|
+
if not timestamp:
|
|
262
|
+
timestamp = str(raw_ts) # first present value, for ts_orig
|
|
263
|
+
try:
|
|
264
|
+
ts = parse_iso8601(raw_ts)
|
|
265
|
+
timestamp = str(raw_ts)
|
|
266
|
+
break
|
|
267
|
+
except (ValueError, OverflowError, OSError):
|
|
268
|
+
continue
|
|
269
|
+
# JSON values may be any scalar (e.g. "message": 123); coerce to str
|
|
270
|
+
# so downstream sanitization/slicing never sees a non-string.
|
|
237
271
|
host = data.get("hostname") or data.get("host")
|
|
272
|
+
host = str(host) if host is not None else None
|
|
238
273
|
app = data.get("app-name") or data.get("programname")
|
|
274
|
+
app = str(app) if app is not None else None
|
|
275
|
+
if app is None and isinstance(data.get("syslogtag"), str):
|
|
276
|
+
# "nginx:" or "app[123]:" -> "nginx" / "app"
|
|
277
|
+
app = data["syslogtag"].rstrip(":").split("[")[0] or None
|
|
239
278
|
pid = data.get("procid") or data.get("pid")
|
|
240
|
-
msg = data.get("msg") or data.get("message") or ""
|
|
279
|
+
msg = str(data.get("msg") or data.get("message") or "")
|
|
241
280
|
sd = {k: v for k, v in data.items() if isinstance(k, str) and k.startswith("structured-data")}
|
|
242
281
|
return ParsedEvent(
|
|
243
282
|
pri=pri,
|
|
@@ -281,20 +320,59 @@ def parse_rsyslog_file(line: str) -> ParsedEvent | None:
|
|
|
281
320
|
)
|
|
282
321
|
|
|
283
322
|
|
|
323
|
+
# A JSON object must carry at least one journald-style key to be claimed
|
|
324
|
+
# by this parser; otherwise generic JSON (e.g. rsyslog's @timestamp/host
|
|
325
|
+
# shape) would be swallowed into an all-empty event instead of falling
|
|
326
|
+
# through to the rsyslog parser.
|
|
327
|
+
_JOURNALD_NATIVE_KEYS = (
|
|
328
|
+
"MESSAGE",
|
|
329
|
+
"__REALTIME_TIMESTAMP",
|
|
330
|
+
"_HOSTNAME",
|
|
331
|
+
"SYSLOG_IDENTIFIER",
|
|
332
|
+
"_PID",
|
|
333
|
+
"PRIORITY",
|
|
334
|
+
)
|
|
335
|
+
# Every lowercase alias the parser body reads must also gate, or explicit
|
|
336
|
+
# --mode journald_json rejects records it can parse.
|
|
337
|
+
_JOURNALD_ALIAS_KEYS = ("timestamp", "hostname", "app-name", "priority", "pid", "msg")
|
|
338
|
+
# Keys only rsyslog emits: a record matched purely via the lowercase
|
|
339
|
+
# aliases but carrying one of these belongs to parse_rsyslog_json, which
|
|
340
|
+
# reads fields (message, programname, ...) journald would drop.
|
|
341
|
+
_RSYSLOG_MARKER_KEYS = (
|
|
342
|
+
"@timestamp",
|
|
343
|
+
"message",
|
|
344
|
+
"programname",
|
|
345
|
+
"syslogtag",
|
|
346
|
+
"timegenerated",
|
|
347
|
+
"timereported",
|
|
348
|
+
"syslogpriority",
|
|
349
|
+
)
|
|
350
|
+
|
|
351
|
+
|
|
284
352
|
def parse_journalctl_json(line: str) -> ParsedEvent | None:
|
|
285
353
|
try:
|
|
286
354
|
data = json.loads(line)
|
|
287
355
|
except json.JSONDecodeError:
|
|
288
356
|
return None
|
|
357
|
+
if not isinstance(data, dict):
|
|
358
|
+
return None
|
|
359
|
+
if not any(key in data for key in _JOURNALD_NATIVE_KEYS):
|
|
360
|
+
if not any(key in data for key in _JOURNALD_ALIAS_KEYS):
|
|
361
|
+
return None
|
|
362
|
+
if any(key in data for key in _RSYSLOG_MARKER_KEYS):
|
|
363
|
+
return None
|
|
289
364
|
pri_raw = data.get("PRIORITY") or data.get("priority")
|
|
290
365
|
pri = int(pri_raw) if pri_raw is not None else None
|
|
291
366
|
facility, severity = convert_pri(pri)
|
|
292
367
|
timestamp = data.get("__REALTIME_TIMESTAMP") or data.get("timestamp")
|
|
293
368
|
ts = parse_iso8601(timestamp) if timestamp else None
|
|
369
|
+
# Coerce JSON scalars to str, as in parse_rsyslog_json.
|
|
294
370
|
host = data.get("_HOSTNAME") or data.get("hostname")
|
|
371
|
+
host = str(host) if host is not None else None
|
|
295
372
|
app = data.get("SYSLOG_IDENTIFIER") or data.get("app-name")
|
|
373
|
+
app = str(app) if app is not None else None
|
|
296
374
|
pid = data.get("_PID") or data.get("pid")
|
|
297
|
-
msg = data.get("MESSAGE") or data.get("msg") or ""
|
|
375
|
+
msg = str(data.get("MESSAGE") or data.get("msg") or "")
|
|
298
376
|
sd = JournalJSONMapping(data).to_structured_data()
|
|
299
377
|
return ParsedEvent(
|
|
300
378
|
pri=pri,
|
|
@@ -434,17 +512,46 @@ def parse_kv_stream(line: str) -> ParsedEvent | None:
|
|
|
434
512
|
if len(pairs) < 3:
|
|
435
513
|
return None
|
|
436
514
|
|
|
515
|
+
# Timestamp candidates in preference order, each falling through to
|
|
516
|
+
# the next when absent or unparseable (an invalid ts= must not mask
|
|
517
|
+
# a valid eventtime=). Overflow/OSError cover out-of-range epoch
|
|
518
|
+
# values, which must never abort explicit kv-mode processing.
|
|
519
|
+
# ts_orig records the candidate that actually parsed.
|
|
437
520
|
ts = None
|
|
521
|
+
ts_orig = ""
|
|
438
522
|
if pairs.get("date") and pairs.get("time"):
|
|
439
523
|
try:
|
|
440
524
|
ts = parse_iso8601(f"{pairs['date']}T{pairs['time']}")
|
|
441
|
-
|
|
525
|
+
ts_orig = f"{pairs['date']} {pairs['time']}"
|
|
526
|
+
except (ValueError, OverflowError, OSError):
|
|
442
527
|
ts = None
|
|
443
|
-
|
|
528
|
+
if ts is None:
|
|
529
|
+
# Each ISO alias tried independently: an invalid ts= must not
|
|
530
|
+
# mask a valid timestamp= or datetime=.
|
|
531
|
+
for iso_key in ("ts", "timestamp", "datetime"):
|
|
532
|
+
raw_iso = pairs.get(iso_key)
|
|
533
|
+
if not raw_iso:
|
|
534
|
+
continue
|
|
535
|
+
try:
|
|
536
|
+
ts = parse_iso8601(raw_iso)
|
|
537
|
+
ts_orig = raw_iso
|
|
538
|
+
break
|
|
539
|
+
except (ValueError, OverflowError, OSError):
|
|
540
|
+
continue
|
|
541
|
+
if ts is None and pairs.get("eventtime", "").isdigit():
|
|
444
542
|
try:
|
|
445
543
|
ts = parse_iso8601(pairs["eventtime"][:16])
|
|
446
|
-
|
|
544
|
+
ts_orig = pairs["eventtime"]
|
|
545
|
+
except (ValueError, OverflowError, OSError):
|
|
447
546
|
ts = None
|
|
547
|
+
if not ts_orig:
|
|
548
|
+
ts_orig = (
|
|
549
|
+
f"{pairs.get('date', '')} {pairs.get('time', '')}".strip()
|
|
550
|
+
or pairs.get("ts")
|
|
551
|
+
or pairs.get("timestamp")
|
|
552
|
+
or pairs.get("datetime")
|
|
553
|
+
or ""
|
|
554
|
+
)
|
|
448
555
|
if ts is not None and ts.tzinfo is None:
|
|
449
556
|
# Apply a numeric offset supplied in the record (tz="-0500");
|
|
450
557
|
# named zones (timezone="CEST") cannot be resolved portably.
|
|
@@ -456,7 +563,12 @@ def parse_kv_stream(line: str) -> ParsedEvent | None:
|
|
|
456
563
|
delta = timedelta(
|
|
457
564
|
hours=int(offset_match.group(2)), minutes=int(offset_match.group(3))
|
|
458
565
|
)
|
|
459
|
-
|
|
566
|
+
try:
|
|
567
|
+
ts = ts.replace(tzinfo=timezone(sign * delta))
|
|
568
|
+
except ValueError:
|
|
569
|
+
# tz=+24:00 and other out-of-range offsets: keep the
|
|
570
|
+
# naive timestamp rather than dropping the event.
|
|
571
|
+
pass
|
|
460
572
|
|
|
461
573
|
host = (
|
|
462
574
|
pairs.get("devname")
|
|
@@ -470,7 +582,7 @@ def parse_kv_stream(line: str) -> ParsedEvent | None:
|
|
|
470
582
|
facility=facility,
|
|
471
583
|
severity=severity,
|
|
472
584
|
ts=ts,
|
|
473
|
-
ts_orig=
|
|
585
|
+
ts_orig=ts_orig,
|
|
474
586
|
host=host,
|
|
475
587
|
app=pairs.get("type") or pairs.get("log_type"),
|
|
476
588
|
pid=None,
|
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
"""Grammar-based property tests: generated syslog with planted values.
|
|
2
|
+
|
|
3
|
+
Where test_fuzz_properties.py asserts the pipeline survives arbitrary
|
|
4
|
+
bytes, these tests *generate* structurally valid syslog in the dialects
|
|
5
|
+
seen in the wild — RFC3164, RFC5424, rsyslog JSON, kv streams, and
|
|
6
|
+
adaptive-only shapes — across payload lengths from empty to multi-KB,
|
|
7
|
+
with known hosts, apps, IPs, and timestamps planted in each line. They
|
|
8
|
+
then assert the pipeline actually detects the format and maps the
|
|
9
|
+
planted fields: host to host, IP to src/dst, time to the parsed
|
|
10
|
+
timestamp — the adaptivity guarantees, not just crash-freedom.
|
|
11
|
+
|
|
12
|
+
Requires hypothesis (skipped where absent, e.g. RPM %check).
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
from datetime import datetime, timezone
|
|
20
|
+
|
|
21
|
+
import pytest
|
|
22
|
+
|
|
23
|
+
hypothesis = pytest.importorskip("hypothesis")
|
|
24
|
+
|
|
25
|
+
from hypothesis import given, settings, strategies as st # noqa: E402
|
|
26
|
+
|
|
27
|
+
from syslogcef import convert_line, normalize_event, parse_syslog # noqa: E402
|
|
28
|
+
|
|
29
|
+
# Profiles are registered centrally in conftest.py; here we only load.
|
|
30
|
+
settings.load_profile(os.environ.get("SYSLOGCEF_HYPOTHESIS_PROFILE", "ci"))
|
|
31
|
+
|
|
32
|
+
hostnames = st.from_regex(r"[a-z][a-z0-9\-]{0,14}[a-z0-9]", fullmatch=True)
|
|
33
|
+
appnames = st.from_regex(r"[A-Za-z][A-Za-z0-9_]{0,11}", fullmatch=True)
|
|
34
|
+
ips = st.ip_addresses(v=4).map(str)
|
|
35
|
+
pids = st.integers(min_value=1, max_value=99999)
|
|
36
|
+
timestamps = st.datetimes(
|
|
37
|
+
min_value=datetime(2021, 1, 10),
|
|
38
|
+
max_value=datetime(2029, 12, 28),
|
|
39
|
+
timezones=st.just(timezone.utc),
|
|
40
|
+
).map(lambda d: d.replace(microsecond=0))
|
|
41
|
+
|
|
42
|
+
# Payloads from empty to ~3KB. '=' is excluded because an early '=' makes
|
|
43
|
+
# the kv detector claim the line (by design); pipes/backslashes stay in —
|
|
44
|
+
# the escaper must handle them.
|
|
45
|
+
payloads = st.text(
|
|
46
|
+
alphabet=st.characters(
|
|
47
|
+
blacklist_characters="\r\n=\"[]{}", blacklist_categories=("Cs",)
|
|
48
|
+
),
|
|
49
|
+
max_size=3000,
|
|
50
|
+
).map(lambda s: " ".join(s.split()))
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@given(host=hostnames, app=appnames, pid=pids, ts=timestamps, payload=payloads)
|
|
54
|
+
def test_rfc3164_fields_detected(host, app, pid, ts, payload):
|
|
55
|
+
line = f"<134>{ts.strftime('%b %d %H:%M:%S')} {host} {app}[{pid}]: {payload}"
|
|
56
|
+
|
|
57
|
+
# Anchor year inference to the generated timestamp so a generated
|
|
58
|
+
# Feb 29 resolves against its own (leap) year, not the wall clock.
|
|
59
|
+
event = parse_syslog(line, now=ts)
|
|
60
|
+
|
|
61
|
+
assert event.host == host
|
|
62
|
+
assert event.app == app
|
|
63
|
+
assert event.pid == str(pid)
|
|
64
|
+
assert event.ts is not None
|
|
65
|
+
assert (event.ts.month, event.ts.day, event.ts.hour, event.ts.minute, event.ts.second) == (
|
|
66
|
+
ts.month, ts.day, ts.hour, ts.minute, ts.second,
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@given(host=hostnames, app=appnames, pid=pids, ts=timestamps, payload=payloads)
|
|
71
|
+
def test_rfc5424_fields_detected(host, app, pid, ts, payload):
|
|
72
|
+
line = f"<134>1 {ts.strftime('%Y-%m-%dT%H:%M:%S')}Z {host} {app} {pid} MSGID - {payload}"
|
|
73
|
+
|
|
74
|
+
event = parse_syslog(line)
|
|
75
|
+
|
|
76
|
+
assert event.host == host
|
|
77
|
+
assert event.app == app
|
|
78
|
+
assert event.pid == str(pid)
|
|
79
|
+
assert event.ts == ts
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@given(host=hostnames, app=appnames, ts=timestamps, payload=payloads)
|
|
83
|
+
def test_rsyslog_json_fields_detected(host, app, ts, payload):
|
|
84
|
+
line = json.dumps(
|
|
85
|
+
{
|
|
86
|
+
"@timestamp": ts.strftime("%Y-%m-%dT%H:%M:%S+00:00"),
|
|
87
|
+
"host": host,
|
|
88
|
+
"severity": "info",
|
|
89
|
+
"facility": "daemon",
|
|
90
|
+
"syslogtag": f"{app}:",
|
|
91
|
+
"message": payload,
|
|
92
|
+
}
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
event = parse_syslog(line)
|
|
96
|
+
|
|
97
|
+
assert event.host == host
|
|
98
|
+
assert event.msg == payload
|
|
99
|
+
assert event.app == app # from syslogtag, colon stripped
|
|
100
|
+
assert event.ts == ts # from @timestamp
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@given(src=ips, dst=ips, ts=timestamps, user=appnames, payload=payloads)
|
|
104
|
+
def test_kv_stream_ips_and_time_mapped(src, dst, ts, user, payload):
|
|
105
|
+
line = (
|
|
106
|
+
f"ts={ts.strftime('%Y-%m-%dT%H:%M:%S')}Z level=info "
|
|
107
|
+
f'src={src} dst={dst} user={user} msg="{payload[:200]}"'
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
event = parse_syslog(line)
|
|
111
|
+
normalized = normalize_event(event)
|
|
112
|
+
|
|
113
|
+
# Planted IPs must surface as the kv fields mappings consume
|
|
114
|
+
# (ip -> src/dst), and the planted time must be the parsed timestamp.
|
|
115
|
+
assert normalized.kv.get("src") == src
|
|
116
|
+
assert normalized.kv.get("dst") == dst
|
|
117
|
+
assert normalized.kv.get("user") == user
|
|
118
|
+
assert event.ts == ts
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@given(src=ips, dst=ips, ts=timestamps)
|
|
122
|
+
def test_asa_ips_reach_cef_extensions(src, dst, ts):
|
|
123
|
+
line = (
|
|
124
|
+
f"<166>{ts.strftime('%b %d %H:%M:%S')} fw01 "
|
|
125
|
+
f"%ASA-6-302013: Built inbound TCP connection src={src} dst={dst}"
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
cef = convert_line(line)
|
|
129
|
+
|
|
130
|
+
# Device lines must map planted IPs into the CEF extension fields.
|
|
131
|
+
assert f"src={src}" in cef
|
|
132
|
+
assert f"dst={dst}" in cef
|
|
133
|
+
assert "|Cisco|ASA|" in cef
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def test_journald_lowercase_only_record_parses_in_explicit_mode():
|
|
137
|
+
# The journald-key gate must include every lowercase alias the parser
|
|
138
|
+
# body reads, or --mode journald_json rejects records it can parse.
|
|
139
|
+
event = parse_syslog('{"priority": "4", "pid": "1", "msg": "hello"}', mode="journald_json")
|
|
140
|
+
|
|
141
|
+
assert event.msg == "hello"
|
|
142
|
+
assert event.pid == "1"
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def test_kv_invalid_iso_falls_back_to_eventtime():
|
|
146
|
+
line = "eventtime=1770000000 ts=not-a-date src=1.2.3.4 dst=5.6.7.8 action=allow"
|
|
147
|
+
|
|
148
|
+
event = parse_syslog(line)
|
|
149
|
+
|
|
150
|
+
assert event.source_hint == "kv"
|
|
151
|
+
assert event.ts is not None # an invalid ts= must not mask eventtime=
|
|
152
|
+
assert event.ts_orig == "1770000000" # records the candidate that parsed
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def test_kv_out_of_range_epoch_keeps_event_in_explicit_mode():
|
|
156
|
+
line = "ts=99999999999999999999 src=1.2.3.4 dst=5.6.7.8 action=allow"
|
|
157
|
+
|
|
158
|
+
event = parse_syslog(line, mode="kv")
|
|
159
|
+
|
|
160
|
+
assert event.ts is None # OverflowError contained, event preserved
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def test_rsyslog_invalid_timestamp_alias_falls_through():
|
|
164
|
+
line = '{"timestamp": "bad", "@timestamp": "2026-08-11T07:00:00Z", "message": "m", "host": "h1"}'
|
|
165
|
+
|
|
166
|
+
event = parse_syslog(line)
|
|
167
|
+
|
|
168
|
+
assert event.source_hint == "rsyslog"
|
|
169
|
+
assert event.ts == datetime(2026, 8, 11, 7, 0, 0, tzinfo=timezone.utc)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def test_kv_invalid_ts_alias_falls_through_to_timestamp():
|
|
173
|
+
line = "ts=bad timestamp=2026-08-11T07:00:00Z src=1.2.3.4 dst=5.6.7.8 action=allow"
|
|
174
|
+
|
|
175
|
+
event = parse_syslog(line)
|
|
176
|
+
|
|
177
|
+
assert event.ts == datetime(2026, 8, 11, 7, 0, 0, tzinfo=timezone.utc)
|
|
178
|
+
assert event.ts_orig == "2026-08-11T07:00:00Z"
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def test_yearless_feb29_rfc3164_survives_non_leap_wall_clock():
|
|
182
|
+
# A real device can emit "Feb 29" while the converting host's clock
|
|
183
|
+
# is in a non-leap year; the parser must not crash.
|
|
184
|
+
ref = datetime(2026, 3, 1, tzinfo=timezone.utc) # 2026 is not a leap year
|
|
185
|
+
|
|
186
|
+
event = parse_syslog("<134>Feb 29 12:00:00 fw01 app[1]: leap traffic", now=ref)
|
|
187
|
+
|
|
188
|
+
assert event.source_hint == "rfc3164"
|
|
189
|
+
assert event.ts is not None
|
|
190
|
+
assert (event.ts.month, event.ts.day) == (2, 29)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def test_non_string_json_message_scalar_survives():
|
|
194
|
+
# JSON allows any scalar for message-ish keys; a non-string must be
|
|
195
|
+
# coerced, not crash sanitization downstream.
|
|
196
|
+
cef = convert_line('{"timestamp": "2026-08-11T07:00:00Z", "message": 123, "host": "h1"}')
|
|
197
|
+
|
|
198
|
+
assert cef.startswith("CEF:0|")
|
|
199
|
+
assert "msg=123" in cef
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def test_huge_digit_severity_cannot_crash_resolution():
|
|
203
|
+
# 5000 digits pass isascii/isdigit but int() hits Python 3.11+'s
|
|
204
|
+
# conversion limit; the length guard must trigger the fallback.
|
|
205
|
+
cef = convert_line("severity=" + "9" * 5000 + " src=1.2.3.4 dst=2.2.2.2 action=x")
|
|
206
|
+
|
|
207
|
+
severity = cef.split("|")[6]
|
|
208
|
+
assert severity.isdigit() and 0 <= int(severity) <= 10
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def test_unicode_digit_severity_cannot_crash_resolution():
|
|
212
|
+
# str.isdigit() is True for "²" but int("²") raises; the severity
|
|
213
|
+
# resolver must fall back, not propagate.
|
|
214
|
+
cef = convert_line("severity=² src=1.2.3.4 dst=2.2.2.2 action=x")
|
|
215
|
+
|
|
216
|
+
severity = cef.split("|")[6]
|
|
217
|
+
assert severity.isascii() and severity.isdigit() and 0 <= int(severity) <= 10
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def test_feb29_resolves_to_nearest_leap_occurrence():
|
|
221
|
+
from syslogcef.parsers import _infer_timestamp
|
|
222
|
+
|
|
223
|
+
# 2026-01-01: 2024-02-29 (~22 months back) is nearer than 2028-02-29
|
|
224
|
+
# (~26 months ahead); a direction-biased search would pick 2028.
|
|
225
|
+
ref = datetime(2026, 1, 1, tzinfo=timezone.utc)
|
|
226
|
+
|
|
227
|
+
ts = _infer_timestamp(2, 29, "12:00:00", now=ref)
|
|
228
|
+
|
|
229
|
+
assert ts.year == 2024
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def test_kv_out_of_range_tz_offset_keeps_event():
|
|
233
|
+
# tz=+24:00 is not a valid UTC offset; it must be ignored, not raise
|
|
234
|
+
# (reachable in explicit kv mode where no detector guard applies).
|
|
235
|
+
event = parse_syslog(
|
|
236
|
+
"ts=2026-08-11T07:00:00 tz=+24:00 src=1.2.3.4 dst=2.2.2.2 action=x", mode="kv"
|
|
237
|
+
)
|
|
238
|
+
|
|
239
|
+
assert event.ts is not None # timestamp kept, bogus offset dropped
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def test_non_numeric_pri_kv_pair_cannot_crash_severity_resolution():
|
|
243
|
+
# Found by ClusterFuzzLite (fuzz_structured): a kv line supplying
|
|
244
|
+
# pri=<garbage> reached int() in _resolve_severity and escaped
|
|
245
|
+
# convert_line, violating the never-raise contract.
|
|
246
|
+
cef = convert_line("pri=zzz src=1.2.3.4 dst=2.2.2.2 action=x")
|
|
247
|
+
|
|
248
|
+
severity = cef.split("|")[6]
|
|
249
|
+
assert severity.isdigit() and 0 <= int(severity) <= 10
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def test_rsyslog_lowercase_aliases_with_rsyslog_markers_route_to_rsyslog():
|
|
253
|
+
line = (
|
|
254
|
+
'{"timestamp": "2026-08-11T07:00:00Z", "hostname": "web7",'
|
|
255
|
+
' "programname": "nginx", "message": "GET / 200"}'
|
|
256
|
+
)
|
|
257
|
+
|
|
258
|
+
event = parse_syslog(line)
|
|
259
|
+
|
|
260
|
+
assert event.source_hint == "rsyslog"
|
|
261
|
+
assert event.host == "web7"
|
|
262
|
+
assert event.app == "nginx"
|
|
263
|
+
assert event.msg == "GET / 200"
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
# Lowercase words only: no digits or colons that could read as a second
|
|
267
|
+
# timestamp, so the planted one is unambiguous and the adaptive
|
|
268
|
+
# assertions below can be unconditional.
|
|
269
|
+
adaptive_payloads = st.text(
|
|
270
|
+
alphabet="abcdefghijklmnopqrstuvwxyz ", max_size=200
|
|
271
|
+
).map(lambda s: " ".join(s.split()))
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
@given(
|
|
275
|
+
punct=st.sampled_from(["#", "@", "~", "^", "!", ";"]),
|
|
276
|
+
reps=st.integers(min_value=1, max_value=4),
|
|
277
|
+
host=hostnames,
|
|
278
|
+
ts=timestamps,
|
|
279
|
+
payload=adaptive_payloads,
|
|
280
|
+
)
|
|
281
|
+
def test_adaptive_engine_finds_time_in_unknown_shapes(punct, reps, host, ts, payload):
|
|
282
|
+
# A prefix no built-in parser accepts: only the adaptive engine can
|
|
283
|
+
# take this, and it MUST locate the unambiguous planted timestamp —
|
|
284
|
+
# falling back to raw would be exactly the regression this guards.
|
|
285
|
+
line = f"{punct * reps} {ts.strftime('%b %d %H:%M:%S')} {host} {payload}"
|
|
286
|
+
|
|
287
|
+
event = parse_syslog(line, now=ts)
|
|
288
|
+
|
|
289
|
+
assert event.source_hint == "adaptive"
|
|
290
|
+
assert event.ts is not None
|
|
291
|
+
assert (event.ts.month, event.ts.day, event.ts.hour, event.ts.minute, event.ts.second) == (
|
|
292
|
+
ts.month, ts.day, ts.hour, ts.minute, ts.second,
|
|
293
|
+
)
|
|
@@ -26,29 +26,20 @@ from __future__ import annotations
|
|
|
26
26
|
import os
|
|
27
27
|
import re
|
|
28
28
|
import time
|
|
29
|
-
from datetime import timedelta
|
|
30
29
|
|
|
31
30
|
import pytest
|
|
32
31
|
|
|
33
32
|
hypothesis = pytest.importorskip("hypothesis")
|
|
34
33
|
|
|
35
|
-
from hypothesis import
|
|
34
|
+
from hypothesis import given, settings, strategies as st # noqa: E402
|
|
36
35
|
|
|
37
36
|
from syslogcef import convert_line # noqa: E402
|
|
38
37
|
from syslogcef import adaptive # noqa: E402
|
|
39
38
|
from syslogcef.validation import CEFValidationError # noqa: E402
|
|
40
39
|
|
|
41
|
-
#
|
|
42
|
-
# regression
|
|
43
|
-
#
|
|
44
|
-
# in-test 1s assertion below stays the primary, stricter budget.
|
|
45
|
-
settings.register_profile(
|
|
46
|
-
"ci",
|
|
47
|
-
max_examples=200,
|
|
48
|
-
deadline=timedelta(seconds=2),
|
|
49
|
-
suppress_health_check=[HealthCheck.too_slow],
|
|
50
|
-
)
|
|
51
|
-
settings.register_profile("deep", max_examples=10_000, deadline=timedelta(seconds=2))
|
|
40
|
+
# Profiles are registered centrally in conftest.py (a 2s deadline so a
|
|
41
|
+
# slow-but-passing regression fails fast in the deep run instead of
|
|
42
|
+
# draining the job budget); here we only load.
|
|
52
43
|
settings.load_profile(os.environ.get("SYSLOGCEF_HYPOTHESIS_PROFILE", "ci"))
|
|
53
44
|
|
|
54
45
|
# Real-world seed lines covering every parser family; mutations of these
|
|
@@ -93,15 +84,21 @@ def split_unescaped_pipes(record: str) -> list:
|
|
|
93
84
|
|
|
94
85
|
def assert_valid_cef(cef: str) -> None:
|
|
95
86
|
parts = split_unescaped_pipes(cef)
|
|
96
|
-
#
|
|
97
|
-
#
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
#
|
|
101
|
-
# the severity slot must be numeric
|
|
102
|
-
# there and fails.
|
|
87
|
+
# The renderer escapes pipes in header AND extension values, so a
|
|
88
|
+
# correct record has exactly 8 parts (7 header + 1 extension chunk);
|
|
89
|
+
# any extra unescaped pipe anywhere is an escaping regression.
|
|
90
|
+
assert len(parts) == 8, f"{len(parts)} pipe-separated parts, expected 8"
|
|
91
|
+
# Positional checks so a shifted header cannot hide: the version slot
|
|
92
|
+
# must be exactly CEF:0 and the severity slot must be numeric.
|
|
103
93
|
assert parts[0] == "CEF:0", f"bad version field: {parts[0]!r}"
|
|
104
94
|
assert re.fullmatch(r"\d{1,2}", parts[6]), f"non-numeric severity: {parts[6]!r}"
|
|
95
|
+
# Extension-boundary check: the chunk after the severity slot must be
|
|
96
|
+
# empty or start like an extension (key=). A header shifted while
|
|
97
|
+
# keeping 8 parts leaves the real severity here, failing the shape.
|
|
98
|
+
if parts[7]:
|
|
99
|
+
assert re.match(r"[A-Za-z0-9_.]+=", parts[7]), (
|
|
100
|
+
f"extension does not start at the expected boundary: {parts[7]!r}"
|
|
101
|
+
)
|
|
105
102
|
for field in parts[:7]:
|
|
106
103
|
assert "\r" not in field and "\n" not in field, "CR/LF in CEF header"
|
|
107
104
|
|
|
@@ -140,15 +137,17 @@ def test_validate_and_strict_never_crash(line):
|
|
|
140
137
|
|
|
141
138
|
|
|
142
139
|
def test_adaptive_cache_stays_bounded():
|
|
140
|
+
from itertools import product
|
|
141
|
+
|
|
143
142
|
adaptive.clear_cache()
|
|
144
|
-
#
|
|
145
|
-
#
|
|
146
|
-
#
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
convert_line(f"{
|
|
151
|
-
#
|
|
152
|
-
|
|
153
|
-
#
|
|
154
|
-
assert adaptive.cache_size()
|
|
143
|
+
# The shape signature keeps punctuation verbatim (letters -> A,
|
|
144
|
+
# digits -> 9), so a punctuation-product first token yields 343
|
|
145
|
+
# distinct signatures — beyond the 256-entry cap — with a
|
|
146
|
+
# recognizable timestamp so every line reaches the adaptive engine.
|
|
147
|
+
puncts = ["#", "@", "~", "^", "!", ";", ":"]
|
|
148
|
+
for p1, p2, p3 in product(puncts, repeat=3):
|
|
149
|
+
convert_line(f"{p1}x{p2}7{p3} Aug 11 07:00:00 host9 payload event")
|
|
150
|
+
# The cap must actually be reached (eviction exercised, not merely
|
|
151
|
+
# never triggered) and held exactly — more than 343 entries or an
|
|
152
|
+
# unbounded cache would exceed it.
|
|
153
|
+
assert adaptive.cache_size() == adaptive._CACHE_MAX
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|