syslog2cef 0.1.2__tar.gz → 0.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {syslog2cef-0.1.2/syslog2cef.egg-info → syslog2cef-0.1.3}/PKG-INFO +3 -1
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/pyproject.toml +4 -1
- {syslog2cef-0.1.2 → syslog2cef-0.1.3/syslog2cef.egg-info}/PKG-INFO +3 -1
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslog2cef.egg-info/SOURCES.txt +7 -1
- syslog2cef-0.1.3/syslog2cef.egg-info/requires.txt +6 -0
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/adaptive.py +28 -11
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/api.py +16 -5
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/cef.py +31 -7
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/cli.py +54 -15
- syslog2cef-0.1.3/syslogcef/net.py +270 -0
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/parsers.py +16 -3
- syslog2cef-0.1.3/syslogcef/validation.py +164 -0
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/tests/test_adaptive.py +54 -0
- syslog2cef-0.1.3/tests/test_net.py +180 -0
- syslog2cef-0.1.3/tests/test_validation.py +66 -0
- syslog2cef-0.1.2/syslog2cef.egg-info/requires.txt +0 -3
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/LICENSE +0 -0
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/README.md +0 -0
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/setup.cfg +0 -0
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslog2cef.egg-info/dependency_links.txt +0 -0
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslog2cef.egg-info/entry_points.txt +0 -0
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslog2cef.egg-info/top_level.txt +0 -0
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/__init__.py +0 -0
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/__main__.py +0 -0
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/mappings/__init__.py +0 -0
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/mappings/cisco_asa.json +0 -0
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/mappings/cisco_ios.json +0 -0
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/mappings/f5.json +0 -0
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/mappings/fortinet.json +0 -0
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/mappings/linux.json +0 -0
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/mappings/sophos.json +0 -0
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/mappings/vmware.json +0 -0
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/normalizer.py +0 -0
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/utils.py +0 -0
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/tests/test_cef.py +0 -0
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/tests/test_cli.py +0 -0
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/tests/test_normalizer.py +0 -0
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/tests/test_parsers.py +0 -0
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/tests/test_parsers_extra.py +0 -0
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/tests/test_pipeline.py +0 -0
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/tests/test_real_world.py +0 -0
- {syslog2cef-0.1.2 → syslog2cef-0.1.3}/tests/test_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: syslog2cef
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.3
|
|
4
4
|
Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
|
|
5
5
|
Author-email: Tamir Suliman <allamiro@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/allamiro/syslogcef
|
|
@@ -15,6 +15,8 @@ Description-Content-Type: text/markdown
|
|
|
15
15
|
License-File: LICENSE
|
|
16
16
|
Provides-Extra: test
|
|
17
17
|
Requires-Dist: pytest>=7; extra == "test"
|
|
18
|
+
Provides-Extra: kafka
|
|
19
|
+
Requires-Dist: kafka-python>=2.0; extra == "kafka"
|
|
18
20
|
Dynamic: license-file
|
|
19
21
|
|
|
20
22
|
<p align="center">
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "syslog2cef"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.3"
|
|
8
8
|
description = "Convert RFC3164/RFC5424 syslog lines into ArcSight CEF"
|
|
9
9
|
authors = [{name = "Tamir Suliman", email = "allamiro@gmail.com"}]
|
|
10
10
|
readme = "README.md"
|
|
@@ -26,6 +26,9 @@ Issues = "https://github.com/allamiro/syslogcef/issues"
|
|
|
26
26
|
test = [
|
|
27
27
|
"pytest>=7",
|
|
28
28
|
]
|
|
29
|
+
kafka = [
|
|
30
|
+
"kafka-python>=2.0",
|
|
31
|
+
]
|
|
29
32
|
|
|
30
33
|
[project.scripts]
|
|
31
34
|
syslogcef = "syslogcef.cli:main"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: syslog2cef
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.3
|
|
4
4
|
Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
|
|
5
5
|
Author-email: Tamir Suliman <allamiro@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/allamiro/syslogcef
|
|
@@ -15,6 +15,8 @@ Description-Content-Type: text/markdown
|
|
|
15
15
|
License-File: LICENSE
|
|
16
16
|
Provides-Extra: test
|
|
17
17
|
Requires-Dist: pytest>=7; extra == "test"
|
|
18
|
+
Provides-Extra: kafka
|
|
19
|
+
Requires-Dist: kafka-python>=2.0; extra == "kafka"
|
|
18
20
|
Dynamic: license-file
|
|
19
21
|
|
|
20
22
|
<p align="center">
|
|
@@ -7,9 +7,11 @@ pyproject.toml
|
|
|
7
7
|
./syslogcef/api.py
|
|
8
8
|
./syslogcef/cef.py
|
|
9
9
|
./syslogcef/cli.py
|
|
10
|
+
./syslogcef/net.py
|
|
10
11
|
./syslogcef/normalizer.py
|
|
11
12
|
./syslogcef/parsers.py
|
|
12
13
|
./syslogcef/utils.py
|
|
14
|
+
./syslogcef/validation.py
|
|
13
15
|
./syslogcef/mappings/__init__.py
|
|
14
16
|
./syslogcef/mappings/cisco_asa.json
|
|
15
17
|
./syslogcef/mappings/cisco_ios.json
|
|
@@ -30,9 +32,11 @@ syslogcef/adaptive.py
|
|
|
30
32
|
syslogcef/api.py
|
|
31
33
|
syslogcef/cef.py
|
|
32
34
|
syslogcef/cli.py
|
|
35
|
+
syslogcef/net.py
|
|
33
36
|
syslogcef/normalizer.py
|
|
34
37
|
syslogcef/parsers.py
|
|
35
38
|
syslogcef/utils.py
|
|
39
|
+
syslogcef/validation.py
|
|
36
40
|
syslogcef/mappings/__init__.py
|
|
37
41
|
syslogcef/mappings/cisco_asa.json
|
|
38
42
|
syslogcef/mappings/cisco_ios.json
|
|
@@ -44,9 +48,11 @@ syslogcef/mappings/vmware.json
|
|
|
44
48
|
tests/test_adaptive.py
|
|
45
49
|
tests/test_cef.py
|
|
46
50
|
tests/test_cli.py
|
|
51
|
+
tests/test_net.py
|
|
47
52
|
tests/test_normalizer.py
|
|
48
53
|
tests/test_parsers.py
|
|
49
54
|
tests/test_parsers_extra.py
|
|
50
55
|
tests/test_pipeline.py
|
|
51
56
|
tests/test_real_world.py
|
|
52
|
-
tests/test_utils.py
|
|
57
|
+
tests/test_utils.py
|
|
58
|
+
tests/test_validation.py
|
|
@@ -31,9 +31,14 @@ def _epoch(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
|
|
|
31
31
|
def _mon_day(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
|
|
32
32
|
month = month_abbr_to_int(match.group("month"))
|
|
33
33
|
day = int(match.group("day"))
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
34
|
+
if match.groupdict().get("year"):
|
|
35
|
+
hour, minute, second = map(int, match.group("hms").split(":"))
|
|
36
|
+
return datetime(int(match.group("year")), month, day, hour, minute, second, tzinfo=timezone.utc)
|
|
37
|
+
# No year: use the same rollover inference as the dedicated parsers so a
|
|
38
|
+
# December event parsed in January lands in the previous year.
|
|
39
|
+
from .parsers import _infer_timestamp
|
|
40
|
+
|
|
41
|
+
return _infer_timestamp(month, day, match.group("hms"), now=now)
|
|
37
42
|
|
|
38
43
|
|
|
39
44
|
def _ymd_slash(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
|
|
@@ -62,6 +67,16 @@ TIMESTAMP_LIB: Tuple[Tuple[str, str, Callable], ...] = (
|
|
|
62
67
|
HOST_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
|
|
63
68
|
_SKIP_HOST_TOKENS = {"error", "warning", "info", "debug", "notice", "kernel"}
|
|
64
69
|
|
|
70
|
+
|
|
71
|
+
def _valid_host(candidate: Optional[str]) -> bool:
|
|
72
|
+
return bool(
|
|
73
|
+
candidate
|
|
74
|
+
and "=" not in candidate
|
|
75
|
+
and HOST_RE.match(candidate)
|
|
76
|
+
and candidate.lower() not in _SKIP_HOST_TOKENS
|
|
77
|
+
and not candidate.isdigit()
|
|
78
|
+
)
|
|
79
|
+
|
|
65
80
|
_CACHE: Dict[str, Tuple[Pattern[str], str, Callable]] = {}
|
|
66
81
|
_CACHE_MAX = 256
|
|
67
82
|
|
|
@@ -131,8 +146,15 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
|
|
|
131
146
|
ts = conv(pattern_ts_match(pattern, match), now)
|
|
132
147
|
except (ValueError, KeyError):
|
|
133
148
|
ts = None
|
|
134
|
-
|
|
135
|
-
|
|
149
|
+
gd = match.groupdict()
|
|
150
|
+
host = gd.get("adaptive_host")
|
|
151
|
+
msg = gd.get("adaptive_msg")
|
|
152
|
+
msg = line if msg is None else msg
|
|
153
|
+
# The shape signature reduces all words to the same class, so a
|
|
154
|
+
# cached host slot may capture a non-host token; revalidate.
|
|
155
|
+
if host is not None and not _valid_host(host.rstrip(":,")):
|
|
156
|
+
msg = f"{host} {msg}".strip()
|
|
157
|
+
host = None
|
|
136
158
|
return _build_event(raw, pri, ts, host, msg)
|
|
137
159
|
|
|
138
160
|
# Analysis pass: find the earliest timestamp in the line head.
|
|
@@ -157,12 +179,7 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
|
|
|
157
179
|
tokens = rest.split(None, 1)
|
|
158
180
|
if tokens:
|
|
159
181
|
candidate = tokens[0].rstrip(":,")
|
|
160
|
-
if (
|
|
161
|
-
"=" not in candidate
|
|
162
|
-
and HOST_RE.match(candidate)
|
|
163
|
-
and candidate.lower() not in _SKIP_HOST_TOKENS
|
|
164
|
-
and not candidate.isdigit()
|
|
165
|
-
):
|
|
182
|
+
if _valid_host(candidate):
|
|
166
183
|
host = candidate
|
|
167
184
|
msg = tokens[1] if len(tokens) > 1 else ""
|
|
168
185
|
|
|
@@ -74,14 +74,23 @@ def _load_mapping(mapping: Mapping[str, Any] | Path | str | None) -> Mapping[str
|
|
|
74
74
|
def to_cef(
|
|
75
75
|
event: NormalizedEvent,
|
|
76
76
|
mapping: Mapping[str, Any] | Path | str | None = None,
|
|
77
|
+
*,
|
|
78
|
+
validate: bool = False,
|
|
79
|
+
strict: bool = False,
|
|
77
80
|
) -> str:
|
|
78
|
-
"""Convert a normalized event into a CEF string.
|
|
81
|
+
"""Convert a normalized event into a CEF string.
|
|
82
|
+
|
|
83
|
+
With ``validate=True``, extension values are checked against the
|
|
84
|
+
ArcSight dictionary and violations logged as warnings; ``strict=True``
|
|
85
|
+
additionally raises :class:`syslogcef.validation.CEFValidationError`
|
|
86
|
+
on type or length violations.
|
|
87
|
+
"""
|
|
79
88
|
|
|
80
89
|
if mapping is None:
|
|
81
90
|
mapping_data = _guess_mapping(event)
|
|
82
91
|
else:
|
|
83
92
|
mapping_data = _load_mapping(mapping)
|
|
84
|
-
cef_event = build_cef(event, mapping_data)
|
|
93
|
+
cef_event = build_cef(event, mapping_data, validate=validate, strict=strict)
|
|
85
94
|
return cef_event.render()
|
|
86
95
|
|
|
87
96
|
|
|
@@ -91,13 +100,15 @@ def convert_line(
|
|
|
91
100
|
mode: Optional[str] = None,
|
|
92
101
|
mapping: Mapping[str, Any] | Path | str | None = None,
|
|
93
102
|
now: Optional[datetime] = None,
|
|
103
|
+
validate: bool = False,
|
|
104
|
+
strict: bool = False,
|
|
94
105
|
) -> str:
|
|
95
106
|
"""Full pipeline that parses, normalizes and converts a syslog line."""
|
|
96
107
|
|
|
97
108
|
parsed = parse_syslog(line, mode=mode, now=now)
|
|
98
109
|
normalized = normalize_event(parsed)
|
|
99
110
|
|
|
100
|
-
return to_cef(normalized, mapping)
|
|
111
|
+
return to_cef(normalized, mapping, validate=validate, strict=strict)
|
|
101
112
|
|
|
102
113
|
|
|
103
114
|
def _guess_mapping(event: NormalizedEvent) -> Mapping[str, Any]:
|
|
@@ -111,9 +122,9 @@ def _guess_mapping(event: NormalizedEvent) -> Mapping[str, Any]:
|
|
|
111
122
|
return CISCO_IOS
|
|
112
123
|
if "BIG-IP" in msg_upper or "F5" in app_upper:
|
|
113
124
|
return F5
|
|
114
|
-
if "
|
|
125
|
+
if "logid" in event.kv:
|
|
115
126
|
return FORTINET
|
|
116
|
-
if "log_id" in event.kv
|
|
127
|
+
if "log_id" in event.kv:
|
|
117
128
|
return SOPHOS
|
|
118
129
|
if "VMWARE" in msg_upper or "ESXI" in msg_upper or "VMWARE" in app_upper:
|
|
119
130
|
return VMWARE
|
|
@@ -56,7 +56,12 @@ class MappingResolver:
|
|
|
56
56
|
header = {}
|
|
57
57
|
for key in ["deviceVendor", "deviceProduct", "deviceVersion", "eventClassId", "name"]:
|
|
58
58
|
template = self.mapping.get(key, DEFAULT_MAPPING[key])
|
|
59
|
-
|
|
59
|
+
value = self._format(template, fields)
|
|
60
|
+
if not value:
|
|
61
|
+
# A template referencing missing fields must not leave an
|
|
62
|
+
# empty CEF header slot; fall back to the default template.
|
|
63
|
+
value = self._format(DEFAULT_MAPPING[key], fields)
|
|
64
|
+
header[key] = cef_escape_header(value)
|
|
60
65
|
severity = self._resolve_severity(fields)
|
|
61
66
|
header["severity"] = severity
|
|
62
67
|
return header
|
|
@@ -71,15 +76,18 @@ class MappingResolver:
|
|
|
71
76
|
return "3"
|
|
72
77
|
return severity_map.get(str(value), str(value))
|
|
73
78
|
|
|
74
|
-
def
|
|
75
|
-
|
|
79
|
+
def resolve_raw_extensions(self, fields: Mapping[str, Any]) -> Dict[str, str]:
|
|
80
|
+
raw = {}
|
|
76
81
|
merged = DEFAULT_MAPPING["extensions"].copy()
|
|
77
82
|
merged.update(self.mapping.get("extensions", {}))
|
|
78
83
|
for key, template in merged.items():
|
|
79
84
|
value = self._format(template, fields)
|
|
80
85
|
if value:
|
|
81
|
-
|
|
82
|
-
return
|
|
86
|
+
raw[key] = str(value)
|
|
87
|
+
return raw
|
|
88
|
+
|
|
89
|
+
def resolve_extensions(self, fields: Mapping[str, Any]) -> Dict[str, str]:
|
|
90
|
+
return {k: cef_escape(v) for k, v in self.resolve_raw_extensions(fields).items()}
|
|
83
91
|
|
|
84
92
|
def _format(self, template: str, fields: Mapping[str, Any]) -> str:
|
|
85
93
|
try:
|
|
@@ -92,11 +100,27 @@ class MappingResolver:
|
|
|
92
100
|
return ""
|
|
93
101
|
|
|
94
102
|
|
|
95
|
-
def build_cef(
|
|
103
|
+
def build_cef(
|
|
104
|
+
event: NormalizedEvent,
|
|
105
|
+
mapping: Mapping[str, Any],
|
|
106
|
+
*,
|
|
107
|
+
validate: bool = False,
|
|
108
|
+
strict: bool = False,
|
|
109
|
+
) -> CEFEvent:
|
|
96
110
|
resolver = MappingResolver(mapping)
|
|
97
111
|
fields = event.as_field_dict()
|
|
98
112
|
header = resolver.resolve_header(fields)
|
|
99
|
-
|
|
113
|
+
raw = resolver.resolve_raw_extensions(fields)
|
|
114
|
+
if validate or strict:
|
|
115
|
+
from .validation import CEFValidationError, validate_extensions
|
|
116
|
+
|
|
117
|
+
findings = validate_extensions(raw)
|
|
118
|
+
for finding in findings:
|
|
119
|
+
logger.warning("CEF validation: %s", finding)
|
|
120
|
+
fatal = [f for f in findings if f.fatal]
|
|
121
|
+
if strict and fatal:
|
|
122
|
+
raise CEFValidationError("; ".join(str(f) for f in fatal))
|
|
123
|
+
extensions = {k: cef_escape(v) for k, v in raw.items()}
|
|
100
124
|
return CEFEvent(header=header, extensions=extensions)
|
|
101
125
|
|
|
102
126
|
|
|
@@ -74,8 +74,8 @@ def follow_files(
|
|
|
74
74
|
handle.close()
|
|
75
75
|
|
|
76
76
|
|
|
77
|
-
def _convert_line_mp(line: str, mode: Optional[str], mapping: Optional[str]) -> str:
|
|
78
|
-
return convert_line(line, mode=mode, mapping=mapping)
|
|
77
|
+
def _convert_line_mp(line: str, mode: Optional[str], mapping: Optional[str], validate: bool = False, strict: bool = False) -> str:
|
|
78
|
+
return convert_line(line, mode=mode, mapping=mapping, validate=validate, strict=strict)
|
|
79
79
|
|
|
80
80
|
|
|
81
81
|
def process_lines(
|
|
@@ -85,16 +85,18 @@ def process_lines(
|
|
|
85
85
|
mapping: Optional[str],
|
|
86
86
|
use_multiprocessing: bool,
|
|
87
87
|
pool_size: Optional[int],
|
|
88
|
+
validate: bool = False,
|
|
89
|
+
strict: bool = False,
|
|
88
90
|
) -> Iterator[str]:
|
|
89
91
|
if use_multiprocessing:
|
|
90
92
|
size = pool_size or max(1, cpu_count() - 1)
|
|
91
93
|
with Pool(size) as pool:
|
|
92
|
-
worker = partial(_convert_line_mp, mode=mode, mapping=mapping)
|
|
94
|
+
worker = partial(_convert_line_mp, mode=mode, mapping=mapping, validate=validate, strict=strict)
|
|
93
95
|
for cef in pool.imap(worker, lines):
|
|
94
96
|
yield cef
|
|
95
97
|
else:
|
|
96
98
|
for line in lines:
|
|
97
|
-
yield convert_line(line, mode=mode, mapping=mapping)
|
|
99
|
+
yield convert_line(line, mode=mode, mapping=mapping, validate=validate, strict=strict)
|
|
98
100
|
|
|
99
101
|
|
|
100
102
|
def main(argv: Optional[list[str]] = None) -> int:
|
|
@@ -105,8 +107,13 @@ def main(argv: Optional[list[str]] = None) -> int:
|
|
|
105
107
|
parser.add_argument("--mode", choices=["rfc3164", "rfc5424", "rsyslog_json", "rsyslog_file", "journald_json", "journald_short", "journald_iso", "cisco_seq", "iso_syslog", "kv"], help="Parser mode override")
|
|
106
108
|
parser.add_argument("--mapping", type=str, help="Mapping JSON file")
|
|
107
109
|
parser.add_argument("--tail", action="store_true", help="Follow file like tail -f")
|
|
110
|
+
parser.add_argument("--listen", metavar="PROTO:PORT", help="Receive syslog over the network instead of reading files/stdin, e.g. udp:514, tcp:5514, or udp:10.0.0.5:514")
|
|
111
|
+
parser.add_argument("--send", metavar="URL", help="Forward CEF records to udp://HOST:PORT, tcp://HOST:PORT (newline-delimited, with reconnect), or kafka://BROKER:PORT/TOPIC (requires the kafka extra)")
|
|
112
|
+
parser.add_argument("--eps", type=float, help="Rate-limit --send to at most this many events per second")
|
|
108
113
|
parser.add_argument("--multiprocess", action="store_true", help="Process lines using a process pool")
|
|
109
114
|
parser.add_argument("--pool-size", type=int, help="Number of worker processes for --multiprocess")
|
|
115
|
+
parser.add_argument("--validate", action="store_true", help="Validate CEF extensions against the ArcSight dictionary; violations are logged as warnings")
|
|
116
|
+
parser.add_argument("--strict", action="store_true", help="Like --validate, but exit with an error on type/length violations")
|
|
110
117
|
parser.add_argument("--log-level", default="WARNING", help="Logging level")
|
|
111
118
|
|
|
112
119
|
args = parser.parse_args(argv)
|
|
@@ -115,20 +122,52 @@ def main(argv: Optional[list[str]] = None) -> int:
|
|
|
115
122
|
mapping = args.mapping
|
|
116
123
|
|
|
117
124
|
outputs: Iterator[str]
|
|
118
|
-
if args.
|
|
119
|
-
|
|
125
|
+
if args.listen:
|
|
126
|
+
from .net import listen_lines, parse_endpoint
|
|
127
|
+
|
|
128
|
+
try:
|
|
129
|
+
proto, host, port = parse_endpoint(args.listen)
|
|
130
|
+
except ValueError as exc:
|
|
131
|
+
parser.error(str(exc))
|
|
132
|
+
lines = listen_lines(proto, host, port)
|
|
133
|
+
outputs = process_lines(lines, mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size, validate=args.validate or args.strict, strict=args.strict)
|
|
134
|
+
elif args.tail and args.paths:
|
|
135
|
+
outputs = process_lines(follow_files(args.paths), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size, validate=args.validate or args.strict, strict=args.strict)
|
|
120
136
|
else:
|
|
121
|
-
outputs = process_lines(iter_lines_from_sources(args.paths), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size)
|
|
137
|
+
outputs = process_lines(iter_lines_from_sources(args.paths), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size, validate=args.validate or args.strict, strict=args.strict)
|
|
122
138
|
|
|
123
|
-
|
|
124
|
-
|
|
139
|
+
sender = None
|
|
140
|
+
if args.send:
|
|
141
|
+
from .net import create_sender
|
|
142
|
+
|
|
143
|
+
try:
|
|
144
|
+
sender = create_sender(args.send, eps=args.eps)
|
|
145
|
+
except (ValueError, RuntimeError) as exc:
|
|
146
|
+
parser.error(str(exc))
|
|
147
|
+
|
|
148
|
+
from .validation import CEFValidationError
|
|
149
|
+
|
|
150
|
+
try:
|
|
151
|
+
if args.output:
|
|
152
|
+
with args.output.open("a" if args.append else "w", encoding="utf-8") as fp:
|
|
153
|
+
for cef in outputs:
|
|
154
|
+
fp.write(cef + "\n")
|
|
155
|
+
if args.tail or args.listen:
|
|
156
|
+
fp.flush()
|
|
157
|
+
if sender:
|
|
158
|
+
sender.send(cef)
|
|
159
|
+
elif sender:
|
|
125
160
|
for cef in outputs:
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
161
|
+
sender.send(cef)
|
|
162
|
+
else:
|
|
163
|
+
for cef in outputs:
|
|
164
|
+
print(cef)
|
|
165
|
+
except CEFValidationError as exc:
|
|
166
|
+
print(f"syslogcef: strict validation failed: {exc}", file=sys.stderr)
|
|
167
|
+
return 1
|
|
168
|
+
finally:
|
|
169
|
+
if sender:
|
|
170
|
+
sender.close()
|
|
132
171
|
return 0
|
|
133
172
|
|
|
134
173
|
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
"""Network input for syslogcef: receive syslog over UDP or TCP.
|
|
2
|
+
|
|
3
|
+
The listener yields raw lines exactly as the rest of the pipeline expects
|
|
4
|
+
them, so ``--listen`` composes with every existing option (mappings, mode
|
|
5
|
+
overrides, outputs).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import logging
|
|
11
|
+
import selectors
|
|
12
|
+
import socket
|
|
13
|
+
from typing import Callable, Iterator, Optional, Tuple
|
|
14
|
+
|
|
15
|
+
logger = logging.getLogger(__name__)
|
|
16
|
+
|
|
17
|
+
MAX_DATAGRAM = 65535
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def parse_endpoint(spec: str) -> Tuple[str, str, int]:
|
|
21
|
+
"""Parse ``udp:514``, ``tcp:5514``, or ``udp:10.0.0.5:514``."""
|
|
22
|
+
|
|
23
|
+
parts = spec.split(":")
|
|
24
|
+
proto = parts[0].lower()
|
|
25
|
+
if proto not in ("udp", "tcp") or len(parts) not in (2, 3):
|
|
26
|
+
raise ValueError(
|
|
27
|
+
f"Invalid endpoint {spec!r}; expected udp:PORT, tcp:PORT, or udp:HOST:PORT"
|
|
28
|
+
)
|
|
29
|
+
host = parts[1] if len(parts) == 3 else "0.0.0.0"
|
|
30
|
+
try:
|
|
31
|
+
port = int(parts[-1])
|
|
32
|
+
except ValueError as exc:
|
|
33
|
+
raise ValueError(f"Invalid port in endpoint {spec!r}") from exc
|
|
34
|
+
if not 0 <= port <= 65535:
|
|
35
|
+
raise ValueError(f"Port out of range in endpoint {spec!r}")
|
|
36
|
+
return proto, host, port
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def listen_lines(
|
|
40
|
+
proto: str,
|
|
41
|
+
host: str,
|
|
42
|
+
port: int,
|
|
43
|
+
*,
|
|
44
|
+
max_messages: Optional[int] = None,
|
|
45
|
+
ready_callback: Optional[Callable[[int], None]] = None,
|
|
46
|
+
) -> Iterator[str]:
|
|
47
|
+
"""Yield syslog lines received on the given endpoint.
|
|
48
|
+
|
|
49
|
+
``max_messages`` bounds the number of yielded lines (used by tests);
|
|
50
|
+
``None`` listens forever. ``ready_callback`` receives the bound port
|
|
51
|
+
once the socket is listening (useful with port 0).
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
if proto == "udp":
|
|
55
|
+
yield from _listen_udp(host, port, max_messages, ready_callback)
|
|
56
|
+
else:
|
|
57
|
+
yield from _listen_tcp(host, port, max_messages, ready_callback)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _listen_udp(host, port, max_messages, ready_callback) -> Iterator[str]:
|
|
61
|
+
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
62
|
+
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
|
63
|
+
sock.bind((host, port))
|
|
64
|
+
if ready_callback:
|
|
65
|
+
ready_callback(sock.getsockname()[1])
|
|
66
|
+
logger.info("Listening on udp:%s:%s", *sock.getsockname())
|
|
67
|
+
count = 0
|
|
68
|
+
try:
|
|
69
|
+
while max_messages is None or count < max_messages:
|
|
70
|
+
data, _addr = sock.recvfrom(MAX_DATAGRAM)
|
|
71
|
+
for raw in data.decode("utf-8", "replace").splitlines():
|
|
72
|
+
if raw.strip():
|
|
73
|
+
yield raw
|
|
74
|
+
count += 1
|
|
75
|
+
if max_messages is not None and count >= max_messages:
|
|
76
|
+
return
|
|
77
|
+
finally:
|
|
78
|
+
sock.close()
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _listen_tcp(host, port, max_messages, ready_callback) -> Iterator[str]:
|
|
82
|
+
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
83
|
+
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
|
84
|
+
server.bind((host, port))
|
|
85
|
+
server.listen(16)
|
|
86
|
+
server.setblocking(False)
|
|
87
|
+
if ready_callback:
|
|
88
|
+
ready_callback(server.getsockname()[1])
|
|
89
|
+
logger.info("Listening on tcp:%s:%s", *server.getsockname())
|
|
90
|
+
|
|
91
|
+
sel = selectors.DefaultSelector()
|
|
92
|
+
sel.register(server, selectors.EVENT_READ)
|
|
93
|
+
buffers: dict[socket.socket, bytes] = {}
|
|
94
|
+
count = 0
|
|
95
|
+
try:
|
|
96
|
+
while max_messages is None or count < max_messages:
|
|
97
|
+
for key, _events in sel.select(timeout=1):
|
|
98
|
+
sock = key.fileobj
|
|
99
|
+
if sock is server:
|
|
100
|
+
conn, addr = server.accept()
|
|
101
|
+
conn.setblocking(False)
|
|
102
|
+
sel.register(conn, selectors.EVENT_READ)
|
|
103
|
+
buffers[conn] = b""
|
|
104
|
+
logger.debug("Connection from %s", addr)
|
|
105
|
+
continue
|
|
106
|
+
try:
|
|
107
|
+
data = sock.recv(65536)
|
|
108
|
+
except (BlockingIOError, InterruptedError):
|
|
109
|
+
continue
|
|
110
|
+
except ConnectionError:
|
|
111
|
+
data = b""
|
|
112
|
+
if not data:
|
|
113
|
+
sel.unregister(sock)
|
|
114
|
+
sock.close()
|
|
115
|
+
remainder = buffers.pop(sock, b"")
|
|
116
|
+
line = remainder.decode("utf-8", "replace").strip()
|
|
117
|
+
if line:
|
|
118
|
+
yield line
|
|
119
|
+
count += 1
|
|
120
|
+
continue
|
|
121
|
+
buffers[sock] += data
|
|
122
|
+
while b"\n" in buffers[sock]:
|
|
123
|
+
raw, buffers[sock] = buffers[sock].split(b"\n", 1)
|
|
124
|
+
line = raw.decode("utf-8", "replace").rstrip("\r")
|
|
125
|
+
if line.strip():
|
|
126
|
+
yield line
|
|
127
|
+
count += 1
|
|
128
|
+
if max_messages is not None and count >= max_messages:
|
|
129
|
+
return
|
|
130
|
+
finally:
|
|
131
|
+
for sock in list(buffers):
|
|
132
|
+
sock.close()
|
|
133
|
+
sel.close()
|
|
134
|
+
server.close()
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
__all__ = ["listen_lines", "parse_endpoint"]
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
# --------------------------------------------------------------------------
|
|
141
|
+
# Network output: forward CEF records to a SIEM or pipeline.
|
|
142
|
+
|
|
143
|
+
import time as _time
|
|
144
|
+
from urllib.parse import urlparse
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
class RateLimiter:
|
|
148
|
+
"""Simple events-per-second pacer."""
|
|
149
|
+
|
|
150
|
+
def __init__(self, eps: float, clock=_time.monotonic, sleep=_time.sleep):
|
|
151
|
+
if eps <= 0:
|
|
152
|
+
raise ValueError("eps must be positive")
|
|
153
|
+
self.interval = 1.0 / eps
|
|
154
|
+
self._clock = clock
|
|
155
|
+
self._sleep = sleep
|
|
156
|
+
self._next = self._clock()
|
|
157
|
+
|
|
158
|
+
def wait(self) -> None:
|
|
159
|
+
now = self._clock()
|
|
160
|
+
if now < self._next:
|
|
161
|
+
self._sleep(self._next - now)
|
|
162
|
+
self._next = max(self._next + self.interval, self._clock())
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
class SyslogSender:
|
|
166
|
+
"""Send CEF records to a syslog destination over UDP or TCP.
|
|
167
|
+
|
|
168
|
+
TCP sends are newline-delimited and reconnect with backoff on failure.
|
|
169
|
+
"""
|
|
170
|
+
|
|
171
|
+
RETRY_DELAYS = (0.5, 1.0, 2.0)
|
|
172
|
+
|
|
173
|
+
def __init__(self, proto: str, host: str, port: int, *, limiter: Optional[RateLimiter] = None):
|
|
174
|
+
self.proto = proto
|
|
175
|
+
self.host = host
|
|
176
|
+
self.port = port
|
|
177
|
+
self.limiter = limiter
|
|
178
|
+
self._sock: Optional[socket.socket] = None
|
|
179
|
+
if proto == "udp":
|
|
180
|
+
self._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
181
|
+
|
|
182
|
+
def _connect(self) -> socket.socket:
|
|
183
|
+
sock = socket.create_connection((self.host, self.port), timeout=10)
|
|
184
|
+
return sock
|
|
185
|
+
|
|
186
|
+
def send(self, record: str) -> None:
|
|
187
|
+
if self.limiter:
|
|
188
|
+
self.limiter.wait()
|
|
189
|
+
payload = record.encode("utf-8")
|
|
190
|
+
if self.proto == "udp":
|
|
191
|
+
self._sock.sendto(payload, (self.host, self.port))
|
|
192
|
+
return
|
|
193
|
+
data = payload + b"\n"
|
|
194
|
+
last_error: Optional[Exception] = None
|
|
195
|
+
for attempt, delay in enumerate((0.0,) + self.RETRY_DELAYS):
|
|
196
|
+
if delay:
|
|
197
|
+
_time.sleep(delay)
|
|
198
|
+
try:
|
|
199
|
+
if self._sock is None:
|
|
200
|
+
self._sock = self._connect()
|
|
201
|
+
self._sock.sendall(data)
|
|
202
|
+
return
|
|
203
|
+
except OSError as exc:
|
|
204
|
+
last_error = exc
|
|
205
|
+
logger.warning("send to %s:%s failed (%s); reconnecting", self.host, self.port, exc)
|
|
206
|
+
if self._sock is not None:
|
|
207
|
+
try:
|
|
208
|
+
self._sock.close()
|
|
209
|
+
except OSError:
|
|
210
|
+
pass
|
|
211
|
+
self._sock = None
|
|
212
|
+
raise ConnectionError(f"Could not deliver record to {self.host}:{self.port}") from last_error
|
|
213
|
+
|
|
214
|
+
def close(self) -> None:
|
|
215
|
+
if self._sock is not None:
|
|
216
|
+
try:
|
|
217
|
+
self._sock.close()
|
|
218
|
+
finally:
|
|
219
|
+
self._sock = None
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
class KafkaSender:
|
|
223
|
+
"""Send CEF records to a Kafka topic (requires the 'kafka' extra)."""
|
|
224
|
+
|
|
225
|
+
def __init__(self, broker: str, topic: str, *, limiter: Optional[RateLimiter] = None, producer=None):
|
|
226
|
+
self.topic = topic
|
|
227
|
+
self.limiter = limiter
|
|
228
|
+
if producer is None:
|
|
229
|
+
try:
|
|
230
|
+
from kafka import KafkaProducer
|
|
231
|
+
except ImportError as exc:
|
|
232
|
+
raise RuntimeError(
|
|
233
|
+
"Kafka output requires the kafka extra: pip install syslog2cef[kafka]"
|
|
234
|
+
) from exc
|
|
235
|
+
producer = KafkaProducer(bootstrap_servers=broker)
|
|
236
|
+
self.producer = producer
|
|
237
|
+
|
|
238
|
+
def send(self, record: str) -> None:
|
|
239
|
+
if self.limiter:
|
|
240
|
+
self.limiter.wait()
|
|
241
|
+
self.producer.send(self.topic, record.encode("utf-8"))
|
|
242
|
+
|
|
243
|
+
def close(self) -> None:
|
|
244
|
+
try:
|
|
245
|
+
self.producer.flush()
|
|
246
|
+
finally:
|
|
247
|
+
close = getattr(self.producer, "close", None)
|
|
248
|
+
if close:
|
|
249
|
+
close()
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def create_sender(url: str, *, eps: Optional[float] = None):
|
|
253
|
+
"""Create a sender from udp://host:port, tcp://host:port, or kafka://broker/topic."""
|
|
254
|
+
|
|
255
|
+
parsed = urlparse(url)
|
|
256
|
+
limiter = RateLimiter(eps) if eps else None
|
|
257
|
+
if parsed.scheme in ("udp", "tcp"):
|
|
258
|
+
if not parsed.hostname or not parsed.port:
|
|
259
|
+
raise ValueError(f"Invalid send target {url!r}; expected {parsed.scheme}://HOST:PORT")
|
|
260
|
+
return SyslogSender(parsed.scheme, parsed.hostname, parsed.port, limiter=limiter)
|
|
261
|
+
if parsed.scheme == "kafka":
|
|
262
|
+
topic = parsed.path.lstrip("/")
|
|
263
|
+
if not parsed.hostname or not topic:
|
|
264
|
+
raise ValueError(f"Invalid send target {url!r}; expected kafka://BROKER:PORT/TOPIC")
|
|
265
|
+
broker = f"{parsed.hostname}:{parsed.port or 9092}"
|
|
266
|
+
return KafkaSender(broker, topic, limiter=limiter)
|
|
267
|
+
raise ValueError(f"Unsupported send scheme {parsed.scheme!r}; use udp://, tcp://, or kafka://")
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
__all__ += ["RateLimiter", "SyslogSender", "KafkaSender", "create_sender"]
|
|
@@ -96,7 +96,7 @@ CISCO_SEQ_RE = re.compile(
|
|
|
96
96
|
r"(?P<month>[A-Z][a-z]{2})\s+"
|
|
97
97
|
r"(?P<day>\d{1,2})\s+"
|
|
98
98
|
r"(?:(?P<year>\d{4})\s+)?"
|
|
99
|
-
r"(?P<time>\d{2}:\d{2}:\d{2})(
|
|
99
|
+
r"(?P<time>\d{2}:\d{2}:\d{2})(?P<frac>\.\d+)?"
|
|
100
100
|
r"(?:\s+[A-Z]{3,4})?:\s+"
|
|
101
101
|
r"(?P<msg>%[A-Z].*)$"
|
|
102
102
|
)
|
|
@@ -369,10 +369,11 @@ def parse_cisco_seq(line: str, *, now: Optional[datetime]) -> ParsedEvent | None
|
|
|
369
369
|
facility, severity = convert_pri(pri)
|
|
370
370
|
month = month_abbr_to_int(gd["month"])
|
|
371
371
|
hour, minute, second = map(int, gd["time"].split(":"))
|
|
372
|
+
microsecond = int(float(gd["frac"]) * 1_000_000) if gd.get("frac") else 0
|
|
372
373
|
if gd.get("year"):
|
|
373
|
-
ts = datetime(int(gd["year"]), month, int(gd["day"]), hour, minute, second, tzinfo=timezone.utc)
|
|
374
|
+
ts = datetime(int(gd["year"]), month, int(gd["day"]), hour, minute, second, microsecond, tzinfo=timezone.utc)
|
|
374
375
|
else:
|
|
375
|
-
ts = _infer_timestamp(month, int(gd["day"]), gd["time"], now=now)
|
|
376
|
+
ts = _infer_timestamp(month, int(gd["day"]), gd["time"], now=now).replace(microsecond=microsecond)
|
|
376
377
|
sd = {"cisco.sequence": gd["seq"]} if gd.get("seq") else {}
|
|
377
378
|
return ParsedEvent(
|
|
378
379
|
pri=pri,
|
|
@@ -444,6 +445,18 @@ def parse_kv_stream(line: str) -> ParsedEvent | None:
|
|
|
444
445
|
ts = parse_iso8601(pairs["eventtime"][:16])
|
|
445
446
|
except ValueError:
|
|
446
447
|
ts = None
|
|
448
|
+
if ts is not None and ts.tzinfo is None:
|
|
449
|
+
# Apply a numeric offset supplied in the record (tz="-0500");
|
|
450
|
+
# named zones (timezone="CEST") cannot be resolved portably.
|
|
451
|
+
offset_match = re.fullmatch(
|
|
452
|
+
r"([+-])(\d{2}):?(\d{2})", pairs.get("tz") or pairs.get("timezone") or ""
|
|
453
|
+
)
|
|
454
|
+
if offset_match:
|
|
455
|
+
sign = 1 if offset_match.group(1) == "+" else -1
|
|
456
|
+
delta = timedelta(
|
|
457
|
+
hours=int(offset_match.group(2)), minutes=int(offset_match.group(3))
|
|
458
|
+
)
|
|
459
|
+
ts = ts.replace(tzinfo=timezone(sign * delta))
|
|
447
460
|
|
|
448
461
|
host = (
|
|
449
462
|
pairs.get("devname")
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
"""Validation of CEF extension keys and values against the ArcSight dictionary.
|
|
2
|
+
|
|
3
|
+
The key metadata below is derived from the ArcSight Extension Dictionary
|
|
4
|
+
(CEF key names for event producers): data type and maximum string length
|
|
5
|
+
per key. Custom keys are permitted by the CEF standard, so unknown keys
|
|
6
|
+
produce advisory findings only; type and length violations are the
|
|
7
|
+
errors that ``strict`` mode raises on.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import ipaddress
|
|
13
|
+
import re
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from typing import Dict, List, Tuple
|
|
16
|
+
|
|
17
|
+
# type tags: str (with max length), int, long, float, port, ip (v4/v6),
|
|
18
|
+
# ipv6, mac, ts (MMM dd yyyy HH:mm:ss or epoch milliseconds)
|
|
19
|
+
_S = "str"
|
|
20
|
+
|
|
21
|
+
CEF_KEYS: Dict[str, Tuple[str, int]] = {
|
|
22
|
+
"act": (_S, 63), "app": (_S, 31), "cat": (_S, 1023), "cnt": ("int", 0),
|
|
23
|
+
"cn1": ("long", 0), "cn2": ("long", 0), "cn3": ("long", 0),
|
|
24
|
+
"cn1Label": (_S, 1023), "cn2Label": (_S, 1023), "cn3Label": (_S, 1023),
|
|
25
|
+
"cfp1": ("float", 0), "cfp2": ("float", 0), "cfp3": ("float", 0), "cfp4": ("float", 0),
|
|
26
|
+
"cfp1Label": (_S, 1023), "cfp2Label": (_S, 1023), "cfp3Label": (_S, 1023), "cfp4Label": (_S, 1023),
|
|
27
|
+
"cs1": (_S, 4000), "cs2": (_S, 4000), "cs3": (_S, 4000),
|
|
28
|
+
"cs4": (_S, 4000), "cs5": (_S, 4000), "cs6": (_S, 4000),
|
|
29
|
+
"cs1Label": (_S, 1023), "cs2Label": (_S, 1023), "cs3Label": (_S, 1023),
|
|
30
|
+
"cs4Label": (_S, 1023), "cs5Label": (_S, 1023), "cs6Label": (_S, 1023),
|
|
31
|
+
"c6a1": ("ipv6", 0), "c6a2": ("ipv6", 0), "c6a3": ("ipv6", 0), "c6a4": ("ipv6", 0),
|
|
32
|
+
"c6a1Label": (_S, 1023), "c6a2Label": (_S, 1023), "c6a3Label": (_S, 1023), "c6a4Label": (_S, 1023),
|
|
33
|
+
"destinationDnsDomain": (_S, 255), "destinationServiceName": (_S, 1023),
|
|
34
|
+
"destinationTranslatedAddress": ("ip", 0), "destinationTranslatedPort": ("port", 0),
|
|
35
|
+
"deviceDirection": ("int", 0), "deviceDnsDomain": (_S, 255),
|
|
36
|
+
"deviceExternalId": (_S, 255), "deviceFacility": (_S, 1023),
|
|
37
|
+
"deviceInboundInterface": (_S, 128), "deviceOutboundInterface": (_S, 128),
|
|
38
|
+
"deviceNtDomain": (_S, 255), "devicePayloadId": (_S, 128),
|
|
39
|
+
"deviceProcessName": (_S, 1023), "deviceTranslatedAddress": ("ip", 0),
|
|
40
|
+
"deviceCustomDate1": ("ts", 0), "deviceCustomDate2": ("ts", 0),
|
|
41
|
+
"deviceCustomDate1Label": (_S, 1023), "deviceCustomDate2Label": (_S, 1023),
|
|
42
|
+
"dhost": (_S, 1023), "dmac": ("mac", 0), "dntdom": (_S, 255),
|
|
43
|
+
"dpid": ("int", 0), "dpriv": (_S, 1023), "dproc": (_S, 1023),
|
|
44
|
+
"dpt": ("port", 0), "dst": ("ip", 0), "dtz": (_S, 255),
|
|
45
|
+
"duid": (_S, 1023), "duser": (_S, 1023),
|
|
46
|
+
"dvc": ("ip", 0), "dvchost": (_S, 100), "dvcpid": ("int", 0),
|
|
47
|
+
"end": ("ts", 0), "externalId": (_S, 40),
|
|
48
|
+
"fileCreateTime": ("ts", 0), "fileHash": (_S, 255), "fileId": (_S, 1023),
|
|
49
|
+
"fileModificationTime": ("ts", 0), "filePath": (_S, 1023),
|
|
50
|
+
"filePermission": (_S, 1023), "fileType": (_S, 1023),
|
|
51
|
+
"flexDate1": ("ts", 0), "flexDate1Label": (_S, 128),
|
|
52
|
+
"flexString1": (_S, 1023), "flexString1Label": (_S, 128),
|
|
53
|
+
"flexString2": (_S, 1023), "flexString2Label": (_S, 128),
|
|
54
|
+
"fname": (_S, 1023), "fsize": ("int", 0),
|
|
55
|
+
"in": ("long", 0), "out": ("long", 0),
|
|
56
|
+
"msg": (_S, 1023), "outcome": (_S, 63), "proto": (_S, 31),
|
|
57
|
+
"reason": (_S, 1023), "request": (_S, 1023),
|
|
58
|
+
"requestClientApplication": (_S, 1023), "requestContext": (_S, 2048),
|
|
59
|
+
"requestCookies": (_S, 1023), "requestMethod": (_S, 1023),
|
|
60
|
+
"rt": ("ts", 0), "start": ("ts", 0),
|
|
61
|
+
"shost": (_S, 1023), "smac": ("mac", 0), "sntdom": (_S, 255),
|
|
62
|
+
"sourceDnsDomain": (_S, 255), "sourceServiceName": (_S, 1023),
|
|
63
|
+
"sourceTranslatedAddress": ("ip", 0), "sourceTranslatedPort": ("port", 0),
|
|
64
|
+
"spid": ("int", 0), "spriv": (_S, 1023), "sproc": (_S, 1023),
|
|
65
|
+
"spt": ("port", 0), "src": ("ip", 0),
|
|
66
|
+
"suid": (_S, 1023), "suser": (_S, 1023),
|
|
67
|
+
"type": ("int", 0),
|
|
68
|
+
"rawEvent": (_S, 4000),
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
MAC_RE = re.compile(r"^[0-9a-fA-F]{2}(:[0-9a-fA-F]{2}){5}$")
|
|
72
|
+
TS_RE = re.compile(r"^[A-Z][a-z]{2} \d{2} \d{4} \d{2}:\d{2}:\d{2}$")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class CEFValidationError(ValueError):
|
|
76
|
+
"""Raised in strict mode when an extension value violates the dictionary."""
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@dataclass
|
|
80
|
+
class Finding:
|
|
81
|
+
key: str
|
|
82
|
+
problem: str
|
|
83
|
+
fatal: bool
|
|
84
|
+
|
|
85
|
+
def __str__(self) -> str:
|
|
86
|
+
return f"{self.key}: {self.problem}"
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _is_ip(value: str, version: int | None = None) -> bool:
|
|
90
|
+
try:
|
|
91
|
+
addr = ipaddress.ip_address(value)
|
|
92
|
+
except ValueError:
|
|
93
|
+
return False
|
|
94
|
+
return version is None or addr.version == version
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _check(key: str, value: str) -> str | None:
|
|
98
|
+
kind, maxlen = CEF_KEYS[key]
|
|
99
|
+
if kind == _S:
|
|
100
|
+
if maxlen and len(value) > maxlen:
|
|
101
|
+
return f"string exceeds maximum length {maxlen} ({len(value)} characters)"
|
|
102
|
+
return None
|
|
103
|
+
if kind in ("int", "long"):
|
|
104
|
+
try:
|
|
105
|
+
int(value)
|
|
106
|
+
except ValueError:
|
|
107
|
+
return f"expected an integer, got {value!r}"
|
|
108
|
+
return None
|
|
109
|
+
if kind == "float":
|
|
110
|
+
try:
|
|
111
|
+
float(value)
|
|
112
|
+
except ValueError:
|
|
113
|
+
return f"expected a number, got {value!r}"
|
|
114
|
+
return None
|
|
115
|
+
if kind == "port":
|
|
116
|
+
try:
|
|
117
|
+
port = int(value)
|
|
118
|
+
except ValueError:
|
|
119
|
+
return f"expected a port number, got {value!r}"
|
|
120
|
+
if not 0 <= port <= 65535:
|
|
121
|
+
return f"port {port} outside 0-65535"
|
|
122
|
+
return None
|
|
123
|
+
if kind == "ip":
|
|
124
|
+
if not _is_ip(value):
|
|
125
|
+
return f"expected an IP address, got {value!r}"
|
|
126
|
+
return None
|
|
127
|
+
if kind == "ipv6":
|
|
128
|
+
if not _is_ip(value, 6):
|
|
129
|
+
return f"expected an IPv6 address, got {value!r}"
|
|
130
|
+
return None
|
|
131
|
+
if kind == "mac":
|
|
132
|
+
if not MAC_RE.match(value):
|
|
133
|
+
return f"expected a MAC address (aa:bb:cc:dd:ee:ff), got {value!r}"
|
|
134
|
+
return None
|
|
135
|
+
if kind == "ts":
|
|
136
|
+
if value.isdigit():
|
|
137
|
+
return None # epoch milliseconds
|
|
138
|
+
if TS_RE.match(value):
|
|
139
|
+
return None
|
|
140
|
+
return f"expected 'MMM dd yyyy HH:mm:ss' or epoch milliseconds, got {value!r}"
|
|
141
|
+
return None
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def validate_extensions(extensions: Dict[str, str]) -> List[Finding]:
|
|
145
|
+
"""Validate raw (pre-escaping) extension values against the dictionary.
|
|
146
|
+
|
|
147
|
+
Returns findings; type/length violations are fatal, unknown keys are
|
|
148
|
+
advisory (the CEF standard allows custom keys).
|
|
149
|
+
"""
|
|
150
|
+
|
|
151
|
+
findings: List[Finding] = []
|
|
152
|
+
for key, value in extensions.items():
|
|
153
|
+
if key not in CEF_KEYS:
|
|
154
|
+
findings.append(
|
|
155
|
+
Finding(key, "not in the ArcSight dictionary (custom keys are allowed)", fatal=False)
|
|
156
|
+
)
|
|
157
|
+
continue
|
|
158
|
+
problem = _check(key, str(value))
|
|
159
|
+
if problem:
|
|
160
|
+
findings.append(Finding(key, problem, fatal=True))
|
|
161
|
+
return findings
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
__all__ = ["CEF_KEYS", "CEFValidationError", "Finding", "validate_extensions"]
|
|
@@ -135,3 +135,57 @@ def test_garbage_still_falls_back_to_unknown():
|
|
|
135
135
|
ev2 = parse_syslog("<30>still garbage but with a pri")
|
|
136
136
|
assert ev2.source_hint == "unknown"
|
|
137
137
|
assert ev2.pri == 30
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
# --- fixes from code review of the adaptive parsing PR -------------------------
|
|
141
|
+
|
|
142
|
+
def test_kv_without_id_falls_back_to_nonempty_event_class():
|
|
143
|
+
line = '<30>device="SFW" date=2020-05-18 time=14:38:48 device_name="XG230" reason="x" a=1'
|
|
144
|
+
cef = convert_line(line)
|
|
145
|
+
header_fields = fields(cef)
|
|
146
|
+
assert header_fields[4] != "" # eventClassId never empty
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def test_kv_numeric_timezone_applied():
|
|
150
|
+
line = 'date=2020-04-23 time=12:32:48 devname="fw1" logid="0100000001" tz="-0500" x=1'
|
|
151
|
+
ev = parse_syslog(line)
|
|
152
|
+
assert ev.ts.utcoffset().total_seconds() == -5 * 3600
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def test_cisco_milliseconds_preserved():
|
|
156
|
+
line = "000123: Feb 8 04:00:47.272: %SEC-6-IPACCESSLOGP: list 100 denied tcp"
|
|
157
|
+
ev = parse_syslog(line)
|
|
158
|
+
assert ev.ts.microsecond == 272000
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def test_adaptive_mon_day_year_rollover():
|
|
162
|
+
import re
|
|
163
|
+
from datetime import datetime, timezone
|
|
164
|
+
|
|
165
|
+
from syslogcef.adaptive import TIMESTAMP_LIB
|
|
166
|
+
|
|
167
|
+
src, conv = next((s, c) for n, s, c in TIMESTAMP_LIB if n == "mon_day")
|
|
168
|
+
match = re.compile(src).match("Dec 31 23:59:59")
|
|
169
|
+
ts = conv(match, datetime(2026, 1, 2, tzinfo=timezone.utc))
|
|
170
|
+
assert ts.year == 2025
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def test_adaptive_cached_host_revalidated():
|
|
174
|
+
clear_cache()
|
|
175
|
+
good = "2021/05/01 10:00:00 fw-edge-01 session established"
|
|
176
|
+
bad = "2021/05/02 11:00:00 ERROR link down"
|
|
177
|
+
ev1 = parse_syslog(good)
|
|
178
|
+
assert ev1.host == "fw-edge-01"
|
|
179
|
+
ev2 = parse_syslog(bad)
|
|
180
|
+
assert ev2.source_hint == "adaptive"
|
|
181
|
+
assert ev2.host is None
|
|
182
|
+
assert ev2.msg.startswith("ERROR")
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def test_adaptive_cached_empty_message_preserved():
|
|
186
|
+
clear_cache()
|
|
187
|
+
ev1 = parse_syslog("2021/05/01 10:00:00 fw-edge-01 something happened")
|
|
188
|
+
assert ev1.msg == "something happened"
|
|
189
|
+
ev2 = parse_syslog("2021/05/02 11:00:00 fw-edge-02")
|
|
190
|
+
assert ev2.host == "fw-edge-02"
|
|
191
|
+
assert ev2.msg == ""
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import socket
|
|
4
|
+
import threading
|
|
5
|
+
|
|
6
|
+
import pytest
|
|
7
|
+
|
|
8
|
+
from syslogcef.net import listen_lines, parse_endpoint
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def test_parse_endpoint_forms():
|
|
12
|
+
assert parse_endpoint("udp:514") == ("udp", "0.0.0.0", 514)
|
|
13
|
+
assert parse_endpoint("tcp:5514") == ("tcp", "0.0.0.0", 5514)
|
|
14
|
+
assert parse_endpoint("udp:10.0.0.5:514") == ("udp", "10.0.0.5", 514)
|
|
15
|
+
for bad in ("http:80", "udp:notaport", "udp:99999", "514"):
|
|
16
|
+
with pytest.raises(ValueError):
|
|
17
|
+
parse_endpoint(bad)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _run_when_ready(send_fn):
|
|
21
|
+
"""Return (ready_callback, thread) that fires send_fn(port) once bound."""
|
|
22
|
+
started = threading.Event()
|
|
23
|
+
bound = {}
|
|
24
|
+
|
|
25
|
+
def ready(port):
|
|
26
|
+
bound["port"] = port
|
|
27
|
+
started.set()
|
|
28
|
+
|
|
29
|
+
def runner():
|
|
30
|
+
started.wait(5)
|
|
31
|
+
send_fn(bound["port"])
|
|
32
|
+
|
|
33
|
+
thread = threading.Thread(target=runner, daemon=True)
|
|
34
|
+
thread.start()
|
|
35
|
+
return ready, thread
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def test_udp_listener_receives_lines():
|
|
39
|
+
def send(port):
|
|
40
|
+
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
41
|
+
sock.sendto(b"<166>Jan 1 12:34:56 fw1 app: one", ("127.0.0.1", port))
|
|
42
|
+
sock.sendto(b"line two\nline three\n", ("127.0.0.1", port))
|
|
43
|
+
sock.close()
|
|
44
|
+
|
|
45
|
+
ready, thread = _run_when_ready(send)
|
|
46
|
+
lines = list(
|
|
47
|
+
listen_lines("udp", "127.0.0.1", 0, max_messages=3, ready_callback=ready)
|
|
48
|
+
)
|
|
49
|
+
thread.join(5)
|
|
50
|
+
assert lines == ["<166>Jan 1 12:34:56 fw1 app: one", "line two", "line three"]
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def test_tcp_listener_receives_lines_across_connections():
|
|
54
|
+
def send(port):
|
|
55
|
+
c1 = socket.create_connection(("127.0.0.1", port), timeout=5)
|
|
56
|
+
c1.sendall(b"alpha\nbeta\n")
|
|
57
|
+
c1.close()
|
|
58
|
+
c2 = socket.create_connection(("127.0.0.1", port), timeout=5)
|
|
59
|
+
c2.sendall(b"gamma without newline")
|
|
60
|
+
c2.close()
|
|
61
|
+
|
|
62
|
+
ready, thread = _run_when_ready(send)
|
|
63
|
+
lines = list(
|
|
64
|
+
listen_lines("tcp", "127.0.0.1", 0, max_messages=3, ready_callback=ready)
|
|
65
|
+
)
|
|
66
|
+
thread.join(5)
|
|
67
|
+
assert sorted(lines) == ["alpha", "beta", "gamma without newline"]
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def test_listener_lines_convert_end_to_end():
|
|
71
|
+
from syslogcef import convert_line
|
|
72
|
+
|
|
73
|
+
def send(port):
|
|
74
|
+
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
75
|
+
sock.sendto(
|
|
76
|
+
b"<166>Jan 1 12:34:56 fw1 %ASA-6-302013: Built inbound src=1.2.3.4",
|
|
77
|
+
("127.0.0.1", port),
|
|
78
|
+
)
|
|
79
|
+
sock.close()
|
|
80
|
+
|
|
81
|
+
ready, thread = _run_when_ready(send)
|
|
82
|
+
lines = list(
|
|
83
|
+
listen_lines("udp", "127.0.0.1", 0, max_messages=1, ready_callback=ready)
|
|
84
|
+
)
|
|
85
|
+
thread.join(5)
|
|
86
|
+
cef = convert_line(lines[0])
|
|
87
|
+
assert cef.startswith("CEF:0|Cisco|ASA|")
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
# --- forwarding ---------------------------------------------------------------
|
|
91
|
+
|
|
92
|
+
def test_create_sender_parses_targets():
|
|
93
|
+
from syslogcef.net import SyslogSender, create_sender
|
|
94
|
+
|
|
95
|
+
s = create_sender("udp://siem.example.com:514")
|
|
96
|
+
assert isinstance(s, SyslogSender) and s.proto == "udp" and s.port == 514
|
|
97
|
+
s.close()
|
|
98
|
+
for bad in ("udp://nohost", "ftp://x:1", "kafka://broker:9092"):
|
|
99
|
+
with pytest.raises(ValueError):
|
|
100
|
+
create_sender(bad)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def test_udp_sender_delivers_to_listener():
|
|
104
|
+
from syslogcef.net import create_sender
|
|
105
|
+
|
|
106
|
+
rx = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
107
|
+
rx.bind(("127.0.0.1", 0))
|
|
108
|
+
rx.settimeout(5)
|
|
109
|
+
port = rx.getsockname()[1]
|
|
110
|
+
sender = create_sender(f"udp://127.0.0.1:{port}")
|
|
111
|
+
sender.send("CEF:0|Cisco|ASA|auto|x|y|2|src=1.2.3.4")
|
|
112
|
+
data, _ = rx.recvfrom(65535)
|
|
113
|
+
sender.close()
|
|
114
|
+
rx.close()
|
|
115
|
+
assert data.decode() == "CEF:0|Cisco|ASA|auto|x|y|2|src=1.2.3.4"
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def test_tcp_sender_delivers_newline_delimited():
|
|
119
|
+
from syslogcef.net import create_sender
|
|
120
|
+
|
|
121
|
+
srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
122
|
+
srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
|
123
|
+
srv.bind(("127.0.0.1", 0))
|
|
124
|
+
srv.listen(1)
|
|
125
|
+
srv.settimeout(5)
|
|
126
|
+
port = srv.getsockname()[1]
|
|
127
|
+
|
|
128
|
+
sender = create_sender(f"tcp://127.0.0.1:{port}")
|
|
129
|
+
sender.send("record one")
|
|
130
|
+
conn, _ = srv.accept()
|
|
131
|
+
conn.settimeout(5)
|
|
132
|
+
sender.send("record two")
|
|
133
|
+
sender.close()
|
|
134
|
+
received = b""
|
|
135
|
+
while b"two\n" not in received:
|
|
136
|
+
received += conn.recv(1024)
|
|
137
|
+
conn.close()
|
|
138
|
+
srv.close()
|
|
139
|
+
assert received == b"record one\nrecord two\n"
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def test_rate_limiter_paces_sends():
|
|
143
|
+
from syslogcef.net import RateLimiter
|
|
144
|
+
|
|
145
|
+
sleeps = []
|
|
146
|
+
clock_value = [0.0]
|
|
147
|
+
|
|
148
|
+
def clock():
|
|
149
|
+
return clock_value[0]
|
|
150
|
+
|
|
151
|
+
def sleep(seconds):
|
|
152
|
+
sleeps.append(seconds)
|
|
153
|
+
clock_value[0] += seconds
|
|
154
|
+
|
|
155
|
+
limiter = RateLimiter(10, clock=clock, sleep=sleep) # 0.1s interval
|
|
156
|
+
for _ in range(3):
|
|
157
|
+
limiter.wait()
|
|
158
|
+
assert sum(sleeps) == pytest.approx(0.2, abs=0.01)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def test_kafka_sender_uses_injected_producer():
|
|
162
|
+
from syslogcef.net import KafkaSender
|
|
163
|
+
|
|
164
|
+
class FakeProducer:
|
|
165
|
+
def __init__(self):
|
|
166
|
+
self.sent = []
|
|
167
|
+
self.flushed = False
|
|
168
|
+
|
|
169
|
+
def send(self, topic, payload):
|
|
170
|
+
self.sent.append((topic, payload))
|
|
171
|
+
|
|
172
|
+
def flush(self):
|
|
173
|
+
self.flushed = True
|
|
174
|
+
|
|
175
|
+
producer = FakeProducer()
|
|
176
|
+
sender = KafkaSender("broker:9092", "cef-events", producer=producer)
|
|
177
|
+
sender.send("CEF:0|a|b|1|c|d|5|")
|
|
178
|
+
sender.close()
|
|
179
|
+
assert producer.sent == [("cef-events", b"CEF:0|a|b|1|c|d|5|")]
|
|
180
|
+
assert producer.flushed
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from syslogcef import convert_line, normalize_event, parse_syslog, to_cef
|
|
6
|
+
from syslogcef.validation import CEFValidationError, validate_extensions
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def test_valid_extensions_produce_no_fatal_findings():
|
|
10
|
+
findings = validate_extensions(
|
|
11
|
+
{"src": "10.0.0.1", "dst": "2001:db8::1", "spt": "443", "msg": "hello", "smac": "00:0d:60:af:1b:61"}
|
|
12
|
+
)
|
|
13
|
+
assert [f for f in findings if f.fatal] == []
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def test_type_violations_are_fatal():
|
|
17
|
+
findings = validate_extensions({"src": "not-an-ip", "spt": "99999", "fsize": "big"})
|
|
18
|
+
assert sorted(f.key for f in findings if f.fatal) == ["fsize", "spt", "src"]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def test_unknown_keys_are_advisory_only():
|
|
22
|
+
findings = validate_extensions({"myCustomKey": "anything"})
|
|
23
|
+
assert len(findings) == 1
|
|
24
|
+
assert not findings[0].fatal
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def test_length_violation_detected():
|
|
28
|
+
findings = validate_extensions({"outcome": "x" * 100}) # max 63
|
|
29
|
+
assert findings and findings[0].fatal
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def test_timestamp_formats():
|
|
33
|
+
ok = validate_extensions({"rt": "Jan 17 2020 08:52:10", "start": "1587231168000"})
|
|
34
|
+
assert [f for f in ok if f.fatal] == []
|
|
35
|
+
bad = validate_extensions({"rt": "yesterday at noon"})
|
|
36
|
+
assert bad and bad[0].fatal
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def test_strict_mode_raises_through_pipeline():
|
|
40
|
+
# The ASA mapping maps src= from the message; a bogus IP violates the
|
|
41
|
+
# dictionary in strict mode.
|
|
42
|
+
line = "<166>Jan 1 12:34:56 fw1 asa[1]: %ASA-6-302013: Built inbound src=notanip dst=10.0.0.2"
|
|
43
|
+
with pytest.raises(CEFValidationError):
|
|
44
|
+
convert_line(line, validate=True, strict=True)
|
|
45
|
+
# Non-strict validation still renders the record.
|
|
46
|
+
assert convert_line(line, validate=True).startswith("CEF:0|")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def test_strict_mode_passes_clean_lines():
|
|
50
|
+
line = "<166>Jan 1 12:34:56 fw1 asa[1]: %ASA-6-302013: Built inbound src=10.0.0.1 dst=10.0.0.2"
|
|
51
|
+
cef = convert_line(line, validate=True, strict=True)
|
|
52
|
+
assert "src=10.0.0.1" in cef
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def test_cli_strict_exits_nonzero(tmp_path):
|
|
56
|
+
import subprocess
|
|
57
|
+
import sys
|
|
58
|
+
|
|
59
|
+
bad = tmp_path / "bad.log"
|
|
60
|
+
bad.write_text("<166>Jan 1 12:34:56 fw1 asa[1]: %ASA-6-302013: x src=notanip\n")
|
|
61
|
+
result = subprocess.run(
|
|
62
|
+
[sys.executable, "-m", "syslogcef", str(bad), "--strict"],
|
|
63
|
+
capture_output=True,
|
|
64
|
+
)
|
|
65
|
+
assert result.returncode == 1
|
|
66
|
+
assert b"strict validation failed" in result.stderr
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|