syslog2cef 0.2.1__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {syslog2cef-0.2.1/syslog2cef.egg-info → syslog2cef-0.3.0}/PKG-INFO +1 -1
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/pyproject.toml +1 -1
- {syslog2cef-0.2.1 → syslog2cef-0.3.0/syslog2cef.egg-info}/PKG-INFO +1 -1
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslog2cef.egg-info/SOURCES.txt +5 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/__init__.py +2 -1
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/adaptive.py +37 -10
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/api.py +62 -1
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/cef.py +3 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/cli.py +188 -18
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/mappings/linux.json +1 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/net.py +51 -1
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/normalizer.py +4 -2
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/parsers.py +131 -31
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/utils.py +38 -6
- syslog2cef-0.3.0/tests/test_cli_validation.py +120 -0
- syslog2cef-0.3.0/tests/test_macos_logs.py +142 -0
- syslog2cef-0.3.0/tests/test_rfc5424_epoch.py +160 -0
- syslog2cef-0.3.0/tests/test_security_hardening.py +107 -0
- syslog2cef-0.3.0/tests/test_tail_rotation.py +246 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/LICENSE +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/README.md +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/setup.cfg +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslog2cef.egg-info/dependency_links.txt +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslog2cef.egg-info/entry_points.txt +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslog2cef.egg-info/requires.txt +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslog2cef.egg-info/top_level.txt +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/__main__.py +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/custom.py +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/dictionary.json +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/dictionary.py +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/mappings/__init__.py +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/mappings/cisco_asa.json +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/mappings/cisco_ios.json +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/mappings/f5.json +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/mappings/fortinet.json +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/mappings/sophos.json +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/mappings/vmware.json +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/validation.py +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_adaptive.py +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_cef.py +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_cli.py +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_custom_parsers.py +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_dictionary.py +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_fuzz_grammar.py +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_fuzz_properties.py +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_net.py +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_normalizer.py +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_parsers.py +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_parsers_extra.py +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_pipeline.py +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_real_world.py +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_utils.py +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_validation.py +0 -0
- {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_zipapp_safe.py +0 -0
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "syslog2cef"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.3.0"
|
|
8
8
|
description = "Convert RFC3164/RFC5424 syslog lines into ArcSight CEF"
|
|
9
9
|
authors = [{name = "Tamir Suliman", email = "allamiro@gmail.com"}]
|
|
10
10
|
readme = "README.md"
|
|
@@ -54,16 +54,21 @@ syslogcef/mappings/vmware.json
|
|
|
54
54
|
tests/test_adaptive.py
|
|
55
55
|
tests/test_cef.py
|
|
56
56
|
tests/test_cli.py
|
|
57
|
+
tests/test_cli_validation.py
|
|
57
58
|
tests/test_custom_parsers.py
|
|
58
59
|
tests/test_dictionary.py
|
|
59
60
|
tests/test_fuzz_grammar.py
|
|
60
61
|
tests/test_fuzz_properties.py
|
|
62
|
+
tests/test_macos_logs.py
|
|
61
63
|
tests/test_net.py
|
|
62
64
|
tests/test_normalizer.py
|
|
63
65
|
tests/test_parsers.py
|
|
64
66
|
tests/test_parsers_extra.py
|
|
65
67
|
tests/test_pipeline.py
|
|
66
68
|
tests/test_real_world.py
|
|
69
|
+
tests/test_rfc5424_epoch.py
|
|
70
|
+
tests/test_security_hardening.py
|
|
71
|
+
tests/test_tail_rotation.py
|
|
67
72
|
tests/test_utils.py
|
|
68
73
|
tests/test_validation.py
|
|
69
74
|
tests/test_zipapp_safe.py
|
|
@@ -9,10 +9,11 @@ conversion workflow.
|
|
|
9
9
|
"""
|
|
10
10
|
from __future__ import annotations
|
|
11
11
|
|
|
12
|
-
from .api import convert_line, normalize_event, parse_syslog, to_cef
|
|
12
|
+
from .api import StreamConverter, convert_line, normalize_event, parse_syslog, to_cef
|
|
13
13
|
from .custom import PatternFileError, load_patterns, register_parser
|
|
14
14
|
|
|
15
15
|
__all__ = [
|
|
16
|
+
"StreamConverter",
|
|
16
17
|
"convert_line",
|
|
17
18
|
"normalize_event",
|
|
18
19
|
"parse_syslog",
|
|
@@ -56,17 +56,26 @@ def _dmy_dash(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
|
|
|
56
56
|
|
|
57
57
|
|
|
58
58
|
TIMESTAMP_LIB: Tuple[Tuple[str, str, Callable], ...] = (
|
|
59
|
-
("iso", r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}
|
|
59
|
+
("iso", r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}(?::?\d{2})?)?", _iso),
|
|
60
60
|
("mon_day_year", r"(?P<month>[A-Z][a-z]{2})\s+(?P<day>\d{1,2})\s+(?P<year>\d{4})\s+(?P<hms>\d{2}:\d{2}:\d{2})(?:\.\d+)?", _mon_day),
|
|
61
61
|
("mon_day", r"(?P<month>[A-Z][a-z]{2})\s+(?P<day>\d{1,2})\s+(?P<hms>\d{2}:\d{2}:\d{2})(?:\.\d+)?", _mon_day),
|
|
62
62
|
("ymd_slash", r"(?P<y>\d{4})/(?P<m>\d{2})/(?P<d>\d{2})[ T](?P<hms>\d{2}:\d{2}:\d{2})", _ymd_slash),
|
|
63
63
|
("dmy_dash", r"(?P<d>\d{1,2})-(?P<month>[A-Za-z]{3})-(?P<y>\d{4})\s+(?P<hms>\d{2}:\d{2}:\d{2})", _dmy_dash),
|
|
64
|
-
|
|
64
|
+
# 10/13/16/19 digits = seconds/ms/us/ns, matching parse_iso8601's
|
|
65
|
+
# precision dispatch (a bare 10-digit run only, then the optional
|
|
66
|
+
# ms/us/ns extensions), so adaptively-parsed epochs get the same
|
|
67
|
+
# correct scaling as the built-in parsers.
|
|
68
|
+
("epoch", r"\b1\d{9}(?:\d{3}|\d{6}|\d{9})?\b", _epoch),
|
|
65
69
|
)
|
|
66
70
|
|
|
67
71
|
HOST_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
|
|
68
72
|
_SKIP_HOST_TOKENS = {"error", "warning", "info", "debug", "notice", "kernel"}
|
|
69
73
|
|
|
74
|
+
# Syslog-style program tag following the host: "app:", "app[pid]:".
|
|
75
|
+
TAG_RE = re.compile(r"^(?P<app>[\w\-/\.]+)(?:\[(?P<pid>[^\]]+)\])?:$")
|
|
76
|
+
# The same shape as a regex fragment for synthesized patterns.
|
|
77
|
+
_TAG_SRC = r"(?:(?P<adaptive_app>[\w\-/\.]+)(?:\[(?P<adaptive_pid>[^\]]+)\])?:\s+)?"
|
|
78
|
+
|
|
70
79
|
|
|
71
80
|
def _valid_host(candidate: Optional[str]) -> bool:
|
|
72
81
|
return bool(
|
|
@@ -105,7 +114,7 @@ def cache_size() -> int:
|
|
|
105
114
|
return len(_CACHE)
|
|
106
115
|
|
|
107
116
|
|
|
108
|
-
def _build_event(line, pri, ts, host, msg):
|
|
117
|
+
def _build_event(line, pri, ts, host, msg, app=None, pid=None):
|
|
109
118
|
from .parsers import ParsedEvent
|
|
110
119
|
|
|
111
120
|
facility, severity = convert_pri(pri)
|
|
@@ -116,8 +125,8 @@ def _build_event(line, pri, ts, host, msg):
|
|
|
116
125
|
ts=ts,
|
|
117
126
|
ts_orig="",
|
|
118
127
|
host=host,
|
|
119
|
-
app=
|
|
120
|
-
pid=
|
|
128
|
+
app=app,
|
|
129
|
+
pid=pid,
|
|
121
130
|
msgid=None,
|
|
122
131
|
sd={},
|
|
123
132
|
msg=sanitize_message(msg),
|
|
@@ -148,14 +157,17 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
|
|
|
148
157
|
ts = None
|
|
149
158
|
gd = match.groupdict()
|
|
150
159
|
host = gd.get("adaptive_host")
|
|
160
|
+
app = gd.get("adaptive_app")
|
|
161
|
+
pid = gd.get("adaptive_pid")
|
|
151
162
|
msg = gd.get("adaptive_msg")
|
|
152
163
|
msg = line if msg is None else msg
|
|
153
164
|
# The shape signature reduces all words to the same class, so a
|
|
154
165
|
# cached host slot may capture a non-host token; revalidate.
|
|
155
166
|
if host is not None and not _valid_host(host.rstrip(":,")):
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
167
|
+
tag = f"{app}[{pid}]: " if app and pid else (f"{app}: " if app else "")
|
|
168
|
+
msg = f"{host} {tag}{msg}".strip()
|
|
169
|
+
host = app = pid = None
|
|
170
|
+
return _build_event(raw, pri, ts, host, msg, app=app, pid=pid)
|
|
159
171
|
|
|
160
172
|
# Analysis pass: find the earliest timestamp in the line head.
|
|
161
173
|
best = None
|
|
@@ -172,9 +184,13 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
|
|
|
172
184
|
return None
|
|
173
185
|
|
|
174
186
|
# Host candidate: the first plausible token after the timestamp that is
|
|
175
|
-
# not a key=value pair or severity word.
|
|
187
|
+
# not a key=value pair or severity word. A leading timezone remnant the
|
|
188
|
+
# timestamp library could not consume (a lone "+02"/"CEST") must not be
|
|
189
|
+
# mistaken for the host — such tokens fail _valid_host and stay in msg.
|
|
176
190
|
rest = line[ts_match.end():].lstrip(" :,\t")
|
|
177
191
|
host = None
|
|
192
|
+
app = None
|
|
193
|
+
pid = None
|
|
178
194
|
msg = rest
|
|
179
195
|
tokens = rest.split(None, 1)
|
|
180
196
|
if tokens:
|
|
@@ -182,12 +198,23 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
|
|
|
182
198
|
if _valid_host(candidate):
|
|
183
199
|
host = candidate
|
|
184
200
|
msg = tokens[1] if len(tokens) > 1 else ""
|
|
201
|
+
# Program tag after the host ("app:" / "app[pid]:"): split it
|
|
202
|
+
# off so it maps to the CEF process field instead of polluting
|
|
203
|
+
# the message.
|
|
204
|
+
tag_tokens = msg.split(None, 1)
|
|
205
|
+
if tag_tokens:
|
|
206
|
+
tag_match = TAG_RE.match(tag_tokens[0])
|
|
207
|
+
if tag_match:
|
|
208
|
+
app = tag_match.group("app")
|
|
209
|
+
pid = tag_match.group("pid")
|
|
210
|
+
msg = tag_tokens[1] if len(tag_tokens) > 1 else ""
|
|
185
211
|
|
|
186
212
|
# Synthesize and cache a pattern for this layout.
|
|
187
213
|
prefix = line[: ts_match.start()]
|
|
188
214
|
parts = [re.escape(prefix), f"(?P<adaptive_ts>{src})", r"[\s:,]*"]
|
|
189
215
|
if host is not None:
|
|
190
216
|
parts.append(r"(?P<adaptive_host>\S+?):?(?:\s+|$)")
|
|
217
|
+
parts.append(_TAG_SRC)
|
|
191
218
|
parts.append(r"(?P<adaptive_msg>.*)$")
|
|
192
219
|
try:
|
|
193
220
|
pattern = re.compile("".join(parts))
|
|
@@ -198,7 +225,7 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
|
|
|
198
225
|
_CACHE.pop(next(iter(_CACHE)))
|
|
199
226
|
_CACHE[sig] = (pattern, name, conv)
|
|
200
227
|
|
|
201
|
-
return _build_event(raw, pri, ts, host, msg)
|
|
228
|
+
return _build_event(raw, pri, ts, host, msg, app=app, pid=pid)
|
|
202
229
|
|
|
203
230
|
|
|
204
231
|
def pattern_ts_match(pattern: Pattern[str], match: "re.Match[str]"):
|
|
@@ -68,7 +68,16 @@ def _load_mapping(mapping: Mapping[str, Any] | Path | str | None) -> Mapping[str
|
|
|
68
68
|
return mapping
|
|
69
69
|
path = Path(mapping)
|
|
70
70
|
with path.open("r", encoding="utf-8") as fp:
|
|
71
|
-
|
|
71
|
+
data = json.load(fp)
|
|
72
|
+
# Structural validation with clear messages: a top-level array would
|
|
73
|
+
# otherwise TypeError deep inside the renderer, and a wrong-shaped
|
|
74
|
+
# extensions/severity_map would fail per event instead of up front.
|
|
75
|
+
if not isinstance(data, dict):
|
|
76
|
+
raise ValueError(f"{path}: mapping must be a JSON object")
|
|
77
|
+
for key in ("extensions", "severity_map"):
|
|
78
|
+
if key in data and not isinstance(data[key], dict):
|
|
79
|
+
raise ValueError(f"{path}: '{key}' must be a JSON object")
|
|
80
|
+
return data
|
|
72
81
|
|
|
73
82
|
|
|
74
83
|
def to_cef(
|
|
@@ -111,6 +120,57 @@ def convert_line(
|
|
|
111
120
|
return to_cef(normalized, mapping, validate=validate, strict=strict)
|
|
112
121
|
|
|
113
122
|
|
|
123
|
+
class StreamConverter:
|
|
124
|
+
"""Stateful converter for an ordered stream of syslog lines.
|
|
125
|
+
|
|
126
|
+
Multi-line records (macOS install.log, wrapped plist/JSON payloads in
|
|
127
|
+
Apple system logs, Java stack traces) continue onto lines that begin
|
|
128
|
+
with whitespace and carry no syslog header of their own. Converted in
|
|
129
|
+
isolation such a line gets no timestamp and the local machine's
|
|
130
|
+
hostname — inside a container that is the container ID, not the host
|
|
131
|
+
that produced the log. Here a whitespace-indented line instead
|
|
132
|
+
inherits host, app, pid, PRI, and timestamp from the most recent
|
|
133
|
+
fully-parsed event and is tagged ``source_hint="continuation"``. One
|
|
134
|
+
CEF record is still emitted per input line.
|
|
135
|
+
"""
|
|
136
|
+
|
|
137
|
+
def __init__(
|
|
138
|
+
self,
|
|
139
|
+
*,
|
|
140
|
+
mode: Optional[str] = None,
|
|
141
|
+
mapping: Mapping[str, Any] | Path | str | None = None,
|
|
142
|
+
validate: bool = False,
|
|
143
|
+
strict: bool = False,
|
|
144
|
+
) -> None:
|
|
145
|
+
self.mode = mode
|
|
146
|
+
self.mapping = mapping
|
|
147
|
+
self.validate = validate
|
|
148
|
+
self.strict = strict
|
|
149
|
+
self._context: Optional[ParsedEvent] = None
|
|
150
|
+
|
|
151
|
+
def convert(self, line: str, *, now: Optional[datetime] = None) -> str:
|
|
152
|
+
parsed = parse_syslog(line, mode=self.mode, now=now)
|
|
153
|
+
if line[:1] in ("\t", " ") and self._context is not None:
|
|
154
|
+
ctx = self._context
|
|
155
|
+
parsed.host = ctx.host
|
|
156
|
+
parsed.app = parsed.app or ctx.app
|
|
157
|
+
parsed.pid = parsed.pid or ctx.pid
|
|
158
|
+
if parsed.ts is None:
|
|
159
|
+
parsed.ts = ctx.ts
|
|
160
|
+
if parsed.pri is None:
|
|
161
|
+
parsed.pri = ctx.pri
|
|
162
|
+
parsed.facility = ctx.facility
|
|
163
|
+
parsed.severity = ctx.severity
|
|
164
|
+
parsed.msg = parsed.msg.lstrip()
|
|
165
|
+
parsed.source_hint = "continuation"
|
|
166
|
+
elif parsed.host is not None and parsed.source_hint != "unknown":
|
|
167
|
+
# Only a line whose host was actually parsed (not the
|
|
168
|
+
# guessed fallback) may become continuation context.
|
|
169
|
+
self._context = parsed
|
|
170
|
+
normalized = normalize_event(parsed)
|
|
171
|
+
return to_cef(normalized, self.mapping, validate=self.validate, strict=self.strict)
|
|
172
|
+
|
|
173
|
+
|
|
114
174
|
def _guess_mapping(event: NormalizedEvent) -> Mapping[str, Any]:
|
|
115
175
|
msg_upper = event.msg.upper()
|
|
116
176
|
app_upper = (event.app or "").upper()
|
|
@@ -139,6 +199,7 @@ __all__ = [
|
|
|
139
199
|
"ParsedEvent",
|
|
140
200
|
"NormalizedEvent",
|
|
141
201
|
"ParseResult",
|
|
202
|
+
"StreamConverter",
|
|
142
203
|
"convert_line",
|
|
143
204
|
"normalize_event",
|
|
144
205
|
"parse_syslog",
|
|
@@ -28,6 +28,9 @@ DEFAULT_MAPPING = {
|
|
|
28
28
|
},
|
|
29
29
|
"extensions": {
|
|
30
30
|
"msg": "%(msg)s",
|
|
31
|
+
# Event time (epoch milliseconds) parsed from the line, or the
|
|
32
|
+
# processing time when the line carried no usable timestamp.
|
|
33
|
+
"rt": "%(rt)s",
|
|
31
34
|
# The raw line travels in a labeled custom string: rawEvent is a
|
|
32
35
|
# consumer-side key producers must not set (see docs/cef_fields.md),
|
|
33
36
|
# and every bundled mapping already uses this shape.
|
|
@@ -8,9 +8,9 @@ from datetime import datetime
|
|
|
8
8
|
from functools import partial
|
|
9
9
|
from multiprocessing import Pool, cpu_count
|
|
10
10
|
from pathlib import Path
|
|
11
|
-
from typing import Iterable, Iterator, Optional
|
|
11
|
+
from typing import Any, Iterable, Iterator, Optional
|
|
12
12
|
|
|
13
|
-
from .api import convert_line
|
|
13
|
+
from .api import StreamConverter, convert_line
|
|
14
14
|
|
|
15
15
|
logger = logging.getLogger(__name__)
|
|
16
16
|
|
|
@@ -39,6 +39,127 @@ def tail_file(path: Path, *, follow: bool = False) -> Iterator[str]:
|
|
|
39
39
|
yield line.rstrip("\n")
|
|
40
40
|
|
|
41
41
|
|
|
42
|
+
class _Follower:
|
|
43
|
+
"""One tailed path, surviving rotation (rename, replace, truncate)."""
|
|
44
|
+
|
|
45
|
+
_FINGERPRINT_LEN = 256
|
|
46
|
+
|
|
47
|
+
def __init__(self, path: Path):
|
|
48
|
+
from collections import deque
|
|
49
|
+
|
|
50
|
+
self.path = path
|
|
51
|
+
self.handle = path.open("r", encoding="utf-8", errors="replace")
|
|
52
|
+
self.signature = self._fsignature()
|
|
53
|
+
# Fingerprint of the *already-consumed* head of the file. Those
|
|
54
|
+
# bytes are stable across appends (which only add data past our
|
|
55
|
+
# offset) but change when a copy-truncate rewrites the file in
|
|
56
|
+
# place, so comparing them detects rewrites the size/inode check
|
|
57
|
+
# misses. Length is bounded by both _FINGERPRINT_LEN and our read
|
|
58
|
+
# offset, so a plain append to a short file is never mistaken for
|
|
59
|
+
# a rewrite.
|
|
60
|
+
self.fp_len = 0
|
|
61
|
+
self.fingerprint = b""
|
|
62
|
+
self.pending = deque()
|
|
63
|
+
|
|
64
|
+
def _fsignature(self):
|
|
65
|
+
import os
|
|
66
|
+
|
|
67
|
+
st = os.fstat(self.handle.fileno())
|
|
68
|
+
return (st.st_dev, st.st_ino)
|
|
69
|
+
|
|
70
|
+
def _read_head(self, length: int) -> bytes:
|
|
71
|
+
# Positional read so the follow position is untouched.
|
|
72
|
+
import os
|
|
73
|
+
|
|
74
|
+
if self.handle is None or length <= 0:
|
|
75
|
+
return b""
|
|
76
|
+
try:
|
|
77
|
+
return os.pread(self.handle.fileno(), length, 0)
|
|
78
|
+
except (OSError, AttributeError):
|
|
79
|
+
return b""
|
|
80
|
+
|
|
81
|
+
def _capture_fingerprint(self) -> None:
|
|
82
|
+
if self.handle is None:
|
|
83
|
+
self.fp_len, self.fingerprint = 0, b""
|
|
84
|
+
return
|
|
85
|
+
self.fp_len = min(self._FINGERPRINT_LEN, self.handle.tell())
|
|
86
|
+
self.fingerprint = self._read_head(self.fp_len)
|
|
87
|
+
|
|
88
|
+
def read_lines(self) -> Iterator[str]:
|
|
89
|
+
while self.pending:
|
|
90
|
+
yield self.pending.popleft()
|
|
91
|
+
if self.handle is None:
|
|
92
|
+
return
|
|
93
|
+
while True:
|
|
94
|
+
line = self.handle.readline()
|
|
95
|
+
if not line:
|
|
96
|
+
# At EOF: record the consumed-head fingerprint so the next
|
|
97
|
+
# rotation check can tell an append from an in-place rewrite.
|
|
98
|
+
self._capture_fingerprint()
|
|
99
|
+
return
|
|
100
|
+
yield line.rstrip("\n")
|
|
101
|
+
|
|
102
|
+
def check_rotation(self) -> bool:
|
|
103
|
+
"""Reattach after rotation events. Returns True if a replacement
|
|
104
|
+
file was (re)opened, so the caller can read it before treating
|
|
105
|
+
the poll as idle.
|
|
106
|
+
|
|
107
|
+
Runs every pass, not only when idle: a file that is renamed away
|
|
108
|
+
while still being written keeps this follower producing from the
|
|
109
|
+
stale descriptor forever, so a replacement at the original path
|
|
110
|
+
would never be picked up if rotation were checked only on idle
|
|
111
|
+
polls.
|
|
112
|
+
|
|
113
|
+
- rename-and-recreate: the path's dev/inode changes; drain any
|
|
114
|
+
remaining lines from the old descriptor, then open the new
|
|
115
|
+
file from the start.
|
|
116
|
+
- copy-truncate: same inode but size below our offset; rewind.
|
|
117
|
+
- temporary disappearance mid-rotation: keep the old descriptor
|
|
118
|
+
(it stays readable) and retry until a replacement appears.
|
|
119
|
+
"""
|
|
120
|
+
if self.handle is None:
|
|
121
|
+
try:
|
|
122
|
+
self.handle = self.path.open("r", encoding="utf-8", errors="replace")
|
|
123
|
+
self.signature = self._fsignature()
|
|
124
|
+
self.fp_len, self.fingerprint = 0, b""
|
|
125
|
+
return True
|
|
126
|
+
except OSError:
|
|
127
|
+
return False
|
|
128
|
+
try:
|
|
129
|
+
st = self.path.stat()
|
|
130
|
+
except OSError:
|
|
131
|
+
return False
|
|
132
|
+
if (st.st_dev, st.st_ino) != self.signature:
|
|
133
|
+
for line in self.handle:
|
|
134
|
+
self.pending.append(line.rstrip("\n"))
|
|
135
|
+
self.handle.close()
|
|
136
|
+
self.handle = None
|
|
137
|
+
return self.check_rotation() # open the replacement immediately
|
|
138
|
+
# Copy-truncate: same inode, but either the file shrank below our
|
|
139
|
+
# offset, or the already-consumed head changed (an in-place rewrite
|
|
140
|
+
# that regrew past our offset). Comparing only the consumed prefix
|
|
141
|
+
# means a plain append — which never touches those bytes — is not
|
|
142
|
+
# mistaken for a rewrite, even for a file shorter than the cap.
|
|
143
|
+
offset = self.handle.tell()
|
|
144
|
+
rewound = False
|
|
145
|
+
if offset > 0:
|
|
146
|
+
if st.st_size < offset:
|
|
147
|
+
rewound = True
|
|
148
|
+
elif self.fp_len and self._read_head(self.fp_len) != self.fingerprint:
|
|
149
|
+
rewound = True
|
|
150
|
+
if rewound:
|
|
151
|
+
self.handle.seek(0)
|
|
152
|
+
self.fp_len, self.fingerprint = 0, b""
|
|
153
|
+
# A rewind exposes fresh content that must be read before the idle
|
|
154
|
+
# limit applies, so report it as activity like a reopen.
|
|
155
|
+
return rewound
|
|
156
|
+
|
|
157
|
+
def close(self) -> None:
|
|
158
|
+
if self.handle is not None:
|
|
159
|
+
self.handle.close()
|
|
160
|
+
self.handle = None
|
|
161
|
+
|
|
162
|
+
|
|
42
163
|
def follow_files(
|
|
43
164
|
paths: Iterable[Path],
|
|
44
165
|
*,
|
|
@@ -47,23 +168,36 @@ def follow_files(
|
|
|
47
168
|
) -> Iterator[str]:
|
|
48
169
|
"""Tail several files at once, round-robin, yielding lines as they appear.
|
|
49
170
|
|
|
171
|
+
Survives log rotation on every path independently: renamed-and-
|
|
172
|
+
recreated files are reopened from the start (after draining the old
|
|
173
|
+
descriptor), copy-truncated files are rewound, and paths that
|
|
174
|
+
disappear mid-rotation are retried until a replacement appears.
|
|
175
|
+
|
|
50
176
|
``max_idle_polls`` bounds how many consecutive empty polls are allowed
|
|
51
177
|
before the generator stops; ``None`` follows forever.
|
|
52
178
|
"""
|
|
53
179
|
|
|
54
|
-
|
|
180
|
+
followers = [_Follower(path) for path in paths]
|
|
55
181
|
idle_polls = 0
|
|
56
182
|
try:
|
|
57
183
|
while True:
|
|
58
|
-
|
|
59
|
-
for
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
184
|
+
any_line = False
|
|
185
|
+
for follower in followers:
|
|
186
|
+
for line in follower.read_lines():
|
|
187
|
+
any_line = True
|
|
188
|
+
yield line
|
|
189
|
+
# Rotation is checked for every follower on every pass —
|
|
190
|
+
# including busy ones — so a file renamed away while still
|
|
191
|
+
# being written cannot pin this follower to the stale
|
|
192
|
+
# descriptor and starve the replacement at the origin path.
|
|
193
|
+
reopened = False
|
|
194
|
+
for follower in followers:
|
|
195
|
+
if follower.check_rotation():
|
|
196
|
+
reopened = True
|
|
197
|
+
# A reopened replacement must be read before the idle limit
|
|
198
|
+
# applies, so treat a reopen as activity: loop again to read
|
|
199
|
+
# it rather than counting this pass as idle.
|
|
200
|
+
if any_line or reopened:
|
|
67
201
|
idle_polls = 0
|
|
68
202
|
continue
|
|
69
203
|
idle_polls += 1
|
|
@@ -71,8 +205,8 @@ def follow_files(
|
|
|
71
205
|
return
|
|
72
206
|
time.sleep(poll_interval)
|
|
73
207
|
finally:
|
|
74
|
-
for
|
|
75
|
-
|
|
208
|
+
for follower in followers:
|
|
209
|
+
follower.close()
|
|
76
210
|
|
|
77
211
|
|
|
78
212
|
# Portable strftime codes with at-coarsest one-second granularity.
|
|
@@ -173,7 +307,7 @@ def write_lines(
|
|
|
173
307
|
fp.close()
|
|
174
308
|
|
|
175
309
|
|
|
176
|
-
def _convert_line_mp(line: str, mode: Optional[str], mapping:
|
|
310
|
+
def _convert_line_mp(line: str, mode: Optional[str], mapping: Any, validate: bool = False, strict: bool = False) -> str:
|
|
177
311
|
return convert_line(line, mode=mode, mapping=mapping, validate=validate, strict=strict)
|
|
178
312
|
|
|
179
313
|
|
|
@@ -193,7 +327,7 @@ def process_lines(
|
|
|
193
327
|
lines: Iterable[str],
|
|
194
328
|
*,
|
|
195
329
|
mode: Optional[str],
|
|
196
|
-
mapping:
|
|
330
|
+
mapping: Any,
|
|
197
331
|
use_multiprocessing: bool,
|
|
198
332
|
pool_size: Optional[int],
|
|
199
333
|
validate: bool = False,
|
|
@@ -207,8 +341,12 @@ def process_lines(
|
|
|
207
341
|
for cef in pool.imap(worker, lines):
|
|
208
342
|
yield cef
|
|
209
343
|
else:
|
|
344
|
+
# Stateful conversion so whitespace-indented continuation lines
|
|
345
|
+
# inherit host/app/timestamp from the preceding event. Unavailable
|
|
346
|
+
# under --multiprocess, where workers cannot share line order.
|
|
347
|
+
converter = StreamConverter(mode=mode, mapping=mapping, validate=validate, strict=strict)
|
|
210
348
|
for line in lines:
|
|
211
|
-
yield
|
|
349
|
+
yield converter.convert(line)
|
|
212
350
|
|
|
213
351
|
|
|
214
352
|
def _optional_path(value: str) -> Optional[Path]:
|
|
@@ -238,6 +376,20 @@ def main(argv: Optional[list[str]] = None) -> int:
|
|
|
238
376
|
args = parser.parse_args(argv)
|
|
239
377
|
logging.basicConfig(level=getattr(logging, args.log_level.upper(), logging.WARNING))
|
|
240
378
|
|
|
379
|
+
# Reject invalid option combinations before any file, socket, or
|
|
380
|
+
# pool side effects (#89): mistakes in service environment files
|
|
381
|
+
# must fail with a clear message, not a traceback or silence.
|
|
382
|
+
if args.pool_size is not None and not args.multiprocess:
|
|
383
|
+
parser.error("--pool-size requires --multiprocess")
|
|
384
|
+
if args.multiprocess and args.pool_size is not None and args.pool_size < 1:
|
|
385
|
+
parser.error(f"--pool-size must be a positive integer (got {args.pool_size})")
|
|
386
|
+
if args.eps is not None and not args.send:
|
|
387
|
+
parser.error("--eps requires --send")
|
|
388
|
+
if args.listen and args.paths:
|
|
389
|
+
parser.error("--listen replaces file input; remove the positional paths")
|
|
390
|
+
if args.tail and not args.paths:
|
|
391
|
+
parser.error("--tail requires input file paths")
|
|
392
|
+
|
|
241
393
|
if args.output:
|
|
242
394
|
try:
|
|
243
395
|
template_has_codes(str(args.output))
|
|
@@ -260,7 +412,25 @@ def main(argv: Optional[list[str]] = None) -> int:
|
|
|
260
412
|
if args.mode not in PARSERS and get_registered(args.mode) is None:
|
|
261
413
|
parser.error(f"unknown --mode '{args.mode}' (not a built-in parser or a loaded pattern name)")
|
|
262
414
|
|
|
263
|
-
mapping
|
|
415
|
+
# Load and validate the mapping eagerly (#84): a missing or malformed
|
|
416
|
+
# file must fail before the output is opened — a truncate-mode open
|
|
417
|
+
# would destroy an existing archive before the lazy first-event load
|
|
418
|
+
# ever ran. Loading once here also avoids re-reading the JSON file
|
|
419
|
+
# for every converted line.
|
|
420
|
+
mapping = None
|
|
421
|
+
if args.mapping is not None:
|
|
422
|
+
# An explicitly empty --mapping (e.g. MAPPING= blank in a service
|
|
423
|
+
# environment file) is a misconfiguration, not "use the default":
|
|
424
|
+
# silently proceeding would truncate an existing --output with an
|
|
425
|
+
# unintended mapping. Fail before any file is opened.
|
|
426
|
+
if not args.mapping.strip():
|
|
427
|
+
parser.error("--mapping was given an empty value")
|
|
428
|
+
from .api import _load_mapping
|
|
429
|
+
|
|
430
|
+
try:
|
|
431
|
+
mapping = _load_mapping(args.mapping)
|
|
432
|
+
except (OSError, ValueError) as exc:
|
|
433
|
+
parser.error(f"invalid --mapping: {exc}")
|
|
264
434
|
|
|
265
435
|
outputs: Iterator[str]
|
|
266
436
|
if args.listen:
|
|
@@ -12,6 +12,8 @@ import selectors
|
|
|
12
12
|
import socket
|
|
13
13
|
from typing import Callable, Iterator, Optional, Tuple
|
|
14
14
|
|
|
15
|
+
import time as _time
|
|
16
|
+
|
|
15
17
|
logger = logging.getLogger(__name__)
|
|
16
18
|
|
|
17
19
|
MAX_DATAGRAM = 65535
|
|
@@ -21,6 +23,12 @@ MAX_DATAGRAM = 65535
|
|
|
21
23
|
# dropped.
|
|
22
24
|
MAX_TCP_BUFFER = 1024 * 1024
|
|
23
25
|
|
|
26
|
+
# Aggregate limits so many well-behaved-per-connection clients cannot
|
|
27
|
+
# still exhaust memory or descriptors in total. New connections beyond
|
|
28
|
+
# the cap are refused; a connection idle past the timeout is closed.
|
|
29
|
+
MAX_TCP_CONNECTIONS = 512
|
|
30
|
+
TCP_IDLE_TIMEOUT = 300.0 # seconds without any received data
|
|
31
|
+
|
|
24
32
|
|
|
25
33
|
def parse_endpoint(spec: str) -> Tuple[str, str, int]:
|
|
26
34
|
"""Parse ``udp:514``, ``tcp:5514``, or ``udp:10.0.0.5:514``."""
|
|
@@ -96,18 +104,58 @@ def _listen_tcp(host, port, max_messages, ready_callback) -> Iterator[str]:
|
|
|
96
104
|
sel = selectors.DefaultSelector()
|
|
97
105
|
sel.register(server, selectors.EVENT_READ)
|
|
98
106
|
buffers: dict[socket.socket, bytes] = {}
|
|
107
|
+
last_seen: dict[socket.socket, float] = {}
|
|
99
108
|
count = 0
|
|
109
|
+
# -inf, not 0.0: within the first TCP_IDLE_TIMEOUT seconds of process
|
|
110
|
+
# uptime monotonic() is small, and a 0.0 seed would suppress the very
|
|
111
|
+
# first capacity warning. -inf guarantees the first refusal logs.
|
|
112
|
+
last_refuse_warn = float("-inf")
|
|
100
113
|
try:
|
|
101
114
|
while max_messages is None or count < max_messages:
|
|
115
|
+
# Reap idle connections between selects so a client that
|
|
116
|
+
# connects and goes silent cannot hold a slot forever.
|
|
117
|
+
# Tradeoff: a client totally silent for the full timeout that
|
|
118
|
+
# then resumes without reading may lose its first post-reap
|
|
119
|
+
# write (its sendall can succeed locally before a later write
|
|
120
|
+
# sees EPIPE). This is inherent to reaping and acceptable —
|
|
121
|
+
# syslog delivery is best-effort and TCP clients reconnect.
|
|
122
|
+
now = _time.monotonic()
|
|
123
|
+
for sock in [s for s, t in last_seen.items() if now - t > TCP_IDLE_TIMEOUT]:
|
|
124
|
+
logger.info("Closing idle connection after %.0fs", TCP_IDLE_TIMEOUT)
|
|
125
|
+
# Discard any buffered remainder: the receive loop already
|
|
126
|
+
# emitted every newline-terminated record, so whatever is
|
|
127
|
+
# left is an incomplete fragment of a record the (still
|
|
128
|
+
# connected) client paused midway through. Unlike the EOF
|
|
129
|
+
# path — where a client close signals the record is done —
|
|
130
|
+
# emitting a reap fragment would produce a truncated event.
|
|
131
|
+
sel.unregister(sock)
|
|
132
|
+
sock.close()
|
|
133
|
+
buffers.pop(sock, None)
|
|
134
|
+
last_seen.pop(sock, None)
|
|
102
135
|
for key, _events in sel.select(timeout=1):
|
|
103
136
|
sock = key.fileobj
|
|
104
137
|
if sock is server:
|
|
105
138
|
conn, addr = server.accept()
|
|
139
|
+
if len(buffers) >= MAX_TCP_CONNECTIONS:
|
|
140
|
+
# At capacity: refuse rather than let aggregate
|
|
141
|
+
# buffers/descriptors grow without bound. Warn at
|
|
142
|
+
# most once per idle-timeout window so a flood of
|
|
143
|
+
# reconnects cannot itself become a log-DoS.
|
|
144
|
+
if now - last_refuse_warn > TCP_IDLE_TIMEOUT:
|
|
145
|
+
logger.warning(
|
|
146
|
+
"Refusing connections: at capacity (%d open)",
|
|
147
|
+
len(buffers),
|
|
148
|
+
)
|
|
149
|
+
last_refuse_warn = now
|
|
150
|
+
conn.close()
|
|
151
|
+
continue
|
|
106
152
|
conn.setblocking(False)
|
|
107
153
|
sel.register(conn, selectors.EVENT_READ)
|
|
108
154
|
buffers[conn] = b""
|
|
155
|
+
last_seen[conn] = _time.monotonic()
|
|
109
156
|
logger.debug("Connection from %s", addr)
|
|
110
157
|
continue
|
|
158
|
+
last_seen[sock] = _time.monotonic()
|
|
111
159
|
try:
|
|
112
160
|
data = sock.recv(65536)
|
|
113
161
|
except (BlockingIOError, InterruptedError):
|
|
@@ -118,6 +166,7 @@ def _listen_tcp(host, port, max_messages, ready_callback) -> Iterator[str]:
|
|
|
118
166
|
sel.unregister(sock)
|
|
119
167
|
sock.close()
|
|
120
168
|
remainder = buffers.pop(sock, b"")
|
|
169
|
+
last_seen.pop(sock, None)
|
|
121
170
|
line = remainder.decode("utf-8", "replace").strip()
|
|
122
171
|
if line:
|
|
123
172
|
yield line
|
|
@@ -137,6 +186,7 @@ def _listen_tcp(host, port, max_messages, ready_callback) -> Iterator[str]:
|
|
|
137
186
|
sel.unregister(sock)
|
|
138
187
|
sock.close()
|
|
139
188
|
buffers.pop(sock, None)
|
|
189
|
+
last_seen.pop(sock, None)
|
|
140
190
|
continue
|
|
141
191
|
while b"\n" in buffers[sock]:
|
|
142
192
|
raw, buffers[sock] = buffers[sock].split(b"\n", 1)
|
|
@@ -149,6 +199,7 @@ def _listen_tcp(host, port, max_messages, ready_callback) -> Iterator[str]:
|
|
|
149
199
|
finally:
|
|
150
200
|
for sock in list(buffers):
|
|
151
201
|
sock.close()
|
|
202
|
+
last_seen.clear()
|
|
152
203
|
sel.close()
|
|
153
204
|
server.close()
|
|
154
205
|
|
|
@@ -159,7 +210,6 @@ __all__ = ["listen_lines", "parse_endpoint"]
|
|
|
159
210
|
# --------------------------------------------------------------------------
|
|
160
211
|
# Network output: forward CEF records to a SIEM or pipeline.
|
|
161
212
|
|
|
162
|
-
import time as _time
|
|
163
213
|
from urllib.parse import urlparse
|
|
164
214
|
|
|
165
215
|
|