syslog2cef 0.2.1__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. {syslog2cef-0.2.1/syslog2cef.egg-info → syslog2cef-0.3.0}/PKG-INFO +1 -1
  2. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/pyproject.toml +1 -1
  3. {syslog2cef-0.2.1 → syslog2cef-0.3.0/syslog2cef.egg-info}/PKG-INFO +1 -1
  4. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslog2cef.egg-info/SOURCES.txt +5 -0
  5. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/__init__.py +2 -1
  6. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/adaptive.py +37 -10
  7. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/api.py +62 -1
  8. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/cef.py +3 -0
  9. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/cli.py +188 -18
  10. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/mappings/linux.json +1 -0
  11. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/net.py +51 -1
  12. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/normalizer.py +4 -2
  13. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/parsers.py +131 -31
  14. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/utils.py +38 -6
  15. syslog2cef-0.3.0/tests/test_cli_validation.py +120 -0
  16. syslog2cef-0.3.0/tests/test_macos_logs.py +142 -0
  17. syslog2cef-0.3.0/tests/test_rfc5424_epoch.py +160 -0
  18. syslog2cef-0.3.0/tests/test_security_hardening.py +107 -0
  19. syslog2cef-0.3.0/tests/test_tail_rotation.py +246 -0
  20. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/LICENSE +0 -0
  21. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/README.md +0 -0
  22. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/setup.cfg +0 -0
  23. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslog2cef.egg-info/dependency_links.txt +0 -0
  24. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslog2cef.egg-info/entry_points.txt +0 -0
  25. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslog2cef.egg-info/requires.txt +0 -0
  26. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslog2cef.egg-info/top_level.txt +0 -0
  27. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/__main__.py +0 -0
  28. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/custom.py +0 -0
  29. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/dictionary.json +0 -0
  30. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/dictionary.py +0 -0
  31. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/mappings/__init__.py +0 -0
  32. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/mappings/cisco_asa.json +0 -0
  33. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/mappings/cisco_ios.json +0 -0
  34. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/mappings/f5.json +0 -0
  35. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/mappings/fortinet.json +0 -0
  36. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/mappings/sophos.json +0 -0
  37. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/mappings/vmware.json +0 -0
  38. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/syslogcef/validation.py +0 -0
  39. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_adaptive.py +0 -0
  40. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_cef.py +0 -0
  41. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_cli.py +0 -0
  42. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_custom_parsers.py +0 -0
  43. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_dictionary.py +0 -0
  44. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_fuzz_grammar.py +0 -0
  45. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_fuzz_properties.py +0 -0
  46. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_net.py +0 -0
  47. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_normalizer.py +0 -0
  48. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_parsers.py +0 -0
  49. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_parsers_extra.py +0 -0
  50. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_pipeline.py +0 -0
  51. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_real_world.py +0 -0
  52. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_utils.py +0 -0
  53. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_validation.py +0 -0
  54. {syslog2cef-0.2.1 → syslog2cef-0.3.0}/tests/test_zipapp_safe.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: syslog2cef
3
- Version: 0.2.1
3
+ Version: 0.3.0
4
4
  Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
5
  Author-email: Tamir Suliman <allamiro@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/allamiro/syslogcef
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "syslog2cef"
7
- version = "0.2.1"
7
+ version = "0.3.0"
8
8
  description = "Convert RFC3164/RFC5424 syslog lines into ArcSight CEF"
9
9
  authors = [{name = "Tamir Suliman", email = "allamiro@gmail.com"}]
10
10
  readme = "README.md"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: syslog2cef
3
- Version: 0.2.1
3
+ Version: 0.3.0
4
4
  Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
5
  Author-email: Tamir Suliman <allamiro@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/allamiro/syslogcef
@@ -54,16 +54,21 @@ syslogcef/mappings/vmware.json
54
54
  tests/test_adaptive.py
55
55
  tests/test_cef.py
56
56
  tests/test_cli.py
57
+ tests/test_cli_validation.py
57
58
  tests/test_custom_parsers.py
58
59
  tests/test_dictionary.py
59
60
  tests/test_fuzz_grammar.py
60
61
  tests/test_fuzz_properties.py
62
+ tests/test_macos_logs.py
61
63
  tests/test_net.py
62
64
  tests/test_normalizer.py
63
65
  tests/test_parsers.py
64
66
  tests/test_parsers_extra.py
65
67
  tests/test_pipeline.py
66
68
  tests/test_real_world.py
69
+ tests/test_rfc5424_epoch.py
70
+ tests/test_security_hardening.py
71
+ tests/test_tail_rotation.py
67
72
  tests/test_utils.py
68
73
  tests/test_validation.py
69
74
  tests/test_zipapp_safe.py
@@ -9,10 +9,11 @@ conversion workflow.
9
9
  """
10
10
  from __future__ import annotations
11
11
 
12
- from .api import convert_line, normalize_event, parse_syslog, to_cef
12
+ from .api import StreamConverter, convert_line, normalize_event, parse_syslog, to_cef
13
13
  from .custom import PatternFileError, load_patterns, register_parser
14
14
 
15
15
  __all__ = [
16
+ "StreamConverter",
16
17
  "convert_line",
17
18
  "normalize_event",
18
19
  "parse_syslog",
@@ -56,17 +56,26 @@ def _dmy_dash(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
56
56
 
57
57
 
58
58
  TIMESTAMP_LIB: Tuple[Tuple[str, str, Callable], ...] = (
59
- ("iso", r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?", _iso),
59
+ ("iso", r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}(?::?\d{2})?)?", _iso),
60
60
  ("mon_day_year", r"(?P<month>[A-Z][a-z]{2})\s+(?P<day>\d{1,2})\s+(?P<year>\d{4})\s+(?P<hms>\d{2}:\d{2}:\d{2})(?:\.\d+)?", _mon_day),
61
61
  ("mon_day", r"(?P<month>[A-Z][a-z]{2})\s+(?P<day>\d{1,2})\s+(?P<hms>\d{2}:\d{2}:\d{2})(?:\.\d+)?", _mon_day),
62
62
  ("ymd_slash", r"(?P<y>\d{4})/(?P<m>\d{2})/(?P<d>\d{2})[ T](?P<hms>\d{2}:\d{2}:\d{2})", _ymd_slash),
63
63
  ("dmy_dash", r"(?P<d>\d{1,2})-(?P<month>[A-Za-z]{3})-(?P<y>\d{4})\s+(?P<hms>\d{2}:\d{2}:\d{2})", _dmy_dash),
64
- ("epoch", r"\b1\d{9}(?:\d{6})?\b", _epoch),
64
+ # 10/13/16/19 digits = seconds/ms/us/ns, matching parse_iso8601's
65
+ # precision dispatch (a bare 10-digit run only, then the optional
66
+ # ms/us/ns extensions), so adaptively-parsed epochs get the same
67
+ # correct scaling as the built-in parsers.
68
+ ("epoch", r"\b1\d{9}(?:\d{3}|\d{6}|\d{9})?\b", _epoch),
65
69
  )
66
70
 
67
71
  HOST_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
68
72
  _SKIP_HOST_TOKENS = {"error", "warning", "info", "debug", "notice", "kernel"}
69
73
 
74
+ # Syslog-style program tag following the host: "app:", "app[pid]:".
75
+ TAG_RE = re.compile(r"^(?P<app>[\w\-/\.]+)(?:\[(?P<pid>[^\]]+)\])?:$")
76
+ # The same shape as a regex fragment for synthesized patterns.
77
+ _TAG_SRC = r"(?:(?P<adaptive_app>[\w\-/\.]+)(?:\[(?P<adaptive_pid>[^\]]+)\])?:\s+)?"
78
+
70
79
 
71
80
  def _valid_host(candidate: Optional[str]) -> bool:
72
81
  return bool(
@@ -105,7 +114,7 @@ def cache_size() -> int:
105
114
  return len(_CACHE)
106
115
 
107
116
 
108
- def _build_event(line, pri, ts, host, msg):
117
+ def _build_event(line, pri, ts, host, msg, app=None, pid=None):
109
118
  from .parsers import ParsedEvent
110
119
 
111
120
  facility, severity = convert_pri(pri)
@@ -116,8 +125,8 @@ def _build_event(line, pri, ts, host, msg):
116
125
  ts=ts,
117
126
  ts_orig="",
118
127
  host=host,
119
- app=None,
120
- pid=None,
128
+ app=app,
129
+ pid=pid,
121
130
  msgid=None,
122
131
  sd={},
123
132
  msg=sanitize_message(msg),
@@ -148,14 +157,17 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
148
157
  ts = None
149
158
  gd = match.groupdict()
150
159
  host = gd.get("adaptive_host")
160
+ app = gd.get("adaptive_app")
161
+ pid = gd.get("adaptive_pid")
151
162
  msg = gd.get("adaptive_msg")
152
163
  msg = line if msg is None else msg
153
164
  # The shape signature reduces all words to the same class, so a
154
165
  # cached host slot may capture a non-host token; revalidate.
155
166
  if host is not None and not _valid_host(host.rstrip(":,")):
156
- msg = f"{host} {msg}".strip()
157
- host = None
158
- return _build_event(raw, pri, ts, host, msg)
167
+ tag = f"{app}[{pid}]: " if app and pid else (f"{app}: " if app else "")
168
+ msg = f"{host} {tag}{msg}".strip()
169
+ host = app = pid = None
170
+ return _build_event(raw, pri, ts, host, msg, app=app, pid=pid)
159
171
 
160
172
  # Analysis pass: find the earliest timestamp in the line head.
161
173
  best = None
@@ -172,9 +184,13 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
172
184
  return None
173
185
 
174
186
  # Host candidate: the first plausible token after the timestamp that is
175
- # not a key=value pair or severity word.
187
+ # not a key=value pair or severity word. A leading timezone remnant the
188
+ # timestamp library could not consume (a lone "+02"/"CEST") must not be
189
+ # mistaken for the host — such tokens fail _valid_host and stay in msg.
176
190
  rest = line[ts_match.end():].lstrip(" :,\t")
177
191
  host = None
192
+ app = None
193
+ pid = None
178
194
  msg = rest
179
195
  tokens = rest.split(None, 1)
180
196
  if tokens:
@@ -182,12 +198,23 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
182
198
  if _valid_host(candidate):
183
199
  host = candidate
184
200
  msg = tokens[1] if len(tokens) > 1 else ""
201
+ # Program tag after the host ("app:" / "app[pid]:"): split it
202
+ # off so it maps to the CEF process field instead of polluting
203
+ # the message.
204
+ tag_tokens = msg.split(None, 1)
205
+ if tag_tokens:
206
+ tag_match = TAG_RE.match(tag_tokens[0])
207
+ if tag_match:
208
+ app = tag_match.group("app")
209
+ pid = tag_match.group("pid")
210
+ msg = tag_tokens[1] if len(tag_tokens) > 1 else ""
185
211
 
186
212
  # Synthesize and cache a pattern for this layout.
187
213
  prefix = line[: ts_match.start()]
188
214
  parts = [re.escape(prefix), f"(?P<adaptive_ts>{src})", r"[\s:,]*"]
189
215
  if host is not None:
190
216
  parts.append(r"(?P<adaptive_host>\S+?):?(?:\s+|$)")
217
+ parts.append(_TAG_SRC)
191
218
  parts.append(r"(?P<adaptive_msg>.*)$")
192
219
  try:
193
220
  pattern = re.compile("".join(parts))
@@ -198,7 +225,7 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
198
225
  _CACHE.pop(next(iter(_CACHE)))
199
226
  _CACHE[sig] = (pattern, name, conv)
200
227
 
201
- return _build_event(raw, pri, ts, host, msg)
228
+ return _build_event(raw, pri, ts, host, msg, app=app, pid=pid)
202
229
 
203
230
 
204
231
  def pattern_ts_match(pattern: Pattern[str], match: "re.Match[str]"):
@@ -68,7 +68,16 @@ def _load_mapping(mapping: Mapping[str, Any] | Path | str | None) -> Mapping[str
68
68
  return mapping
69
69
  path = Path(mapping)
70
70
  with path.open("r", encoding="utf-8") as fp:
71
- return json.load(fp)
71
+ data = json.load(fp)
72
+ # Structural validation with clear messages: a top-level array would
73
+ # otherwise TypeError deep inside the renderer, and a wrong-shaped
74
+ # extensions/severity_map would fail per event instead of up front.
75
+ if not isinstance(data, dict):
76
+ raise ValueError(f"{path}: mapping must be a JSON object")
77
+ for key in ("extensions", "severity_map"):
78
+ if key in data and not isinstance(data[key], dict):
79
+ raise ValueError(f"{path}: '{key}' must be a JSON object")
80
+ return data
72
81
 
73
82
 
74
83
  def to_cef(
@@ -111,6 +120,57 @@ def convert_line(
111
120
  return to_cef(normalized, mapping, validate=validate, strict=strict)
112
121
 
113
122
 
123
+ class StreamConverter:
124
+ """Stateful converter for an ordered stream of syslog lines.
125
+
126
+ Multi-line records (macOS install.log, wrapped plist/JSON payloads in
127
+ Apple system logs, Java stack traces) continue onto lines that begin
128
+ with whitespace and carry no syslog header of their own. Converted in
129
+ isolation such a line gets no timestamp and the local machine's
130
+ hostname — inside a container that is the container ID, not the host
131
+ that produced the log. Here a whitespace-indented line instead
132
+ inherits host, app, pid, PRI, and timestamp from the most recent
133
+ fully-parsed event and is tagged ``source_hint="continuation"``. One
134
+ CEF record is still emitted per input line.
135
+ """
136
+
137
+ def __init__(
138
+ self,
139
+ *,
140
+ mode: Optional[str] = None,
141
+ mapping: Mapping[str, Any] | Path | str | None = None,
142
+ validate: bool = False,
143
+ strict: bool = False,
144
+ ) -> None:
145
+ self.mode = mode
146
+ self.mapping = mapping
147
+ self.validate = validate
148
+ self.strict = strict
149
+ self._context: Optional[ParsedEvent] = None
150
+
151
+ def convert(self, line: str, *, now: Optional[datetime] = None) -> str:
152
+ parsed = parse_syslog(line, mode=self.mode, now=now)
153
+ if line[:1] in ("\t", " ") and self._context is not None:
154
+ ctx = self._context
155
+ parsed.host = ctx.host
156
+ parsed.app = parsed.app or ctx.app
157
+ parsed.pid = parsed.pid or ctx.pid
158
+ if parsed.ts is None:
159
+ parsed.ts = ctx.ts
160
+ if parsed.pri is None:
161
+ parsed.pri = ctx.pri
162
+ parsed.facility = ctx.facility
163
+ parsed.severity = ctx.severity
164
+ parsed.msg = parsed.msg.lstrip()
165
+ parsed.source_hint = "continuation"
166
+ elif parsed.host is not None and parsed.source_hint != "unknown":
167
+ # Only a line whose host was actually parsed (not the
168
+ # guessed fallback) may become continuation context.
169
+ self._context = parsed
170
+ normalized = normalize_event(parsed)
171
+ return to_cef(normalized, self.mapping, validate=self.validate, strict=self.strict)
172
+
173
+
114
174
  def _guess_mapping(event: NormalizedEvent) -> Mapping[str, Any]:
115
175
  msg_upper = event.msg.upper()
116
176
  app_upper = (event.app or "").upper()
@@ -139,6 +199,7 @@ __all__ = [
139
199
  "ParsedEvent",
140
200
  "NormalizedEvent",
141
201
  "ParseResult",
202
+ "StreamConverter",
142
203
  "convert_line",
143
204
  "normalize_event",
144
205
  "parse_syslog",
@@ -28,6 +28,9 @@ DEFAULT_MAPPING = {
28
28
  },
29
29
  "extensions": {
30
30
  "msg": "%(msg)s",
31
+ # Event time (epoch milliseconds) parsed from the line, or the
32
+ # processing time when the line carried no usable timestamp.
33
+ "rt": "%(rt)s",
31
34
  # The raw line travels in a labeled custom string: rawEvent is a
32
35
  # consumer-side key producers must not set (see docs/cef_fields.md),
33
36
  # and every bundled mapping already uses this shape.
@@ -8,9 +8,9 @@ from datetime import datetime
8
8
  from functools import partial
9
9
  from multiprocessing import Pool, cpu_count
10
10
  from pathlib import Path
11
- from typing import Iterable, Iterator, Optional
11
+ from typing import Any, Iterable, Iterator, Optional
12
12
 
13
- from .api import convert_line
13
+ from .api import StreamConverter, convert_line
14
14
 
15
15
  logger = logging.getLogger(__name__)
16
16
 
@@ -39,6 +39,127 @@ def tail_file(path: Path, *, follow: bool = False) -> Iterator[str]:
39
39
  yield line.rstrip("\n")
40
40
 
41
41
 
42
+ class _Follower:
43
+ """One tailed path, surviving rotation (rename, replace, truncate)."""
44
+
45
+ _FINGERPRINT_LEN = 256
46
+
47
+ def __init__(self, path: Path):
48
+ from collections import deque
49
+
50
+ self.path = path
51
+ self.handle = path.open("r", encoding="utf-8", errors="replace")
52
+ self.signature = self._fsignature()
53
+ # Fingerprint of the *already-consumed* head of the file. Those
54
+ # bytes are stable across appends (which only add data past our
55
+ # offset) but change when a copy-truncate rewrites the file in
56
+ # place, so comparing them detects rewrites the size/inode check
57
+ # misses. Length is bounded by both _FINGERPRINT_LEN and our read
58
+ # offset, so a plain append to a short file is never mistaken for
59
+ # a rewrite.
60
+ self.fp_len = 0
61
+ self.fingerprint = b""
62
+ self.pending = deque()
63
+
64
+ def _fsignature(self):
65
+ import os
66
+
67
+ st = os.fstat(self.handle.fileno())
68
+ return (st.st_dev, st.st_ino)
69
+
70
+ def _read_head(self, length: int) -> bytes:
71
+ # Positional read so the follow position is untouched.
72
+ import os
73
+
74
+ if self.handle is None or length <= 0:
75
+ return b""
76
+ try:
77
+ return os.pread(self.handle.fileno(), length, 0)
78
+ except (OSError, AttributeError):
79
+ return b""
80
+
81
+ def _capture_fingerprint(self) -> None:
82
+ if self.handle is None:
83
+ self.fp_len, self.fingerprint = 0, b""
84
+ return
85
+ self.fp_len = min(self._FINGERPRINT_LEN, self.handle.tell())
86
+ self.fingerprint = self._read_head(self.fp_len)
87
+
88
+ def read_lines(self) -> Iterator[str]:
89
+ while self.pending:
90
+ yield self.pending.popleft()
91
+ if self.handle is None:
92
+ return
93
+ while True:
94
+ line = self.handle.readline()
95
+ if not line:
96
+ # At EOF: record the consumed-head fingerprint so the next
97
+ # rotation check can tell an append from an in-place rewrite.
98
+ self._capture_fingerprint()
99
+ return
100
+ yield line.rstrip("\n")
101
+
102
+ def check_rotation(self) -> bool:
103
+ """Reattach after rotation events. Returns True if a replacement
104
+ file was (re)opened, so the caller can read it before treating
105
+ the poll as idle.
106
+
107
+ Runs every pass, not only when idle: a file that is renamed away
108
+ while still being written keeps this follower producing from the
109
+ stale descriptor forever, so a replacement at the original path
110
+ would never be picked up if rotation were checked only on idle
111
+ polls.
112
+
113
+ - rename-and-recreate: the path's dev/inode changes; drain any
114
+ remaining lines from the old descriptor, then open the new
115
+ file from the start.
116
+ - copy-truncate: same inode but size below our offset; rewind.
117
+ - temporary disappearance mid-rotation: keep the old descriptor
118
+ (it stays readable) and retry until a replacement appears.
119
+ """
120
+ if self.handle is None:
121
+ try:
122
+ self.handle = self.path.open("r", encoding="utf-8", errors="replace")
123
+ self.signature = self._fsignature()
124
+ self.fp_len, self.fingerprint = 0, b""
125
+ return True
126
+ except OSError:
127
+ return False
128
+ try:
129
+ st = self.path.stat()
130
+ except OSError:
131
+ return False
132
+ if (st.st_dev, st.st_ino) != self.signature:
133
+ for line in self.handle:
134
+ self.pending.append(line.rstrip("\n"))
135
+ self.handle.close()
136
+ self.handle = None
137
+ return self.check_rotation() # open the replacement immediately
138
+ # Copy-truncate: same inode, but either the file shrank below our
139
+ # offset, or the already-consumed head changed (an in-place rewrite
140
+ # that regrew past our offset). Comparing only the consumed prefix
141
+ # means a plain append — which never touches those bytes — is not
142
+ # mistaken for a rewrite, even for a file shorter than the cap.
143
+ offset = self.handle.tell()
144
+ rewound = False
145
+ if offset > 0:
146
+ if st.st_size < offset:
147
+ rewound = True
148
+ elif self.fp_len and self._read_head(self.fp_len) != self.fingerprint:
149
+ rewound = True
150
+ if rewound:
151
+ self.handle.seek(0)
152
+ self.fp_len, self.fingerprint = 0, b""
153
+ # A rewind exposes fresh content that must be read before the idle
154
+ # limit applies, so report it as activity like a reopen.
155
+ return rewound
156
+
157
+ def close(self) -> None:
158
+ if self.handle is not None:
159
+ self.handle.close()
160
+ self.handle = None
161
+
162
+
42
163
  def follow_files(
43
164
  paths: Iterable[Path],
44
165
  *,
@@ -47,23 +168,36 @@ def follow_files(
47
168
  ) -> Iterator[str]:
48
169
  """Tail several files at once, round-robin, yielding lines as they appear.
49
170
 
171
+ Survives log rotation on every path independently: renamed-and-
172
+ recreated files are reopened from the start (after draining the old
173
+ descriptor), copy-truncated files are rewound, and paths that
174
+ disappear mid-rotation are retried until a replacement appears.
175
+
50
176
  ``max_idle_polls`` bounds how many consecutive empty polls are allowed
51
177
  before the generator stops; ``None`` follows forever.
52
178
  """
53
179
 
54
- handles = [path.open("r", encoding="utf-8", errors="replace") for path in paths]
180
+ followers = [_Follower(path) for path in paths]
55
181
  idle_polls = 0
56
182
  try:
57
183
  while True:
58
- got_line = False
59
- for handle in handles:
60
- while True:
61
- line = handle.readline()
62
- if not line:
63
- break
64
- got_line = True
65
- yield line.rstrip("\n")
66
- if got_line:
184
+ any_line = False
185
+ for follower in followers:
186
+ for line in follower.read_lines():
187
+ any_line = True
188
+ yield line
189
+ # Rotation is checked for every follower on every pass —
190
+ # including busy ones — so a file renamed away while still
191
+ # being written cannot pin this follower to the stale
192
+ # descriptor and starve the replacement at the origin path.
193
+ reopened = False
194
+ for follower in followers:
195
+ if follower.check_rotation():
196
+ reopened = True
197
+ # A reopened replacement must be read before the idle limit
198
+ # applies, so treat a reopen as activity: loop again to read
199
+ # it rather than counting this pass as idle.
200
+ if any_line or reopened:
67
201
  idle_polls = 0
68
202
  continue
69
203
  idle_polls += 1
@@ -71,8 +205,8 @@ def follow_files(
71
205
  return
72
206
  time.sleep(poll_interval)
73
207
  finally:
74
- for handle in handles:
75
- handle.close()
208
+ for follower in followers:
209
+ follower.close()
76
210
 
77
211
 
78
212
  # Portable strftime codes with at-coarsest one-second granularity.
@@ -173,7 +307,7 @@ def write_lines(
173
307
  fp.close()
174
308
 
175
309
 
176
- def _convert_line_mp(line: str, mode: Optional[str], mapping: Optional[str], validate: bool = False, strict: bool = False) -> str:
310
+ def _convert_line_mp(line: str, mode: Optional[str], mapping: Any, validate: bool = False, strict: bool = False) -> str:
177
311
  return convert_line(line, mode=mode, mapping=mapping, validate=validate, strict=strict)
178
312
 
179
313
 
@@ -193,7 +327,7 @@ def process_lines(
193
327
  lines: Iterable[str],
194
328
  *,
195
329
  mode: Optional[str],
196
- mapping: Optional[str],
330
+ mapping: Any,
197
331
  use_multiprocessing: bool,
198
332
  pool_size: Optional[int],
199
333
  validate: bool = False,
@@ -207,8 +341,12 @@ def process_lines(
207
341
  for cef in pool.imap(worker, lines):
208
342
  yield cef
209
343
  else:
344
+ # Stateful conversion so whitespace-indented continuation lines
345
+ # inherit host/app/timestamp from the preceding event. Unavailable
346
+ # under --multiprocess, where workers cannot share line order.
347
+ converter = StreamConverter(mode=mode, mapping=mapping, validate=validate, strict=strict)
210
348
  for line in lines:
211
- yield convert_line(line, mode=mode, mapping=mapping, validate=validate, strict=strict)
349
+ yield converter.convert(line)
212
350
 
213
351
 
214
352
  def _optional_path(value: str) -> Optional[Path]:
@@ -238,6 +376,20 @@ def main(argv: Optional[list[str]] = None) -> int:
238
376
  args = parser.parse_args(argv)
239
377
  logging.basicConfig(level=getattr(logging, args.log_level.upper(), logging.WARNING))
240
378
 
379
+ # Reject invalid option combinations before any file, socket, or
380
+ # pool side effects (#89): mistakes in service environment files
381
+ # must fail with a clear message, not a traceback or silence.
382
+ if args.pool_size is not None and not args.multiprocess:
383
+ parser.error("--pool-size requires --multiprocess")
384
+ if args.multiprocess and args.pool_size is not None and args.pool_size < 1:
385
+ parser.error(f"--pool-size must be a positive integer (got {args.pool_size})")
386
+ if args.eps is not None and not args.send:
387
+ parser.error("--eps requires --send")
388
+ if args.listen and args.paths:
389
+ parser.error("--listen replaces file input; remove the positional paths")
390
+ if args.tail and not args.paths:
391
+ parser.error("--tail requires input file paths")
392
+
241
393
  if args.output:
242
394
  try:
243
395
  template_has_codes(str(args.output))
@@ -260,7 +412,25 @@ def main(argv: Optional[list[str]] = None) -> int:
260
412
  if args.mode not in PARSERS and get_registered(args.mode) is None:
261
413
  parser.error(f"unknown --mode '{args.mode}' (not a built-in parser or a loaded pattern name)")
262
414
 
263
- mapping = args.mapping
415
+ # Load and validate the mapping eagerly (#84): a missing or malformed
416
+ # file must fail before the output is opened — a truncate-mode open
417
+ # would destroy an existing archive before the lazy first-event load
418
+ # ever ran. Loading once here also avoids re-reading the JSON file
419
+ # for every converted line.
420
+ mapping = None
421
+ if args.mapping is not None:
422
+ # An explicitly empty --mapping (e.g. MAPPING= blank in a service
423
+ # environment file) is a misconfiguration, not "use the default":
424
+ # silently proceeding would truncate an existing --output with an
425
+ # unintended mapping. Fail before any file is opened.
426
+ if not args.mapping.strip():
427
+ parser.error("--mapping was given an empty value")
428
+ from .api import _load_mapping
429
+
430
+ try:
431
+ mapping = _load_mapping(args.mapping)
432
+ except (OSError, ValueError) as exc:
433
+ parser.error(f"invalid --mapping: {exc}")
264
434
 
265
435
  outputs: Iterator[str]
266
436
  if args.listen:
@@ -7,6 +7,7 @@
7
7
  "extensions": {
8
8
  "dhost": "%(host)s",
9
9
  "sproc": "%(app)s",
10
+ "dvcpid": "%(pid)s",
10
11
  "cs1Label": "rawEvent",
11
12
  "cs1": "%(raw)s"
12
13
  }
@@ -12,6 +12,8 @@ import selectors
12
12
  import socket
13
13
  from typing import Callable, Iterator, Optional, Tuple
14
14
 
15
+ import time as _time
16
+
15
17
  logger = logging.getLogger(__name__)
16
18
 
17
19
  MAX_DATAGRAM = 65535
@@ -21,6 +23,12 @@ MAX_DATAGRAM = 65535
21
23
  # dropped.
22
24
  MAX_TCP_BUFFER = 1024 * 1024
23
25
 
26
+ # Aggregate limits so many well-behaved-per-connection clients cannot
27
+ # still exhaust memory or descriptors in total. New connections beyond
28
+ # the cap are refused; a connection idle past the timeout is closed.
29
+ MAX_TCP_CONNECTIONS = 512
30
+ TCP_IDLE_TIMEOUT = 300.0 # seconds without any received data
31
+
24
32
 
25
33
  def parse_endpoint(spec: str) -> Tuple[str, str, int]:
26
34
  """Parse ``udp:514``, ``tcp:5514``, or ``udp:10.0.0.5:514``."""
@@ -96,18 +104,58 @@ def _listen_tcp(host, port, max_messages, ready_callback) -> Iterator[str]:
96
104
  sel = selectors.DefaultSelector()
97
105
  sel.register(server, selectors.EVENT_READ)
98
106
  buffers: dict[socket.socket, bytes] = {}
107
+ last_seen: dict[socket.socket, float] = {}
99
108
  count = 0
109
+ # -inf, not 0.0: within the first TCP_IDLE_TIMEOUT seconds of process
110
+ # uptime monotonic() is small, and a 0.0 seed would suppress the very
111
+ # first capacity warning. -inf guarantees the first refusal logs.
112
+ last_refuse_warn = float("-inf")
100
113
  try:
101
114
  while max_messages is None or count < max_messages:
115
+ # Reap idle connections between selects so a client that
116
+ # connects and goes silent cannot hold a slot forever.
117
+ # Tradeoff: a client totally silent for the full timeout that
118
+ # then resumes without reading may lose its first post-reap
119
+ # write (its sendall can succeed locally before a later write
120
+ # sees EPIPE). This is inherent to reaping and acceptable —
121
+ # syslog delivery is best-effort and TCP clients reconnect.
122
+ now = _time.monotonic()
123
+ for sock in [s for s, t in last_seen.items() if now - t > TCP_IDLE_TIMEOUT]:
124
+ logger.info("Closing idle connection after %.0fs", TCP_IDLE_TIMEOUT)
125
+ # Discard any buffered remainder: the receive loop already
126
+ # emitted every newline-terminated record, so whatever is
127
+ # left is an incomplete fragment of a record the (still
128
+ # connected) client paused midway through. Unlike the EOF
129
+ # path — where a client close signals the record is done —
130
+ # emitting a reap fragment would produce a truncated event.
131
+ sel.unregister(sock)
132
+ sock.close()
133
+ buffers.pop(sock, None)
134
+ last_seen.pop(sock, None)
102
135
  for key, _events in sel.select(timeout=1):
103
136
  sock = key.fileobj
104
137
  if sock is server:
105
138
  conn, addr = server.accept()
139
+ if len(buffers) >= MAX_TCP_CONNECTIONS:
140
+ # At capacity: refuse rather than let aggregate
141
+ # buffers/descriptors grow without bound. Warn at
142
+ # most once per idle-timeout window so a flood of
143
+ # reconnects cannot itself become a log-DoS.
144
+ if now - last_refuse_warn > TCP_IDLE_TIMEOUT:
145
+ logger.warning(
146
+ "Refusing connections: at capacity (%d open)",
147
+ len(buffers),
148
+ )
149
+ last_refuse_warn = now
150
+ conn.close()
151
+ continue
106
152
  conn.setblocking(False)
107
153
  sel.register(conn, selectors.EVENT_READ)
108
154
  buffers[conn] = b""
155
+ last_seen[conn] = _time.monotonic()
109
156
  logger.debug("Connection from %s", addr)
110
157
  continue
158
+ last_seen[sock] = _time.monotonic()
111
159
  try:
112
160
  data = sock.recv(65536)
113
161
  except (BlockingIOError, InterruptedError):
@@ -118,6 +166,7 @@ def _listen_tcp(host, port, max_messages, ready_callback) -> Iterator[str]:
118
166
  sel.unregister(sock)
119
167
  sock.close()
120
168
  remainder = buffers.pop(sock, b"")
169
+ last_seen.pop(sock, None)
121
170
  line = remainder.decode("utf-8", "replace").strip()
122
171
  if line:
123
172
  yield line
@@ -137,6 +186,7 @@ def _listen_tcp(host, port, max_messages, ready_callback) -> Iterator[str]:
137
186
  sel.unregister(sock)
138
187
  sock.close()
139
188
  buffers.pop(sock, None)
189
+ last_seen.pop(sock, None)
140
190
  continue
141
191
  while b"\n" in buffers[sock]:
142
192
  raw, buffers[sock] = buffers[sock].split(b"\n", 1)
@@ -149,6 +199,7 @@ def _listen_tcp(host, port, max_messages, ready_callback) -> Iterator[str]:
149
199
  finally:
150
200
  for sock in list(buffers):
151
201
  sock.close()
202
+ last_seen.clear()
152
203
  sel.close()
153
204
  server.close()
154
205
 
@@ -159,7 +210,6 @@ __all__ = ["listen_lines", "parse_endpoint"]
159
210
  # --------------------------------------------------------------------------
160
211
  # Network output: forward CEF records to a SIEM or pipeline.
161
212
 
162
- import time as _time
163
213
  from urllib.parse import urlparse
164
214
 
165
215