syslog2cef 0.2.0__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. {syslog2cef-0.2.0/syslog2cef.egg-info → syslog2cef-0.3.0}/PKG-INFO +1 -1
  2. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/pyproject.toml +1 -1
  3. {syslog2cef-0.2.0 → syslog2cef-0.3.0/syslog2cef.egg-info}/PKG-INFO +1 -1
  4. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/syslog2cef.egg-info/SOURCES.txt +7 -1
  5. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/syslogcef/__init__.py +2 -1
  6. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/syslogcef/adaptive.py +37 -10
  7. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/syslogcef/api.py +62 -1
  8. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/syslogcef/cef.py +3 -0
  9. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/syslogcef/cli.py +188 -18
  10. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/syslogcef/dictionary.py +9 -5
  11. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/syslogcef/mappings/__init__.py +1 -1
  12. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/syslogcef/mappings/linux.json +1 -0
  13. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/syslogcef/net.py +51 -1
  14. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/syslogcef/normalizer.py +4 -2
  15. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/syslogcef/parsers.py +131 -31
  16. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/syslogcef/utils.py +38 -6
  17. syslog2cef-0.3.0/tests/test_cli_validation.py +120 -0
  18. syslog2cef-0.3.0/tests/test_macos_logs.py +142 -0
  19. syslog2cef-0.3.0/tests/test_rfc5424_epoch.py +160 -0
  20. syslog2cef-0.3.0/tests/test_security_hardening.py +107 -0
  21. syslog2cef-0.3.0/tests/test_tail_rotation.py +246 -0
  22. syslog2cef-0.3.0/tests/test_zipapp_safe.py +47 -0
  23. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/LICENSE +0 -0
  24. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/README.md +0 -0
  25. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/setup.cfg +0 -0
  26. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/syslog2cef.egg-info/dependency_links.txt +0 -0
  27. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/syslog2cef.egg-info/entry_points.txt +0 -0
  28. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/syslog2cef.egg-info/requires.txt +0 -0
  29. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/syslog2cef.egg-info/top_level.txt +0 -0
  30. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/syslogcef/__main__.py +0 -0
  31. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/syslogcef/custom.py +0 -0
  32. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/syslogcef/dictionary.json +0 -0
  33. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/syslogcef/mappings/cisco_asa.json +0 -0
  34. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/syslogcef/mappings/cisco_ios.json +0 -0
  35. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/syslogcef/mappings/f5.json +0 -0
  36. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/syslogcef/mappings/fortinet.json +0 -0
  37. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/syslogcef/mappings/sophos.json +0 -0
  38. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/syslogcef/mappings/vmware.json +0 -0
  39. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/syslogcef/validation.py +0 -0
  40. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/tests/test_adaptive.py +0 -0
  41. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/tests/test_cef.py +0 -0
  42. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/tests/test_cli.py +0 -0
  43. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/tests/test_custom_parsers.py +0 -0
  44. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/tests/test_dictionary.py +0 -0
  45. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/tests/test_fuzz_grammar.py +0 -0
  46. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/tests/test_fuzz_properties.py +0 -0
  47. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/tests/test_net.py +0 -0
  48. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/tests/test_normalizer.py +0 -0
  49. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/tests/test_parsers.py +0 -0
  50. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/tests/test_parsers_extra.py +0 -0
  51. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/tests/test_pipeline.py +0 -0
  52. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/tests/test_real_world.py +0 -0
  53. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/tests/test_utils.py +0 -0
  54. {syslog2cef-0.2.0 → syslog2cef-0.3.0}/tests/test_validation.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: syslog2cef
3
- Version: 0.2.0
3
+ Version: 0.3.0
4
4
  Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
5
  Author-email: Tamir Suliman <allamiro@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/allamiro/syslogcef
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "syslog2cef"
7
- version = "0.2.0"
7
+ version = "0.3.0"
8
8
  description = "Convert RFC3164/RFC5424 syslog lines into ArcSight CEF"
9
9
  authors = [{name = "Tamir Suliman", email = "allamiro@gmail.com"}]
10
10
  readme = "README.md"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: syslog2cef
3
- Version: 0.2.0
3
+ Version: 0.3.0
4
4
  Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
5
  Author-email: Tamir Suliman <allamiro@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/allamiro/syslogcef
@@ -54,15 +54,21 @@ syslogcef/mappings/vmware.json
54
54
  tests/test_adaptive.py
55
55
  tests/test_cef.py
56
56
  tests/test_cli.py
57
+ tests/test_cli_validation.py
57
58
  tests/test_custom_parsers.py
58
59
  tests/test_dictionary.py
59
60
  tests/test_fuzz_grammar.py
60
61
  tests/test_fuzz_properties.py
62
+ tests/test_macos_logs.py
61
63
  tests/test_net.py
62
64
  tests/test_normalizer.py
63
65
  tests/test_parsers.py
64
66
  tests/test_parsers_extra.py
65
67
  tests/test_pipeline.py
66
68
  tests/test_real_world.py
69
+ tests/test_rfc5424_epoch.py
70
+ tests/test_security_hardening.py
71
+ tests/test_tail_rotation.py
67
72
  tests/test_utils.py
68
- tests/test_validation.py
73
+ tests/test_validation.py
74
+ tests/test_zipapp_safe.py
@@ -9,10 +9,11 @@ conversion workflow.
9
9
  """
10
10
  from __future__ import annotations
11
11
 
12
- from .api import convert_line, normalize_event, parse_syslog, to_cef
12
+ from .api import StreamConverter, convert_line, normalize_event, parse_syslog, to_cef
13
13
  from .custom import PatternFileError, load_patterns, register_parser
14
14
 
15
15
  __all__ = [
16
+ "StreamConverter",
16
17
  "convert_line",
17
18
  "normalize_event",
18
19
  "parse_syslog",
@@ -56,17 +56,26 @@ def _dmy_dash(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
56
56
 
57
57
 
58
58
  TIMESTAMP_LIB: Tuple[Tuple[str, str, Callable], ...] = (
59
- ("iso", r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?", _iso),
59
+ ("iso", r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}(?::?\d{2})?)?", _iso),
60
60
  ("mon_day_year", r"(?P<month>[A-Z][a-z]{2})\s+(?P<day>\d{1,2})\s+(?P<year>\d{4})\s+(?P<hms>\d{2}:\d{2}:\d{2})(?:\.\d+)?", _mon_day),
61
61
  ("mon_day", r"(?P<month>[A-Z][a-z]{2})\s+(?P<day>\d{1,2})\s+(?P<hms>\d{2}:\d{2}:\d{2})(?:\.\d+)?", _mon_day),
62
62
  ("ymd_slash", r"(?P<y>\d{4})/(?P<m>\d{2})/(?P<d>\d{2})[ T](?P<hms>\d{2}:\d{2}:\d{2})", _ymd_slash),
63
63
  ("dmy_dash", r"(?P<d>\d{1,2})-(?P<month>[A-Za-z]{3})-(?P<y>\d{4})\s+(?P<hms>\d{2}:\d{2}:\d{2})", _dmy_dash),
64
- ("epoch", r"\b1\d{9}(?:\d{6})?\b", _epoch),
64
+ # 10/13/16/19 digits = seconds/ms/us/ns, matching parse_iso8601's
65
+ # precision dispatch (a bare 10-digit run only, then the optional
66
+ # ms/us/ns extensions), so adaptively-parsed epochs get the same
67
+ # correct scaling as the built-in parsers.
68
+ ("epoch", r"\b1\d{9}(?:\d{3}|\d{6}|\d{9})?\b", _epoch),
65
69
  )
66
70
 
67
71
  HOST_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
68
72
  _SKIP_HOST_TOKENS = {"error", "warning", "info", "debug", "notice", "kernel"}
69
73
 
74
+ # Syslog-style program tag following the host: "app:", "app[pid]:".
75
+ TAG_RE = re.compile(r"^(?P<app>[\w\-/\.]+)(?:\[(?P<pid>[^\]]+)\])?:$")
76
+ # The same shape as a regex fragment for synthesized patterns.
77
+ _TAG_SRC = r"(?:(?P<adaptive_app>[\w\-/\.]+)(?:\[(?P<adaptive_pid>[^\]]+)\])?:\s+)?"
78
+
70
79
 
71
80
  def _valid_host(candidate: Optional[str]) -> bool:
72
81
  return bool(
@@ -105,7 +114,7 @@ def cache_size() -> int:
105
114
  return len(_CACHE)
106
115
 
107
116
 
108
- def _build_event(line, pri, ts, host, msg):
117
+ def _build_event(line, pri, ts, host, msg, app=None, pid=None):
109
118
  from .parsers import ParsedEvent
110
119
 
111
120
  facility, severity = convert_pri(pri)
@@ -116,8 +125,8 @@ def _build_event(line, pri, ts, host, msg):
116
125
  ts=ts,
117
126
  ts_orig="",
118
127
  host=host,
119
- app=None,
120
- pid=None,
128
+ app=app,
129
+ pid=pid,
121
130
  msgid=None,
122
131
  sd={},
123
132
  msg=sanitize_message(msg),
@@ -148,14 +157,17 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
148
157
  ts = None
149
158
  gd = match.groupdict()
150
159
  host = gd.get("adaptive_host")
160
+ app = gd.get("adaptive_app")
161
+ pid = gd.get("adaptive_pid")
151
162
  msg = gd.get("adaptive_msg")
152
163
  msg = line if msg is None else msg
153
164
  # The shape signature reduces all words to the same class, so a
154
165
  # cached host slot may capture a non-host token; revalidate.
155
166
  if host is not None and not _valid_host(host.rstrip(":,")):
156
- msg = f"{host} {msg}".strip()
157
- host = None
158
- return _build_event(raw, pri, ts, host, msg)
167
+ tag = f"{app}[{pid}]: " if app and pid else (f"{app}: " if app else "")
168
+ msg = f"{host} {tag}{msg}".strip()
169
+ host = app = pid = None
170
+ return _build_event(raw, pri, ts, host, msg, app=app, pid=pid)
159
171
 
160
172
  # Analysis pass: find the earliest timestamp in the line head.
161
173
  best = None
@@ -172,9 +184,13 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
172
184
  return None
173
185
 
174
186
  # Host candidate: the first plausible token after the timestamp that is
175
- # not a key=value pair or severity word.
187
+ # not a key=value pair or severity word. A leading timezone remnant the
188
+ # timestamp library could not consume (a lone "+02"/"CEST") must not be
189
+ # mistaken for the host — such tokens fail _valid_host and stay in msg.
176
190
  rest = line[ts_match.end():].lstrip(" :,\t")
177
191
  host = None
192
+ app = None
193
+ pid = None
178
194
  msg = rest
179
195
  tokens = rest.split(None, 1)
180
196
  if tokens:
@@ -182,12 +198,23 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
182
198
  if _valid_host(candidate):
183
199
  host = candidate
184
200
  msg = tokens[1] if len(tokens) > 1 else ""
201
+ # Program tag after the host ("app:" / "app[pid]:"): split it
202
+ # off so it maps to the CEF process field instead of polluting
203
+ # the message.
204
+ tag_tokens = msg.split(None, 1)
205
+ if tag_tokens:
206
+ tag_match = TAG_RE.match(tag_tokens[0])
207
+ if tag_match:
208
+ app = tag_match.group("app")
209
+ pid = tag_match.group("pid")
210
+ msg = tag_tokens[1] if len(tag_tokens) > 1 else ""
185
211
 
186
212
  # Synthesize and cache a pattern for this layout.
187
213
  prefix = line[: ts_match.start()]
188
214
  parts = [re.escape(prefix), f"(?P<adaptive_ts>{src})", r"[\s:,]*"]
189
215
  if host is not None:
190
216
  parts.append(r"(?P<adaptive_host>\S+?):?(?:\s+|$)")
217
+ parts.append(_TAG_SRC)
191
218
  parts.append(r"(?P<adaptive_msg>.*)$")
192
219
  try:
193
220
  pattern = re.compile("".join(parts))
@@ -198,7 +225,7 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
198
225
  _CACHE.pop(next(iter(_CACHE)))
199
226
  _CACHE[sig] = (pattern, name, conv)
200
227
 
201
- return _build_event(raw, pri, ts, host, msg)
228
+ return _build_event(raw, pri, ts, host, msg, app=app, pid=pid)
202
229
 
203
230
 
204
231
  def pattern_ts_match(pattern: Pattern[str], match: "re.Match[str]"):
@@ -68,7 +68,16 @@ def _load_mapping(mapping: Mapping[str, Any] | Path | str | None) -> Mapping[str
68
68
  return mapping
69
69
  path = Path(mapping)
70
70
  with path.open("r", encoding="utf-8") as fp:
71
- return json.load(fp)
71
+ data = json.load(fp)
72
+ # Structural validation with clear messages: a top-level array would
73
+ # otherwise TypeError deep inside the renderer, and a wrong-shaped
74
+ # extensions/severity_map would fail per event instead of up front.
75
+ if not isinstance(data, dict):
76
+ raise ValueError(f"{path}: mapping must be a JSON object")
77
+ for key in ("extensions", "severity_map"):
78
+ if key in data and not isinstance(data[key], dict):
79
+ raise ValueError(f"{path}: '{key}' must be a JSON object")
80
+ return data
72
81
 
73
82
 
74
83
  def to_cef(
@@ -111,6 +120,57 @@ def convert_line(
111
120
  return to_cef(normalized, mapping, validate=validate, strict=strict)
112
121
 
113
122
 
123
+ class StreamConverter:
124
+ """Stateful converter for an ordered stream of syslog lines.
125
+
126
+ Multi-line records (macOS install.log, wrapped plist/JSON payloads in
127
+ Apple system logs, Java stack traces) continue onto lines that begin
128
+ with whitespace and carry no syslog header of their own. Converted in
129
+ isolation such a line gets no timestamp and the local machine's
130
+ hostname — inside a container that is the container ID, not the host
131
+ that produced the log. Here a whitespace-indented line instead
132
+ inherits host, app, pid, PRI, and timestamp from the most recent
133
+ fully-parsed event and is tagged ``source_hint="continuation"``. One
134
+ CEF record is still emitted per input line.
135
+ """
136
+
137
+ def __init__(
138
+ self,
139
+ *,
140
+ mode: Optional[str] = None,
141
+ mapping: Mapping[str, Any] | Path | str | None = None,
142
+ validate: bool = False,
143
+ strict: bool = False,
144
+ ) -> None:
145
+ self.mode = mode
146
+ self.mapping = mapping
147
+ self.validate = validate
148
+ self.strict = strict
149
+ self._context: Optional[ParsedEvent] = None
150
+
151
+ def convert(self, line: str, *, now: Optional[datetime] = None) -> str:
152
+ parsed = parse_syslog(line, mode=self.mode, now=now)
153
+ if line[:1] in ("\t", " ") and self._context is not None:
154
+ ctx = self._context
155
+ parsed.host = ctx.host
156
+ parsed.app = parsed.app or ctx.app
157
+ parsed.pid = parsed.pid or ctx.pid
158
+ if parsed.ts is None:
159
+ parsed.ts = ctx.ts
160
+ if parsed.pri is None:
161
+ parsed.pri = ctx.pri
162
+ parsed.facility = ctx.facility
163
+ parsed.severity = ctx.severity
164
+ parsed.msg = parsed.msg.lstrip()
165
+ parsed.source_hint = "continuation"
166
+ elif parsed.host is not None and parsed.source_hint != "unknown":
167
+ # Only a line whose host was actually parsed (not the
168
+ # guessed fallback) may become continuation context.
169
+ self._context = parsed
170
+ normalized = normalize_event(parsed)
171
+ return to_cef(normalized, self.mapping, validate=self.validate, strict=self.strict)
172
+
173
+
114
174
  def _guess_mapping(event: NormalizedEvent) -> Mapping[str, Any]:
115
175
  msg_upper = event.msg.upper()
116
176
  app_upper = (event.app or "").upper()
@@ -139,6 +199,7 @@ __all__ = [
139
199
  "ParsedEvent",
140
200
  "NormalizedEvent",
141
201
  "ParseResult",
202
+ "StreamConverter",
142
203
  "convert_line",
143
204
  "normalize_event",
144
205
  "parse_syslog",
@@ -28,6 +28,9 @@ DEFAULT_MAPPING = {
28
28
  },
29
29
  "extensions": {
30
30
  "msg": "%(msg)s",
31
+ # Event time (epoch milliseconds) parsed from the line, or the
32
+ # processing time when the line carried no usable timestamp.
33
+ "rt": "%(rt)s",
31
34
  # The raw line travels in a labeled custom string: rawEvent is a
32
35
  # consumer-side key producers must not set (see docs/cef_fields.md),
33
36
  # and every bundled mapping already uses this shape.
@@ -8,9 +8,9 @@ from datetime import datetime
8
8
  from functools import partial
9
9
  from multiprocessing import Pool, cpu_count
10
10
  from pathlib import Path
11
- from typing import Iterable, Iterator, Optional
11
+ from typing import Any, Iterable, Iterator, Optional
12
12
 
13
- from .api import convert_line
13
+ from .api import StreamConverter, convert_line
14
14
 
15
15
  logger = logging.getLogger(__name__)
16
16
 
@@ -39,6 +39,127 @@ def tail_file(path: Path, *, follow: bool = False) -> Iterator[str]:
39
39
  yield line.rstrip("\n")
40
40
 
41
41
 
42
+ class _Follower:
43
+ """One tailed path, surviving rotation (rename, replace, truncate)."""
44
+
45
+ _FINGERPRINT_LEN = 256
46
+
47
+ def __init__(self, path: Path):
48
+ from collections import deque
49
+
50
+ self.path = path
51
+ self.handle = path.open("r", encoding="utf-8", errors="replace")
52
+ self.signature = self._fsignature()
53
+ # Fingerprint of the *already-consumed* head of the file. Those
54
+ # bytes are stable across appends (which only add data past our
55
+ # offset) but change when a copy-truncate rewrites the file in
56
+ # place, so comparing them detects rewrites the size/inode check
57
+ # misses. Length is bounded by both _FINGERPRINT_LEN and our read
58
+ # offset, so a plain append to a short file is never mistaken for
59
+ # a rewrite.
60
+ self.fp_len = 0
61
+ self.fingerprint = b""
62
+ self.pending = deque()
63
+
64
+ def _fsignature(self):
65
+ import os
66
+
67
+ st = os.fstat(self.handle.fileno())
68
+ return (st.st_dev, st.st_ino)
69
+
70
+ def _read_head(self, length: int) -> bytes:
71
+ # Positional read so the follow position is untouched.
72
+ import os
73
+
74
+ if self.handle is None or length <= 0:
75
+ return b""
76
+ try:
77
+ return os.pread(self.handle.fileno(), length, 0)
78
+ except (OSError, AttributeError):
79
+ return b""
80
+
81
+ def _capture_fingerprint(self) -> None:
82
+ if self.handle is None:
83
+ self.fp_len, self.fingerprint = 0, b""
84
+ return
85
+ self.fp_len = min(self._FINGERPRINT_LEN, self.handle.tell())
86
+ self.fingerprint = self._read_head(self.fp_len)
87
+
88
+ def read_lines(self) -> Iterator[str]:
89
+ while self.pending:
90
+ yield self.pending.popleft()
91
+ if self.handle is None:
92
+ return
93
+ while True:
94
+ line = self.handle.readline()
95
+ if not line:
96
+ # At EOF: record the consumed-head fingerprint so the next
97
+ # rotation check can tell an append from an in-place rewrite.
98
+ self._capture_fingerprint()
99
+ return
100
+ yield line.rstrip("\n")
101
+
102
+ def check_rotation(self) -> bool:
103
+ """Reattach after rotation events. Returns True if a replacement
104
+ file was (re)opened, so the caller can read it before treating
105
+ the poll as idle.
106
+
107
+ Runs every pass, not only when idle: a file that is renamed away
108
+ while still being written keeps this follower producing from the
109
+ stale descriptor forever, so a replacement at the original path
110
+ would never be picked up if rotation were checked only on idle
111
+ polls.
112
+
113
+ - rename-and-recreate: the path's dev/inode changes; drain any
114
+ remaining lines from the old descriptor, then open the new
115
+ file from the start.
116
+ - copy-truncate: same inode but size below our offset; rewind.
117
+ - temporary disappearance mid-rotation: keep the old descriptor
118
+ (it stays readable) and retry until a replacement appears.
119
+ """
120
+ if self.handle is None:
121
+ try:
122
+ self.handle = self.path.open("r", encoding="utf-8", errors="replace")
123
+ self.signature = self._fsignature()
124
+ self.fp_len, self.fingerprint = 0, b""
125
+ return True
126
+ except OSError:
127
+ return False
128
+ try:
129
+ st = self.path.stat()
130
+ except OSError:
131
+ return False
132
+ if (st.st_dev, st.st_ino) != self.signature:
133
+ for line in self.handle:
134
+ self.pending.append(line.rstrip("\n"))
135
+ self.handle.close()
136
+ self.handle = None
137
+ return self.check_rotation() # open the replacement immediately
138
+ # Copy-truncate: same inode, but either the file shrank below our
139
+ # offset, or the already-consumed head changed (an in-place rewrite
140
+ # that regrew past our offset). Comparing only the consumed prefix
141
+ # means a plain append — which never touches those bytes — is not
142
+ # mistaken for a rewrite, even for a file shorter than the cap.
143
+ offset = self.handle.tell()
144
+ rewound = False
145
+ if offset > 0:
146
+ if st.st_size < offset:
147
+ rewound = True
148
+ elif self.fp_len and self._read_head(self.fp_len) != self.fingerprint:
149
+ rewound = True
150
+ if rewound:
151
+ self.handle.seek(0)
152
+ self.fp_len, self.fingerprint = 0, b""
153
+ # A rewind exposes fresh content that must be read before the idle
154
+ # limit applies, so report it as activity like a reopen.
155
+ return rewound
156
+
157
+ def close(self) -> None:
158
+ if self.handle is not None:
159
+ self.handle.close()
160
+ self.handle = None
161
+
162
+
42
163
  def follow_files(
43
164
  paths: Iterable[Path],
44
165
  *,
@@ -47,23 +168,36 @@ def follow_files(
47
168
  ) -> Iterator[str]:
48
169
  """Tail several files at once, round-robin, yielding lines as they appear.
49
170
 
171
+ Survives log rotation on every path independently: renamed-and-
172
+ recreated files are reopened from the start (after draining the old
173
+ descriptor), copy-truncated files are rewound, and paths that
174
+ disappear mid-rotation are retried until a replacement appears.
175
+
50
176
  ``max_idle_polls`` bounds how many consecutive empty polls are allowed
51
177
  before the generator stops; ``None`` follows forever.
52
178
  """
53
179
 
54
- handles = [path.open("r", encoding="utf-8", errors="replace") for path in paths]
180
+ followers = [_Follower(path) for path in paths]
55
181
  idle_polls = 0
56
182
  try:
57
183
  while True:
58
- got_line = False
59
- for handle in handles:
60
- while True:
61
- line = handle.readline()
62
- if not line:
63
- break
64
- got_line = True
65
- yield line.rstrip("\n")
66
- if got_line:
184
+ any_line = False
185
+ for follower in followers:
186
+ for line in follower.read_lines():
187
+ any_line = True
188
+ yield line
189
+ # Rotation is checked for every follower on every pass —
190
+ # including busy ones — so a file renamed away while still
191
+ # being written cannot pin this follower to the stale
192
+ # descriptor and starve the replacement at the origin path.
193
+ reopened = False
194
+ for follower in followers:
195
+ if follower.check_rotation():
196
+ reopened = True
197
+ # A reopened replacement must be read before the idle limit
198
+ # applies, so treat a reopen as activity: loop again to read
199
+ # it rather than counting this pass as idle.
200
+ if any_line or reopened:
67
201
  idle_polls = 0
68
202
  continue
69
203
  idle_polls += 1
@@ -71,8 +205,8 @@ def follow_files(
71
205
  return
72
206
  time.sleep(poll_interval)
73
207
  finally:
74
- for handle in handles:
75
- handle.close()
208
+ for follower in followers:
209
+ follower.close()
76
210
 
77
211
 
78
212
  # Portable strftime codes with at-coarsest one-second granularity.
@@ -173,7 +307,7 @@ def write_lines(
173
307
  fp.close()
174
308
 
175
309
 
176
- def _convert_line_mp(line: str, mode: Optional[str], mapping: Optional[str], validate: bool = False, strict: bool = False) -> str:
310
+ def _convert_line_mp(line: str, mode: Optional[str], mapping: Any, validate: bool = False, strict: bool = False) -> str:
177
311
  return convert_line(line, mode=mode, mapping=mapping, validate=validate, strict=strict)
178
312
 
179
313
 
@@ -193,7 +327,7 @@ def process_lines(
193
327
  lines: Iterable[str],
194
328
  *,
195
329
  mode: Optional[str],
196
- mapping: Optional[str],
330
+ mapping: Any,
197
331
  use_multiprocessing: bool,
198
332
  pool_size: Optional[int],
199
333
  validate: bool = False,
@@ -207,8 +341,12 @@ def process_lines(
207
341
  for cef in pool.imap(worker, lines):
208
342
  yield cef
209
343
  else:
344
+ # Stateful conversion so whitespace-indented continuation lines
345
+ # inherit host/app/timestamp from the preceding event. Unavailable
346
+ # under --multiprocess, where workers cannot share line order.
347
+ converter = StreamConverter(mode=mode, mapping=mapping, validate=validate, strict=strict)
210
348
  for line in lines:
211
- yield convert_line(line, mode=mode, mapping=mapping, validate=validate, strict=strict)
349
+ yield converter.convert(line)
212
350
 
213
351
 
214
352
  def _optional_path(value: str) -> Optional[Path]:
@@ -238,6 +376,20 @@ def main(argv: Optional[list[str]] = None) -> int:
238
376
  args = parser.parse_args(argv)
239
377
  logging.basicConfig(level=getattr(logging, args.log_level.upper(), logging.WARNING))
240
378
 
379
+ # Reject invalid option combinations before any file, socket, or
380
+ # pool side effects (#89): mistakes in service environment files
381
+ # must fail with a clear message, not a traceback or silence.
382
+ if args.pool_size is not None and not args.multiprocess:
383
+ parser.error("--pool-size requires --multiprocess")
384
+ if args.multiprocess and args.pool_size is not None and args.pool_size < 1:
385
+ parser.error(f"--pool-size must be a positive integer (got {args.pool_size})")
386
+ if args.eps is not None and not args.send:
387
+ parser.error("--eps requires --send")
388
+ if args.listen and args.paths:
389
+ parser.error("--listen replaces file input; remove the positional paths")
390
+ if args.tail and not args.paths:
391
+ parser.error("--tail requires input file paths")
392
+
241
393
  if args.output:
242
394
  try:
243
395
  template_has_codes(str(args.output))
@@ -260,7 +412,25 @@ def main(argv: Optional[list[str]] = None) -> int:
260
412
  if args.mode not in PARSERS and get_registered(args.mode) is None:
261
413
  parser.error(f"unknown --mode '{args.mode}' (not a built-in parser or a loaded pattern name)")
262
414
 
263
- mapping = args.mapping
415
+ # Load and validate the mapping eagerly (#84): a missing or malformed
416
+ # file must fail before the output is opened — a truncate-mode open
417
+ # would destroy an existing archive before the lazy first-event load
418
+ # ever ran. Loading once here also avoids re-reading the JSON file
419
+ # for every converted line.
420
+ mapping = None
421
+ if args.mapping is not None:
422
+ # An explicitly empty --mapping (e.g. MAPPING= blank in a service
423
+ # environment file) is a misconfiguration, not "use the default":
424
+ # silently proceeding would truncate an existing --output with an
425
+ # unintended mapping. Fail before any file is opened.
426
+ if not args.mapping.strip():
427
+ parser.error("--mapping was given an empty value")
428
+ from .api import _load_mapping
429
+
430
+ try:
431
+ mapping = _load_mapping(args.mapping)
432
+ except (OSError, ValueError) as exc:
433
+ parser.error(f"invalid --mapping: {exc}")
264
434
 
265
435
  outputs: Iterator[str]
266
436
  if args.listen:
@@ -21,18 +21,22 @@ from __future__ import annotations
21
21
 
22
22
  import json
23
23
  from functools import lru_cache
24
- from pathlib import Path
24
+ from importlib import resources
25
25
  from typing import Dict, FrozenSet, Tuple
26
26
 
27
- _DICTIONARY_PATH = Path(__file__).parent / "dictionary.json"
28
-
29
27
 
30
28
  @lru_cache(maxsize=1)
31
29
  def _load() -> dict:
32
- with _DICTIONARY_PATH.open(encoding="utf-8") as fp:
30
+ # importlib.resources, not a filesystem path: the package also runs
31
+ # from the standalone zipapp, where package files are inside the
32
+ # archive and Path(__file__)-style access fails (mappings/__init__.py
33
+ # uses the same mechanism for the same reason).
34
+ with resources.files("syslogcef").joinpath("dictionary.json").open(
35
+ "r", encoding="utf-8"
36
+ ) as fp:
33
37
  data = json.load(fp)
34
38
  if not isinstance(data.get("keys"), dict) or not isinstance(data.get("aliases"), dict):
35
- raise ValueError(f"{_DICTIONARY_PATH}: expected 'keys' and 'aliases' objects")
39
+ raise ValueError("dictionary.json: expected 'keys' and 'aliases' objects")
36
40
  return data
37
41
 
38
42
 
@@ -17,7 +17,7 @@ __all__ = [
17
17
 
18
18
 
19
19
  def _load(name: str) -> Mapping[str, Any]:
20
- with resources.files(__package__).joinpath(f"{name}.json").open("r", encoding="utf-8") as fp:
20
+ with resources.files("syslogcef.mappings").joinpath(f"{name}.json").open("r", encoding="utf-8") as fp:
21
21
  return json.load(fp)
22
22
 
23
23
 
@@ -7,6 +7,7 @@
7
7
  "extensions": {
8
8
  "dhost": "%(host)s",
9
9
  "sproc": "%(app)s",
10
+ "dvcpid": "%(pid)s",
10
11
  "cs1Label": "rawEvent",
11
12
  "cs1": "%(raw)s"
12
13
  }