syslog2cef 0.2.1__tar.gz → 0.3.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. {syslog2cef-0.2.1/syslog2cef.egg-info → syslog2cef-0.3.1}/PKG-INFO +1 -1
  2. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/pyproject.toml +1 -1
  3. {syslog2cef-0.2.1 → syslog2cef-0.3.1/syslog2cef.egg-info}/PKG-INFO +1 -1
  4. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/syslog2cef.egg-info/SOURCES.txt +5 -0
  5. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/syslogcef/__init__.py +2 -1
  6. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/syslogcef/adaptive.py +37 -10
  7. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/syslogcef/api.py +91 -2
  8. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/syslogcef/cef.py +3 -0
  9. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/syslogcef/cli.py +223 -27
  10. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/syslogcef/mappings/linux.json +1 -0
  11. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/syslogcef/net.py +51 -1
  12. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/syslogcef/normalizer.py +13 -2
  13. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/syslogcef/parsers.py +131 -31
  14. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/syslogcef/utils.py +38 -6
  15. syslog2cef-0.3.1/tests/test_cli_validation.py +120 -0
  16. syslog2cef-0.3.1/tests/test_macos_logs.py +234 -0
  17. syslog2cef-0.3.1/tests/test_rfc5424_epoch.py +160 -0
  18. syslog2cef-0.3.1/tests/test_security_hardening.py +107 -0
  19. syslog2cef-0.3.1/tests/test_tail_rotation.py +246 -0
  20. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/LICENSE +0 -0
  21. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/README.md +0 -0
  22. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/setup.cfg +0 -0
  23. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/syslog2cef.egg-info/dependency_links.txt +0 -0
  24. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/syslog2cef.egg-info/entry_points.txt +0 -0
  25. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/syslog2cef.egg-info/requires.txt +0 -0
  26. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/syslog2cef.egg-info/top_level.txt +0 -0
  27. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/syslogcef/__main__.py +0 -0
  28. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/syslogcef/custom.py +0 -0
  29. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/syslogcef/dictionary.json +0 -0
  30. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/syslogcef/dictionary.py +0 -0
  31. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/syslogcef/mappings/__init__.py +0 -0
  32. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/syslogcef/mappings/cisco_asa.json +0 -0
  33. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/syslogcef/mappings/cisco_ios.json +0 -0
  34. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/syslogcef/mappings/f5.json +0 -0
  35. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/syslogcef/mappings/fortinet.json +0 -0
  36. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/syslogcef/mappings/sophos.json +0 -0
  37. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/syslogcef/mappings/vmware.json +0 -0
  38. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/syslogcef/validation.py +0 -0
  39. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/tests/test_adaptive.py +0 -0
  40. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/tests/test_cef.py +0 -0
  41. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/tests/test_cli.py +0 -0
  42. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/tests/test_custom_parsers.py +0 -0
  43. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/tests/test_dictionary.py +0 -0
  44. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/tests/test_fuzz_grammar.py +0 -0
  45. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/tests/test_fuzz_properties.py +0 -0
  46. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/tests/test_net.py +0 -0
  47. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/tests/test_normalizer.py +0 -0
  48. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/tests/test_parsers.py +0 -0
  49. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/tests/test_parsers_extra.py +0 -0
  50. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/tests/test_pipeline.py +0 -0
  51. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/tests/test_real_world.py +0 -0
  52. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/tests/test_utils.py +0 -0
  53. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/tests/test_validation.py +0 -0
  54. {syslog2cef-0.2.1 → syslog2cef-0.3.1}/tests/test_zipapp_safe.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: syslog2cef
3
- Version: 0.2.1
3
+ Version: 0.3.1
4
4
  Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
5
  Author-email: Tamir Suliman <allamiro@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/allamiro/syslogcef
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "syslog2cef"
7
- version = "0.2.1"
7
+ version = "0.3.1"
8
8
  description = "Convert RFC3164/RFC5424 syslog lines into ArcSight CEF"
9
9
  authors = [{name = "Tamir Suliman", email = "allamiro@gmail.com"}]
10
10
  readme = "README.md"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: syslog2cef
3
- Version: 0.2.1
3
+ Version: 0.3.1
4
4
  Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
5
  Author-email: Tamir Suliman <allamiro@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/allamiro/syslogcef
@@ -54,16 +54,21 @@ syslogcef/mappings/vmware.json
54
54
  tests/test_adaptive.py
55
55
  tests/test_cef.py
56
56
  tests/test_cli.py
57
+ tests/test_cli_validation.py
57
58
  tests/test_custom_parsers.py
58
59
  tests/test_dictionary.py
59
60
  tests/test_fuzz_grammar.py
60
61
  tests/test_fuzz_properties.py
62
+ tests/test_macos_logs.py
61
63
  tests/test_net.py
62
64
  tests/test_normalizer.py
63
65
  tests/test_parsers.py
64
66
  tests/test_parsers_extra.py
65
67
  tests/test_pipeline.py
66
68
  tests/test_real_world.py
69
+ tests/test_rfc5424_epoch.py
70
+ tests/test_security_hardening.py
71
+ tests/test_tail_rotation.py
67
72
  tests/test_utils.py
68
73
  tests/test_validation.py
69
74
  tests/test_zipapp_safe.py
@@ -9,10 +9,11 @@ conversion workflow.
9
9
  """
10
10
  from __future__ import annotations
11
11
 
12
- from .api import convert_line, normalize_event, parse_syslog, to_cef
12
+ from .api import StreamConverter, convert_line, normalize_event, parse_syslog, to_cef
13
13
  from .custom import PatternFileError, load_patterns, register_parser
14
14
 
15
15
  __all__ = [
16
+ "StreamConverter",
16
17
  "convert_line",
17
18
  "normalize_event",
18
19
  "parse_syslog",
@@ -56,17 +56,26 @@ def _dmy_dash(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
56
56
 
57
57
 
58
58
  TIMESTAMP_LIB: Tuple[Tuple[str, str, Callable], ...] = (
59
- ("iso", r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?", _iso),
59
+ ("iso", r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}(?::?\d{2})?)?", _iso),
60
60
  ("mon_day_year", r"(?P<month>[A-Z][a-z]{2})\s+(?P<day>\d{1,2})\s+(?P<year>\d{4})\s+(?P<hms>\d{2}:\d{2}:\d{2})(?:\.\d+)?", _mon_day),
61
61
  ("mon_day", r"(?P<month>[A-Z][a-z]{2})\s+(?P<day>\d{1,2})\s+(?P<hms>\d{2}:\d{2}:\d{2})(?:\.\d+)?", _mon_day),
62
62
  ("ymd_slash", r"(?P<y>\d{4})/(?P<m>\d{2})/(?P<d>\d{2})[ T](?P<hms>\d{2}:\d{2}:\d{2})", _ymd_slash),
63
63
  ("dmy_dash", r"(?P<d>\d{1,2})-(?P<month>[A-Za-z]{3})-(?P<y>\d{4})\s+(?P<hms>\d{2}:\d{2}:\d{2})", _dmy_dash),
64
- ("epoch", r"\b1\d{9}(?:\d{6})?\b", _epoch),
64
+ # 10/13/16/19 digits = seconds/ms/us/ns, matching parse_iso8601's
65
+ # precision dispatch (a bare 10-digit run only, then the optional
66
+ # ms/us/ns extensions), so adaptively-parsed epochs get the same
67
+ # correct scaling as the built-in parsers.
68
+ ("epoch", r"\b1\d{9}(?:\d{3}|\d{6}|\d{9})?\b", _epoch),
65
69
  )
66
70
 
67
71
  HOST_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
68
72
  _SKIP_HOST_TOKENS = {"error", "warning", "info", "debug", "notice", "kernel"}
69
73
 
74
+ # Syslog-style program tag following the host: "app:", "app[pid]:".
75
+ TAG_RE = re.compile(r"^(?P<app>[\w\-/\.]+)(?:\[(?P<pid>[^\]]+)\])?:$")
76
+ # The same shape as a regex fragment for synthesized patterns.
77
+ _TAG_SRC = r"(?:(?P<adaptive_app>[\w\-/\.]+)(?:\[(?P<adaptive_pid>[^\]]+)\])?:\s+)?"
78
+
70
79
 
71
80
  def _valid_host(candidate: Optional[str]) -> bool:
72
81
  return bool(
@@ -105,7 +114,7 @@ def cache_size() -> int:
105
114
  return len(_CACHE)
106
115
 
107
116
 
108
- def _build_event(line, pri, ts, host, msg):
117
+ def _build_event(line, pri, ts, host, msg, app=None, pid=None):
109
118
  from .parsers import ParsedEvent
110
119
 
111
120
  facility, severity = convert_pri(pri)
@@ -116,8 +125,8 @@ def _build_event(line, pri, ts, host, msg):
116
125
  ts=ts,
117
126
  ts_orig="",
118
127
  host=host,
119
- app=None,
120
- pid=None,
128
+ app=app,
129
+ pid=pid,
121
130
  msgid=None,
122
131
  sd={},
123
132
  msg=sanitize_message(msg),
@@ -148,14 +157,17 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
148
157
  ts = None
149
158
  gd = match.groupdict()
150
159
  host = gd.get("adaptive_host")
160
+ app = gd.get("adaptive_app")
161
+ pid = gd.get("adaptive_pid")
151
162
  msg = gd.get("adaptive_msg")
152
163
  msg = line if msg is None else msg
153
164
  # The shape signature reduces all words to the same class, so a
154
165
  # cached host slot may capture a non-host token; revalidate.
155
166
  if host is not None and not _valid_host(host.rstrip(":,")):
156
- msg = f"{host} {msg}".strip()
157
- host = None
158
- return _build_event(raw, pri, ts, host, msg)
167
+ tag = f"{app}[{pid}]: " if app and pid else (f"{app}: " if app else "")
168
+ msg = f"{host} {tag}{msg}".strip()
169
+ host = app = pid = None
170
+ return _build_event(raw, pri, ts, host, msg, app=app, pid=pid)
159
171
 
160
172
  # Analysis pass: find the earliest timestamp in the line head.
161
173
  best = None
@@ -172,9 +184,13 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
172
184
  return None
173
185
 
174
186
  # Host candidate: the first plausible token after the timestamp that is
175
- # not a key=value pair or severity word.
187
+ # not a key=value pair or severity word. A leading timezone remnant the
188
+ # timestamp library could not consume (a lone "+02"/"CEST") must not be
189
+ # mistaken for the host — such tokens fail _valid_host and stay in msg.
176
190
  rest = line[ts_match.end():].lstrip(" :,\t")
177
191
  host = None
192
+ app = None
193
+ pid = None
178
194
  msg = rest
179
195
  tokens = rest.split(None, 1)
180
196
  if tokens:
@@ -182,12 +198,23 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
182
198
  if _valid_host(candidate):
183
199
  host = candidate
184
200
  msg = tokens[1] if len(tokens) > 1 else ""
201
+ # Program tag after the host ("app:" / "app[pid]:"): split it
202
+ # off so it maps to the CEF process field instead of polluting
203
+ # the message.
204
+ tag_tokens = msg.split(None, 1)
205
+ if tag_tokens:
206
+ tag_match = TAG_RE.match(tag_tokens[0])
207
+ if tag_match:
208
+ app = tag_match.group("app")
209
+ pid = tag_match.group("pid")
210
+ msg = tag_tokens[1] if len(tag_tokens) > 1 else ""
185
211
 
186
212
  # Synthesize and cache a pattern for this layout.
187
213
  prefix = line[: ts_match.start()]
188
214
  parts = [re.escape(prefix), f"(?P<adaptive_ts>{src})", r"[\s:,]*"]
189
215
  if host is not None:
190
216
  parts.append(r"(?P<adaptive_host>\S+?):?(?:\s+|$)")
217
+ parts.append(_TAG_SRC)
191
218
  parts.append(r"(?P<adaptive_msg>.*)$")
192
219
  try:
193
220
  pattern = re.compile("".join(parts))
@@ -198,7 +225,7 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
198
225
  _CACHE.pop(next(iter(_CACHE)))
199
226
  _CACHE[sig] = (pattern, name, conv)
200
227
 
201
- return _build_event(raw, pri, ts, host, msg)
228
+ return _build_event(raw, pri, ts, host, msg, app=app, pid=pid)
202
229
 
203
230
 
204
231
  def pattern_ts_match(pattern: Pattern[str], match: "re.Match[str]"):
@@ -7,8 +7,9 @@ from datetime import datetime
7
7
  from pathlib import Path
8
8
  from typing import Any, Mapping, Optional
9
9
 
10
- from .parsers import ParsedEvent, autodetect_and_parse
10
+ from .parsers import ParsedEvent, ParserError, autodetect_and_parse
11
11
  from .normalizer import NormalizedEvent, normalize
12
+ from .utils import sanitize_message
12
13
  from .mappings import CISCO_ASA, CISCO_IOS, F5, FORTINET, LINUX, SOPHOS, VMWARE
13
14
  from .cef import CEFEvent, build_cef
14
15
 
@@ -68,7 +69,16 @@ def _load_mapping(mapping: Mapping[str, Any] | Path | str | None) -> Mapping[str
68
69
  return mapping
69
70
  path = Path(mapping)
70
71
  with path.open("r", encoding="utf-8") as fp:
71
- return json.load(fp)
72
+ data = json.load(fp)
73
+ # Structural validation with clear messages: a top-level array would
74
+ # otherwise TypeError deep inside the renderer, and a wrong-shaped
75
+ # extensions/severity_map would fail per event instead of up front.
76
+ if not isinstance(data, dict):
77
+ raise ValueError(f"{path}: mapping must be a JSON object")
78
+ for key in ("extensions", "severity_map"):
79
+ if key in data and not isinstance(data[key], dict):
80
+ raise ValueError(f"{path}: '{key}' must be a JSON object")
81
+ return data
72
82
 
73
83
 
74
84
  def to_cef(
@@ -111,6 +121,84 @@ def convert_line(
111
121
  return to_cef(normalized, mapping, validate=validate, strict=strict)
112
122
 
113
123
 
124
+ class StreamConverter:
125
+ """Stateful converter for an ordered stream of syslog lines.
126
+
127
+ Multi-line records (macOS install.log, wrapped plist/JSON payloads in
128
+ Apple system logs, Java stack traces) continue onto lines that begin
129
+ with whitespace and carry no syslog header of their own. Converted in
130
+ isolation such a line gets no timestamp and the local machine's
131
+ hostname — inside a container that is the container ID, not the host
132
+ that produced the log. Here a whitespace-indented line instead
133
+ inherits host, app, pid, PRI, and timestamp from the most recent
134
+ fully-parsed event and is tagged ``source_hint="continuation"``. One
135
+ CEF record is still emitted per input line.
136
+
137
+ Context is tracked per ``source`` (e.g. one entry per input file), so
138
+ interleaved streams — several files tailed at once — never inherit
139
+ another file's host or timestamp. Callers with a single stream can
140
+ ignore the parameter.
141
+ """
142
+
143
+ def __init__(
144
+ self,
145
+ *,
146
+ mode: Optional[str] = None,
147
+ mapping: Mapping[str, Any] | Path | str | None = None,
148
+ validate: bool = False,
149
+ strict: bool = False,
150
+ ) -> None:
151
+ self.mode = mode
152
+ self.mapping = mapping
153
+ self.validate = validate
154
+ self.strict = strict
155
+ self._contexts: dict[str, ParsedEvent] = {}
156
+
157
+ def convert(self, line: str, *, now: Optional[datetime] = None, source: str = "") -> str:
158
+ ctx = self._contexts.get(source)
159
+ is_continuation = line[:1] in ("\t", " ") and ctx is not None
160
+ try:
161
+ parsed = parse_syslog(line, mode=self.mode, now=now)
162
+ except ParserError:
163
+ if not is_continuation:
164
+ raise
165
+ # A forced --mode rejects headerless continuation lines; a
166
+ # continuation with context must inherit, not abort the run.
167
+ parsed = ParsedEvent(
168
+ pri=None,
169
+ facility=None,
170
+ severity=None,
171
+ ts=None,
172
+ ts_orig="",
173
+ host=None,
174
+ app=None,
175
+ pid=None,
176
+ msgid=None,
177
+ sd={},
178
+ msg=sanitize_message(line),
179
+ raw=line,
180
+ source_hint="unknown",
181
+ )
182
+ if is_continuation:
183
+ parsed.host = ctx.host
184
+ parsed.app = parsed.app or ctx.app
185
+ parsed.pid = parsed.pid or ctx.pid
186
+ if parsed.ts is None:
187
+ parsed.ts = ctx.ts
188
+ if parsed.pri is None:
189
+ parsed.pri = ctx.pri
190
+ parsed.facility = ctx.facility
191
+ parsed.severity = ctx.severity
192
+ parsed.msg = parsed.msg.lstrip()
193
+ parsed.source_hint = "continuation"
194
+ elif parsed.host is not None and parsed.source_hint != "unknown":
195
+ # Only a line whose host was actually parsed (not the
196
+ # guessed fallback) may become continuation context.
197
+ self._contexts[source] = parsed
198
+ normalized = normalize_event(parsed)
199
+ return to_cef(normalized, self.mapping, validate=self.validate, strict=self.strict)
200
+
201
+
114
202
  def _guess_mapping(event: NormalizedEvent) -> Mapping[str, Any]:
115
203
  msg_upper = event.msg.upper()
116
204
  app_upper = (event.app or "").upper()
@@ -139,6 +227,7 @@ __all__ = [
139
227
  "ParsedEvent",
140
228
  "NormalizedEvent",
141
229
  "ParseResult",
230
+ "StreamConverter",
142
231
  "convert_line",
143
232
  "normalize_event",
144
233
  "parse_syslog",
@@ -28,6 +28,9 @@ DEFAULT_MAPPING = {
28
28
  },
29
29
  "extensions": {
30
30
  "msg": "%(msg)s",
31
+ # Event time (epoch milliseconds) parsed from the line, or the
32
+ # processing time when the line carried no usable timestamp.
33
+ "rt": "%(rt)s",
31
34
  # The raw line travels in a labeled custom string: rawEvent is a
32
35
  # consumer-side key producers must not set (see docs/cef_fields.md),
33
36
  # and every bundled mapping already uses this shape.
@@ -8,23 +8,33 @@ from datetime import datetime
8
8
  from functools import partial
9
9
  from multiprocessing import Pool, cpu_count
10
10
  from pathlib import Path
11
- from typing import Iterable, Iterator, Optional
11
+ from typing import Any, Iterable, Iterator, Optional
12
12
 
13
- from .api import convert_line
13
+ from .api import StreamConverter, convert_line
14
14
 
15
15
  logger = logging.getLogger(__name__)
16
16
 
17
17
 
18
- def iter_lines_from_sources(paths: Iterable[Path] | None) -> Iterator[str]:
18
+ def iter_lines_from_sources(paths: Iterable[Path] | None, *, tag_sources: bool = False) -> Iterator[Any]:
19
+ """Yield lines from stdin or the given files.
20
+
21
+ With ``tag_sources`` each item is ``(source, line)`` so downstream
22
+ continuation handling can keep per-file context instead of letting
23
+ an indented line at the top of one file inherit the previous file's
24
+ host and timestamp.
25
+ """
19
26
  if not paths:
20
27
  for line in sys.stdin:
21
- yield line.rstrip("\n")
28
+ line = line.rstrip("\n")
29
+ yield ("stdin", line) if tag_sources else line
22
30
  return
23
31
 
24
32
  for path in paths:
33
+ source = str(path)
25
34
  with path.open("r", encoding="utf-8", errors="replace") as handle:
26
35
  for line in handle:
27
- yield line.rstrip("\n")
36
+ line = line.rstrip("\n")
37
+ yield (source, line) if tag_sources else line
28
38
 
29
39
 
30
40
  def tail_file(path: Path, *, follow: bool = False) -> Iterator[str]:
@@ -39,31 +49,171 @@ def tail_file(path: Path, *, follow: bool = False) -> Iterator[str]:
39
49
  yield line.rstrip("\n")
40
50
 
41
51
 
52
+ class _Follower:
53
+ """One tailed path, surviving rotation (rename, replace, truncate)."""
54
+
55
+ _FINGERPRINT_LEN = 256
56
+
57
+ def __init__(self, path: Path):
58
+ from collections import deque
59
+
60
+ self.path = path
61
+ self.handle = path.open("r", encoding="utf-8", errors="replace")
62
+ self.signature = self._fsignature()
63
+ # Fingerprint of the *already-consumed* head of the file. Those
64
+ # bytes are stable across appends (which only add data past our
65
+ # offset) but change when a copy-truncate rewrites the file in
66
+ # place, so comparing them detects rewrites the size/inode check
67
+ # misses. Length is bounded by both _FINGERPRINT_LEN and our read
68
+ # offset, so a plain append to a short file is never mistaken for
69
+ # a rewrite.
70
+ self.fp_len = 0
71
+ self.fingerprint = b""
72
+ self.pending = deque()
73
+
74
+ def _fsignature(self):
75
+ import os
76
+
77
+ st = os.fstat(self.handle.fileno())
78
+ return (st.st_dev, st.st_ino)
79
+
80
+ def _read_head(self, length: int) -> bytes:
81
+ # Positional read so the follow position is untouched.
82
+ import os
83
+
84
+ if self.handle is None or length <= 0:
85
+ return b""
86
+ try:
87
+ return os.pread(self.handle.fileno(), length, 0)
88
+ except (OSError, AttributeError):
89
+ return b""
90
+
91
+ def _capture_fingerprint(self) -> None:
92
+ if self.handle is None:
93
+ self.fp_len, self.fingerprint = 0, b""
94
+ return
95
+ self.fp_len = min(self._FINGERPRINT_LEN, self.handle.tell())
96
+ self.fingerprint = self._read_head(self.fp_len)
97
+
98
+ def read_lines(self) -> Iterator[str]:
99
+ while self.pending:
100
+ yield self.pending.popleft()
101
+ if self.handle is None:
102
+ return
103
+ while True:
104
+ line = self.handle.readline()
105
+ if not line:
106
+ # At EOF: record the consumed-head fingerprint so the next
107
+ # rotation check can tell an append from an in-place rewrite.
108
+ self._capture_fingerprint()
109
+ return
110
+ yield line.rstrip("\n")
111
+
112
+ def check_rotation(self) -> bool:
113
+ """Reattach after rotation events. Returns True if a replacement
114
+ file was (re)opened, so the caller can read it before treating
115
+ the poll as idle.
116
+
117
+ Runs every pass, not only when idle: a file that is renamed away
118
+ while still being written keeps this follower producing from the
119
+ stale descriptor forever, so a replacement at the original path
120
+ would never be picked up if rotation were checked only on idle
121
+ polls.
122
+
123
+ - rename-and-recreate: the path's dev/inode changes; drain any
124
+ remaining lines from the old descriptor, then open the new
125
+ file from the start.
126
+ - copy-truncate: same inode but size below our offset; rewind.
127
+ - temporary disappearance mid-rotation: keep the old descriptor
128
+ (it stays readable) and retry until a replacement appears.
129
+ """
130
+ if self.handle is None:
131
+ try:
132
+ self.handle = self.path.open("r", encoding="utf-8", errors="replace")
133
+ self.signature = self._fsignature()
134
+ self.fp_len, self.fingerprint = 0, b""
135
+ return True
136
+ except OSError:
137
+ return False
138
+ try:
139
+ st = self.path.stat()
140
+ except OSError:
141
+ return False
142
+ if (st.st_dev, st.st_ino) != self.signature:
143
+ for line in self.handle:
144
+ self.pending.append(line.rstrip("\n"))
145
+ self.handle.close()
146
+ self.handle = None
147
+ return self.check_rotation() # open the replacement immediately
148
+ # Copy-truncate: same inode, but either the file shrank below our
149
+ # offset, or the already-consumed head changed (an in-place rewrite
150
+ # that regrew past our offset). Comparing only the consumed prefix
151
+ # means a plain append — which never touches those bytes — is not
152
+ # mistaken for a rewrite, even for a file shorter than the cap.
153
+ offset = self.handle.tell()
154
+ rewound = False
155
+ if offset > 0:
156
+ if st.st_size < offset:
157
+ rewound = True
158
+ elif self.fp_len and self._read_head(self.fp_len) != self.fingerprint:
159
+ rewound = True
160
+ if rewound:
161
+ self.handle.seek(0)
162
+ self.fp_len, self.fingerprint = 0, b""
163
+ # A rewind exposes fresh content that must be read before the idle
164
+ # limit applies, so report it as activity like a reopen.
165
+ return rewound
166
+
167
+ def close(self) -> None:
168
+ if self.handle is not None:
169
+ self.handle.close()
170
+ self.handle = None
171
+
172
+
42
173
  def follow_files(
43
174
  paths: Iterable[Path],
44
175
  *,
45
176
  poll_interval: float = 0.5,
46
177
  max_idle_polls: Optional[int] = None,
47
- ) -> Iterator[str]:
178
+ tag_sources: bool = False,
179
+ ) -> Iterator[Any]:
48
180
  """Tail several files at once, round-robin, yielding lines as they appear.
49
181
 
182
+ Survives log rotation on every path independently: renamed-and-
183
+ recreated files are reopened from the start (after draining the old
184
+ descriptor), copy-truncated files are rewound, and paths that
185
+ disappear mid-rotation are retried until a replacement appears.
186
+
50
187
  ``max_idle_polls`` bounds how many consecutive empty polls are allowed
51
188
  before the generator stops; ``None`` follows forever.
189
+
190
+ With ``tag_sources`` each item is ``(path, line)``: the round-robin
191
+ interleaves files, so continuation context downstream must be keyed
192
+ per file rather than shared across them.
52
193
  """
53
194
 
54
- handles = [path.open("r", encoding="utf-8", errors="replace") for path in paths]
195
+ followers = [_Follower(path) for path in paths]
55
196
  idle_polls = 0
56
197
  try:
57
198
  while True:
58
- got_line = False
59
- for handle in handles:
60
- while True:
61
- line = handle.readline()
62
- if not line:
63
- break
64
- got_line = True
65
- yield line.rstrip("\n")
66
- if got_line:
199
+ any_line = False
200
+ for follower in followers:
201
+ source = str(follower.path)
202
+ for line in follower.read_lines():
203
+ any_line = True
204
+ yield (source, line) if tag_sources else line
205
+ # Rotation is checked for every follower on every pass —
206
+ # including busy ones — so a file renamed away while still
207
+ # being written cannot pin this follower to the stale
208
+ # descriptor and starve the replacement at the origin path.
209
+ reopened = False
210
+ for follower in followers:
211
+ if follower.check_rotation():
212
+ reopened = True
213
+ # A reopened replacement must be read before the idle limit
214
+ # applies, so treat a reopen as activity: loop again to read
215
+ # it rather than counting this pass as idle.
216
+ if any_line or reopened:
67
217
  idle_polls = 0
68
218
  continue
69
219
  idle_polls += 1
@@ -71,8 +221,8 @@ def follow_files(
71
221
  return
72
222
  time.sleep(poll_interval)
73
223
  finally:
74
- for handle in handles:
75
- handle.close()
224
+ for follower in followers:
225
+ follower.close()
76
226
 
77
227
 
78
228
  # Portable strftime codes with at-coarsest one-second granularity.
@@ -173,7 +323,7 @@ def write_lines(
173
323
  fp.close()
174
324
 
175
325
 
176
- def _convert_line_mp(line: str, mode: Optional[str], mapping: Optional[str], validate: bool = False, strict: bool = False) -> str:
326
+ def _convert_line_mp(line: str, mode: Optional[str], mapping: Any, validate: bool = False, strict: bool = False) -> str:
177
327
  return convert_line(line, mode=mode, mapping=mapping, validate=validate, strict=strict)
178
328
 
179
329
 
@@ -189,11 +339,19 @@ def _mp_load_patterns(pattern_files: list) -> None:
189
339
  load_patterns(path)
190
340
 
191
341
 
342
+ def _split_item(item: Any) -> tuple:
343
+ # Line iterators may tag items as (source, line); untagged plain
344
+ # strings share the "" source.
345
+ if isinstance(item, tuple):
346
+ return item
347
+ return ("", item)
348
+
349
+
192
350
  def process_lines(
193
- lines: Iterable[str],
351
+ lines: Iterable[Any],
194
352
  *,
195
353
  mode: Optional[str],
196
- mapping: Optional[str],
354
+ mapping: Any,
197
355
  use_multiprocessing: bool,
198
356
  pool_size: Optional[int],
199
357
  validate: bool = False,
@@ -204,11 +362,17 @@ def process_lines(
204
362
  size = pool_size or max(1, cpu_count() - 1)
205
363
  with Pool(size, initializer=_mp_load_patterns, initargs=(pattern_files or [],)) as pool:
206
364
  worker = partial(_convert_line_mp, mode=mode, mapping=mapping, validate=validate, strict=strict)
207
- for cef in pool.imap(worker, lines):
365
+ for cef in pool.imap(worker, (_split_item(item)[1] for item in lines)):
208
366
  yield cef
209
367
  else:
210
- for line in lines:
211
- yield convert_line(line, mode=mode, mapping=mapping, validate=validate, strict=strict)
368
+ # Stateful conversion so whitespace-indented continuation lines
369
+ # inherit host/app/timestamp from the preceding event, keyed by
370
+ # source so interleaved files never cross-inherit. Unavailable
371
+ # under --multiprocess, where workers cannot share line order.
372
+ converter = StreamConverter(mode=mode, mapping=mapping, validate=validate, strict=strict)
373
+ for item in lines:
374
+ source, line = _split_item(item)
375
+ yield converter.convert(line, source=source)
212
376
 
213
377
 
214
378
  def _optional_path(value: str) -> Optional[Path]:
@@ -238,6 +402,20 @@ def main(argv: Optional[list[str]] = None) -> int:
238
402
  args = parser.parse_args(argv)
239
403
  logging.basicConfig(level=getattr(logging, args.log_level.upper(), logging.WARNING))
240
404
 
405
+ # Reject invalid option combinations before any file, socket, or
406
+ # pool side effects (#89): mistakes in service environment files
407
+ # must fail with a clear message, not a traceback or silence.
408
+ if args.pool_size is not None and not args.multiprocess:
409
+ parser.error("--pool-size requires --multiprocess")
410
+ if args.multiprocess and args.pool_size is not None and args.pool_size < 1:
411
+ parser.error(f"--pool-size must be a positive integer (got {args.pool_size})")
412
+ if args.eps is not None and not args.send:
413
+ parser.error("--eps requires --send")
414
+ if args.listen and args.paths:
415
+ parser.error("--listen replaces file input; remove the positional paths")
416
+ if args.tail and not args.paths:
417
+ parser.error("--tail requires input file paths")
418
+
241
419
  if args.output:
242
420
  try:
243
421
  template_has_codes(str(args.output))
@@ -260,7 +438,25 @@ def main(argv: Optional[list[str]] = None) -> int:
260
438
  if args.mode not in PARSERS and get_registered(args.mode) is None:
261
439
  parser.error(f"unknown --mode '{args.mode}' (not a built-in parser or a loaded pattern name)")
262
440
 
263
- mapping = args.mapping
441
+ # Load and validate the mapping eagerly (#84): a missing or malformed
442
+ # file must fail before the output is opened — a truncate-mode open
443
+ # would destroy an existing archive before the lazy first-event load
444
+ # ever ran. Loading once here also avoids re-reading the JSON file
445
+ # for every converted line.
446
+ mapping = None
447
+ if args.mapping is not None:
448
+ # An explicitly empty --mapping (e.g. MAPPING= blank in a service
449
+ # environment file) is a misconfiguration, not "use the default":
450
+ # silently proceeding would truncate an existing --output with an
451
+ # unintended mapping. Fail before any file is opened.
452
+ if not args.mapping.strip():
453
+ parser.error("--mapping was given an empty value")
454
+ from .api import _load_mapping
455
+
456
+ try:
457
+ mapping = _load_mapping(args.mapping)
458
+ except (OSError, ValueError) as exc:
459
+ parser.error(f"invalid --mapping: {exc}")
264
460
 
265
461
  outputs: Iterator[str]
266
462
  if args.listen:
@@ -273,9 +469,9 @@ def main(argv: Optional[list[str]] = None) -> int:
273
469
  lines = listen_lines(proto, host, port)
274
470
  outputs = process_lines(lines, mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size, validate=args.validate or args.strict, strict=args.strict, pattern_files=args.patterns)
275
471
  elif args.tail and args.paths:
276
- outputs = process_lines(follow_files(args.paths), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size, validate=args.validate or args.strict, strict=args.strict, pattern_files=args.patterns)
472
+ outputs = process_lines(follow_files(args.paths, tag_sources=True), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size, validate=args.validate or args.strict, strict=args.strict, pattern_files=args.patterns)
277
473
  else:
278
- outputs = process_lines(iter_lines_from_sources(args.paths), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size, validate=args.validate or args.strict, strict=args.strict, pattern_files=args.patterns)
474
+ outputs = process_lines(iter_lines_from_sources(args.paths, tag_sources=True), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size, validate=args.validate or args.strict, strict=args.strict, pattern_files=args.patterns)
279
475
 
280
476
  sender = None
281
477
  if args.send:
@@ -7,6 +7,7 @@
7
7
  "extensions": {
8
8
  "dhost": "%(host)s",
9
9
  "sproc": "%(app)s",
10
+ "dvcpid": "%(dvcpid)s",
10
11
  "cs1Label": "rawEvent",
11
12
  "cs1": "%(raw)s"
12
13
  }