syslog2cef 0.3.0__tar.gz → 0.3.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. {syslog2cef-0.3.0/syslog2cef.egg-info → syslog2cef-0.3.1}/PKG-INFO +1 -1
  2. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/pyproject.toml +1 -1
  3. {syslog2cef-0.3.0 → syslog2cef-0.3.1/syslog2cef.egg-info}/PKG-INFO +1 -1
  4. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/syslogcef/api.py +36 -8
  5. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/syslogcef/cli.py +38 -12
  6. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/syslogcef/mappings/linux.json +1 -1
  7. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/syslogcef/normalizer.py +9 -0
  8. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/tests/test_macos_logs.py +92 -0
  9. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/LICENSE +0 -0
  10. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/README.md +0 -0
  11. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/setup.cfg +0 -0
  12. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/syslog2cef.egg-info/SOURCES.txt +0 -0
  13. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/syslog2cef.egg-info/dependency_links.txt +0 -0
  14. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/syslog2cef.egg-info/entry_points.txt +0 -0
  15. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/syslog2cef.egg-info/requires.txt +0 -0
  16. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/syslog2cef.egg-info/top_level.txt +0 -0
  17. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/syslogcef/__init__.py +0 -0
  18. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/syslogcef/__main__.py +0 -0
  19. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/syslogcef/adaptive.py +0 -0
  20. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/syslogcef/cef.py +0 -0
  21. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/syslogcef/custom.py +0 -0
  22. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/syslogcef/dictionary.json +0 -0
  23. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/syslogcef/dictionary.py +0 -0
  24. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/syslogcef/mappings/__init__.py +0 -0
  25. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/syslogcef/mappings/cisco_asa.json +0 -0
  26. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/syslogcef/mappings/cisco_ios.json +0 -0
  27. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/syslogcef/mappings/f5.json +0 -0
  28. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/syslogcef/mappings/fortinet.json +0 -0
  29. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/syslogcef/mappings/sophos.json +0 -0
  30. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/syslogcef/mappings/vmware.json +0 -0
  31. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/syslogcef/net.py +0 -0
  32. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/syslogcef/parsers.py +0 -0
  33. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/syslogcef/utils.py +0 -0
  34. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/syslogcef/validation.py +0 -0
  35. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/tests/test_adaptive.py +0 -0
  36. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/tests/test_cef.py +0 -0
  37. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/tests/test_cli.py +0 -0
  38. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/tests/test_cli_validation.py +0 -0
  39. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/tests/test_custom_parsers.py +0 -0
  40. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/tests/test_dictionary.py +0 -0
  41. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/tests/test_fuzz_grammar.py +0 -0
  42. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/tests/test_fuzz_properties.py +0 -0
  43. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/tests/test_net.py +0 -0
  44. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/tests/test_normalizer.py +0 -0
  45. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/tests/test_parsers.py +0 -0
  46. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/tests/test_parsers_extra.py +0 -0
  47. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/tests/test_pipeline.py +0 -0
  48. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/tests/test_real_world.py +0 -0
  49. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/tests/test_rfc5424_epoch.py +0 -0
  50. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/tests/test_security_hardening.py +0 -0
  51. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/tests/test_tail_rotation.py +0 -0
  52. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/tests/test_utils.py +0 -0
  53. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/tests/test_validation.py +0 -0
  54. {syslog2cef-0.3.0 → syslog2cef-0.3.1}/tests/test_zipapp_safe.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: syslog2cef
3
- Version: 0.3.0
3
+ Version: 0.3.1
4
4
  Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
5
  Author-email: Tamir Suliman <allamiro@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/allamiro/syslogcef
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "syslog2cef"
7
- version = "0.3.0"
7
+ version = "0.3.1"
8
8
  description = "Convert RFC3164/RFC5424 syslog lines into ArcSight CEF"
9
9
  authors = [{name = "Tamir Suliman", email = "allamiro@gmail.com"}]
10
10
  readme = "README.md"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: syslog2cef
3
- Version: 0.3.0
3
+ Version: 0.3.1
4
4
  Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
5
  Author-email: Tamir Suliman <allamiro@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/allamiro/syslogcef
@@ -7,8 +7,9 @@ from datetime import datetime
7
7
  from pathlib import Path
8
8
  from typing import Any, Mapping, Optional
9
9
 
10
- from .parsers import ParsedEvent, autodetect_and_parse
10
+ from .parsers import ParsedEvent, ParserError, autodetect_and_parse
11
11
  from .normalizer import NormalizedEvent, normalize
12
+ from .utils import sanitize_message
12
13
  from .mappings import CISCO_ASA, CISCO_IOS, F5, FORTINET, LINUX, SOPHOS, VMWARE
13
14
  from .cef import CEFEvent, build_cef
14
15
 
@@ -132,6 +133,11 @@ class StreamConverter:
132
133
  inherits host, app, pid, PRI, and timestamp from the most recent
133
134
  fully-parsed event and is tagged ``source_hint="continuation"``. One
134
135
  CEF record is still emitted per input line.
136
+
137
+ Context is tracked per ``source`` (e.g. one entry per input file), so
138
+ interleaved streams — several files tailed at once — never inherit
139
+ another file's host or timestamp. Callers with a single stream can
140
+ ignore the parameter.
135
141
  """
136
142
 
137
143
  def __init__(
@@ -146,12 +152,34 @@ class StreamConverter:
146
152
  self.mapping = mapping
147
153
  self.validate = validate
148
154
  self.strict = strict
149
- self._context: Optional[ParsedEvent] = None
150
-
151
- def convert(self, line: str, *, now: Optional[datetime] = None) -> str:
152
- parsed = parse_syslog(line, mode=self.mode, now=now)
153
- if line[:1] in ("\t", " ") and self._context is not None:
154
- ctx = self._context
155
+ self._contexts: dict[str, ParsedEvent] = {}
156
+
157
+ def convert(self, line: str, *, now: Optional[datetime] = None, source: str = "") -> str:
158
+ ctx = self._contexts.get(source)
159
+ is_continuation = line[:1] in ("\t", " ") and ctx is not None
160
+ try:
161
+ parsed = parse_syslog(line, mode=self.mode, now=now)
162
+ except ParserError:
163
+ if not is_continuation:
164
+ raise
165
+ # A forced --mode rejects headerless continuation lines; a
166
+ # continuation with context must inherit, not abort the run.
167
+ parsed = ParsedEvent(
168
+ pri=None,
169
+ facility=None,
170
+ severity=None,
171
+ ts=None,
172
+ ts_orig="",
173
+ host=None,
174
+ app=None,
175
+ pid=None,
176
+ msgid=None,
177
+ sd={},
178
+ msg=sanitize_message(line),
179
+ raw=line,
180
+ source_hint="unknown",
181
+ )
182
+ if is_continuation:
155
183
  parsed.host = ctx.host
156
184
  parsed.app = parsed.app or ctx.app
157
185
  parsed.pid = parsed.pid or ctx.pid
@@ -166,7 +194,7 @@ class StreamConverter:
166
194
  elif parsed.host is not None and parsed.source_hint != "unknown":
167
195
  # Only a line whose host was actually parsed (not the
168
196
  # guessed fallback) may become continuation context.
169
- self._context = parsed
197
+ self._contexts[source] = parsed
170
198
  normalized = normalize_event(parsed)
171
199
  return to_cef(normalized, self.mapping, validate=self.validate, strict=self.strict)
172
200
 
@@ -15,16 +15,26 @@ from .api import StreamConverter, convert_line
15
15
  logger = logging.getLogger(__name__)
16
16
 
17
17
 
18
- def iter_lines_from_sources(paths: Iterable[Path] | None) -> Iterator[str]:
18
+ def iter_lines_from_sources(paths: Iterable[Path] | None, *, tag_sources: bool = False) -> Iterator[Any]:
19
+ """Yield lines from stdin or the given files.
20
+
21
+ With ``tag_sources`` each item is ``(source, line)`` so downstream
22
+ continuation handling can keep per-file context instead of letting
23
+ an indented line at the top of one file inherit the previous file's
24
+ host and timestamp.
25
+ """
19
26
  if not paths:
20
27
  for line in sys.stdin:
21
- yield line.rstrip("\n")
28
+ line = line.rstrip("\n")
29
+ yield ("stdin", line) if tag_sources else line
22
30
  return
23
31
 
24
32
  for path in paths:
33
+ source = str(path)
25
34
  with path.open("r", encoding="utf-8", errors="replace") as handle:
26
35
  for line in handle:
27
- yield line.rstrip("\n")
36
+ line = line.rstrip("\n")
37
+ yield (source, line) if tag_sources else line
28
38
 
29
39
 
30
40
  def tail_file(path: Path, *, follow: bool = False) -> Iterator[str]:
@@ -165,7 +175,8 @@ def follow_files(
165
175
  *,
166
176
  poll_interval: float = 0.5,
167
177
  max_idle_polls: Optional[int] = None,
168
- ) -> Iterator[str]:
178
+ tag_sources: bool = False,
179
+ ) -> Iterator[Any]:
169
180
  """Tail several files at once, round-robin, yielding lines as they appear.
170
181
 
171
182
  Survives log rotation on every path independently: renamed-and-
@@ -175,6 +186,10 @@ def follow_files(
175
186
 
176
187
  ``max_idle_polls`` bounds how many consecutive empty polls are allowed
177
188
  before the generator stops; ``None`` follows forever.
189
+
190
+ With ``tag_sources`` each item is ``(path, line)``: the round-robin
191
+ interleaves files, so continuation context downstream must be keyed
192
+ per file rather than shared across them.
178
193
  """
179
194
 
180
195
  followers = [_Follower(path) for path in paths]
@@ -183,9 +198,10 @@ def follow_files(
183
198
  while True:
184
199
  any_line = False
185
200
  for follower in followers:
201
+ source = str(follower.path)
186
202
  for line in follower.read_lines():
187
203
  any_line = True
188
- yield line
204
+ yield (source, line) if tag_sources else line
189
205
  # Rotation is checked for every follower on every pass —
190
206
  # including busy ones — so a file renamed away while still
191
207
  # being written cannot pin this follower to the stale
@@ -323,8 +339,16 @@ def _mp_load_patterns(pattern_files: list) -> None:
323
339
  load_patterns(path)
324
340
 
325
341
 
342
+ def _split_item(item: Any) -> tuple:
343
+ # Line iterators may tag items as (source, line); untagged plain
344
+ # strings share the "" source.
345
+ if isinstance(item, tuple):
346
+ return item
347
+ return ("", item)
348
+
349
+
326
350
  def process_lines(
327
- lines: Iterable[str],
351
+ lines: Iterable[Any],
328
352
  *,
329
353
  mode: Optional[str],
330
354
  mapping: Any,
@@ -338,15 +362,17 @@ def process_lines(
338
362
  size = pool_size or max(1, cpu_count() - 1)
339
363
  with Pool(size, initializer=_mp_load_patterns, initargs=(pattern_files or [],)) as pool:
340
364
  worker = partial(_convert_line_mp, mode=mode, mapping=mapping, validate=validate, strict=strict)
341
- for cef in pool.imap(worker, lines):
365
+ for cef in pool.imap(worker, (_split_item(item)[1] for item in lines)):
342
366
  yield cef
343
367
  else:
344
368
  # Stateful conversion so whitespace-indented continuation lines
345
- # inherit host/app/timestamp from the preceding event. Unavailable
369
+ # inherit host/app/timestamp from the preceding event, keyed by
370
+ # source so interleaved files never cross-inherit. Unavailable
346
371
  # under --multiprocess, where workers cannot share line order.
347
372
  converter = StreamConverter(mode=mode, mapping=mapping, validate=validate, strict=strict)
348
- for line in lines:
349
- yield converter.convert(line)
373
+ for item in lines:
374
+ source, line = _split_item(item)
375
+ yield converter.convert(line, source=source)
350
376
 
351
377
 
352
378
  def _optional_path(value: str) -> Optional[Path]:
@@ -443,9 +469,9 @@ def main(argv: Optional[list[str]] = None) -> int:
443
469
  lines = listen_lines(proto, host, port)
444
470
  outputs = process_lines(lines, mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size, validate=args.validate or args.strict, strict=args.strict, pattern_files=args.patterns)
445
471
  elif args.tail and args.paths:
446
- outputs = process_lines(follow_files(args.paths), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size, validate=args.validate or args.strict, strict=args.strict, pattern_files=args.patterns)
472
+ outputs = process_lines(follow_files(args.paths, tag_sources=True), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size, validate=args.validate or args.strict, strict=args.strict, pattern_files=args.patterns)
447
473
  else:
448
- outputs = process_lines(iter_lines_from_sources(args.paths), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size, validate=args.validate or args.strict, strict=args.strict, pattern_files=args.patterns)
474
+ outputs = process_lines(iter_lines_from_sources(args.paths, tag_sources=True), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size, validate=args.validate or args.strict, strict=args.strict, pattern_files=args.patterns)
449
475
 
450
476
  sender = None
451
477
  if args.send:
@@ -7,7 +7,7 @@
7
7
  "extensions": {
8
8
  "dhost": "%(host)s",
9
9
  "sproc": "%(app)s",
10
- "dvcpid": "%(pid)s",
10
+ "dvcpid": "%(dvcpid)s",
11
11
  "cs1Label": "rawEvent",
12
12
  "cs1": "%(raw)s"
13
13
  }
@@ -58,6 +58,15 @@ def normalize(event: ParsedEvent) -> NormalizedEvent:
58
58
  "rt": str(int(ts.timestamp() * 1000)),
59
59
  }
60
60
 
61
+ # dvcpid is an integer field in the CEF dictionary, but RFC 5424
62
+ # PROCID (and kv pids) may be arbitrary strings ("worker-A"); expose
63
+ # a numeric-only alias so mappings can emit dvcpid without tripping
64
+ # strict validation. ASCII check: isdigit() alone accepts Unicode
65
+ # digits that int() consumers reject.
66
+ pid = str(event.pid) if event.pid is not None else ""
67
+ if pid.isascii() and pid.isdigit():
68
+ extras["dvcpid"] = pid
69
+
61
70
  if event.source_hint in {"rfc3164", "rsyslog", "journald"} and event.host is None:
62
71
  extras["host"] = "localhost"
63
72
 
@@ -140,3 +140,95 @@ class TestEventTimeInOutput:
140
140
  datetime(2026, 7, 20, 3, 23, 5, tzinfo=timezone(timedelta(hours=2))).timestamp() * 1000
141
141
  )
142
142
  assert f"rt={expected_ms}" in cef
143
+
144
+
145
+ class TestPerSourceContext:
146
+ """Continuation context must not leak across input sources (files)."""
147
+
148
+ def test_continuation_does_not_inherit_across_sources(self):
149
+ conv = StreamConverter()
150
+ conv.convert("May 28 16:04:52 MacBookPro opendirectoryd[69]: parent", source="a.log")
151
+ cef = conv.convert("\tindented start of another file", source="b.log")
152
+ assert "dhost=MacBookPro" not in cef
153
+
154
+ def test_each_source_keeps_its_own_context(self):
155
+ conv = StreamConverter()
156
+ conv.convert("May 28 16:04:52 host-a appa[1]: parent A", source="a.log")
157
+ conv.convert("May 28 16:04:53 host-b appb[2]: parent B", source="b.log")
158
+ # Interleaved continuations inherit from their own file.
159
+ cef_a = conv.convert("\tcontinuation for A", source="a.log")
160
+ cef_b = conv.convert("\tcontinuation for B", source="b.log")
161
+ assert "dhost=host-a" in cef_a and "sproc=appa" in cef_a
162
+ assert "dhost=host-b" in cef_b and "sproc=appb" in cef_b
163
+
164
+ def test_process_lines_tags_sources(self):
165
+ from syslogcef.cli import process_lines
166
+
167
+ items = [
168
+ ("a.log", "May 28 16:04:52 host-a appa[1]: parent A"),
169
+ ("b.log", "\tindented first line of file B"),
170
+ ]
171
+ out = list(
172
+ process_lines(items, mode=None, mapping=None, use_multiprocessing=False, pool_size=None)
173
+ )
174
+ assert "dhost=host-a" in out[0]
175
+ assert "dhost=host-a" not in out[1]
176
+
177
+
178
+ class TestForcedModeContinuation:
179
+ """A forced --mode must inherit continuation lines, not abort on them."""
180
+
181
+ def test_forced_mode_continuation_inherits(self):
182
+ conv = StreamConverter(mode="journald_short")
183
+ parent = conv.convert("May 28 16:04:52 MacBookPro opendirectoryd[69]: parent event")
184
+ cont = conv.convert("\t wrapped payload line")
185
+ assert "dhost=MacBookPro" in parent
186
+ assert "dhost=MacBookPro" in cont
187
+ assert "sproc=opendirectoryd" in cont
188
+
189
+ def test_forced_mode_still_rejects_non_continuations(self):
190
+ from syslogcef.parsers import ParserError
191
+
192
+ conv = StreamConverter(mode="journald_short")
193
+ with pytest.raises(ParserError):
194
+ conv.convert("not a journald line at all")
195
+
196
+ def test_forced_mode_indented_without_context_still_raises(self):
197
+ from syslogcef.parsers import ParserError
198
+
199
+ conv = StreamConverter(mode="journald_short")
200
+ with pytest.raises(ParserError):
201
+ conv.convert("\tindented but no preceding event")
202
+
203
+
204
+ class TestDvcpidNumericOnly:
205
+ """dvcpid is an integer CEF field; non-numeric PROCIDs must not land there."""
206
+
207
+ def test_numeric_pid_emits_dvcpid(self):
208
+ from syslogcef import convert_line
209
+ from syslogcef.mappings import LINUX
210
+
211
+ cef = convert_line(
212
+ "<34>1 2026-07-20T03:23:05Z host1 myapp 4711 ID47 - message", mapping=LINUX
213
+ )
214
+ assert "dvcpid=4711" in cef
215
+
216
+ def test_nonnumeric_procid_omits_dvcpid(self):
217
+ from syslogcef import convert_line
218
+ from syslogcef.mappings import LINUX
219
+
220
+ cef = convert_line(
221
+ "<34>1 2026-07-20T03:23:05Z host1 myapp worker-A ID47 - message", mapping=LINUX
222
+ )
223
+ assert "dvcpid=" not in cef
224
+
225
+ def test_nonnumeric_procid_passes_strict_validation(self):
226
+ from syslogcef import convert_line
227
+ from syslogcef.mappings import LINUX
228
+
229
+ cef = convert_line(
230
+ "<34>1 2026-07-20T03:23:05Z host1 myapp worker-A ID47 - message",
231
+ mapping=LINUX,
232
+ strict=True,
233
+ )
234
+ assert cef.startswith("CEF:0|")
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes