syslog2cef 0.1.2__tar.gz → 0.1.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. {syslog2cef-0.1.2/syslog2cef.egg-info → syslog2cef-0.1.3}/PKG-INFO +3 -1
  2. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/pyproject.toml +4 -1
  3. {syslog2cef-0.1.2 → syslog2cef-0.1.3/syslog2cef.egg-info}/PKG-INFO +3 -1
  4. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslog2cef.egg-info/SOURCES.txt +7 -1
  5. syslog2cef-0.1.3/syslog2cef.egg-info/requires.txt +6 -0
  6. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/adaptive.py +28 -11
  7. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/api.py +16 -5
  8. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/cef.py +31 -7
  9. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/cli.py +54 -15
  10. syslog2cef-0.1.3/syslogcef/net.py +270 -0
  11. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/parsers.py +16 -3
  12. syslog2cef-0.1.3/syslogcef/validation.py +164 -0
  13. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/tests/test_adaptive.py +54 -0
  14. syslog2cef-0.1.3/tests/test_net.py +180 -0
  15. syslog2cef-0.1.3/tests/test_validation.py +66 -0
  16. syslog2cef-0.1.2/syslog2cef.egg-info/requires.txt +0 -3
  17. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/LICENSE +0 -0
  18. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/README.md +0 -0
  19. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/setup.cfg +0 -0
  20. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslog2cef.egg-info/dependency_links.txt +0 -0
  21. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslog2cef.egg-info/entry_points.txt +0 -0
  22. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslog2cef.egg-info/top_level.txt +0 -0
  23. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/__init__.py +0 -0
  24. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/__main__.py +0 -0
  25. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/mappings/__init__.py +0 -0
  26. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/mappings/cisco_asa.json +0 -0
  27. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/mappings/cisco_ios.json +0 -0
  28. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/mappings/f5.json +0 -0
  29. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/mappings/fortinet.json +0 -0
  30. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/mappings/linux.json +0 -0
  31. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/mappings/sophos.json +0 -0
  32. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/mappings/vmware.json +0 -0
  33. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/normalizer.py +0 -0
  34. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/syslogcef/utils.py +0 -0
  35. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/tests/test_cef.py +0 -0
  36. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/tests/test_cli.py +0 -0
  37. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/tests/test_normalizer.py +0 -0
  38. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/tests/test_parsers.py +0 -0
  39. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/tests/test_parsers_extra.py +0 -0
  40. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/tests/test_pipeline.py +0 -0
  41. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/tests/test_real_world.py +0 -0
  42. {syslog2cef-0.1.2 → syslog2cef-0.1.3}/tests/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: syslog2cef
3
- Version: 0.1.2
3
+ Version: 0.1.3
4
4
  Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
5
  Author-email: Tamir Suliman <allamiro@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/allamiro/syslogcef
@@ -15,6 +15,8 @@ Description-Content-Type: text/markdown
15
15
  License-File: LICENSE
16
16
  Provides-Extra: test
17
17
  Requires-Dist: pytest>=7; extra == "test"
18
+ Provides-Extra: kafka
19
+ Requires-Dist: kafka-python>=2.0; extra == "kafka"
18
20
  Dynamic: license-file
19
21
 
20
22
  <p align="center">
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "syslog2cef"
7
- version = "0.1.2"
7
+ version = "0.1.3"
8
8
  description = "Convert RFC3164/RFC5424 syslog lines into ArcSight CEF"
9
9
  authors = [{name = "Tamir Suliman", email = "allamiro@gmail.com"}]
10
10
  readme = "README.md"
@@ -26,6 +26,9 @@ Issues = "https://github.com/allamiro/syslogcef/issues"
26
26
  test = [
27
27
  "pytest>=7",
28
28
  ]
29
+ kafka = [
30
+ "kafka-python>=2.0",
31
+ ]
29
32
 
30
33
  [project.scripts]
31
34
  syslogcef = "syslogcef.cli:main"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: syslog2cef
3
- Version: 0.1.2
3
+ Version: 0.1.3
4
4
  Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
5
  Author-email: Tamir Suliman <allamiro@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/allamiro/syslogcef
@@ -15,6 +15,8 @@ Description-Content-Type: text/markdown
15
15
  License-File: LICENSE
16
16
  Provides-Extra: test
17
17
  Requires-Dist: pytest>=7; extra == "test"
18
+ Provides-Extra: kafka
19
+ Requires-Dist: kafka-python>=2.0; extra == "kafka"
18
20
  Dynamic: license-file
19
21
 
20
22
  <p align="center">
@@ -7,9 +7,11 @@ pyproject.toml
7
7
  ./syslogcef/api.py
8
8
  ./syslogcef/cef.py
9
9
  ./syslogcef/cli.py
10
+ ./syslogcef/net.py
10
11
  ./syslogcef/normalizer.py
11
12
  ./syslogcef/parsers.py
12
13
  ./syslogcef/utils.py
14
+ ./syslogcef/validation.py
13
15
  ./syslogcef/mappings/__init__.py
14
16
  ./syslogcef/mappings/cisco_asa.json
15
17
  ./syslogcef/mappings/cisco_ios.json
@@ -30,9 +32,11 @@ syslogcef/adaptive.py
30
32
  syslogcef/api.py
31
33
  syslogcef/cef.py
32
34
  syslogcef/cli.py
35
+ syslogcef/net.py
33
36
  syslogcef/normalizer.py
34
37
  syslogcef/parsers.py
35
38
  syslogcef/utils.py
39
+ syslogcef/validation.py
36
40
  syslogcef/mappings/__init__.py
37
41
  syslogcef/mappings/cisco_asa.json
38
42
  syslogcef/mappings/cisco_ios.json
@@ -44,9 +48,11 @@ syslogcef/mappings/vmware.json
44
48
  tests/test_adaptive.py
45
49
  tests/test_cef.py
46
50
  tests/test_cli.py
51
+ tests/test_net.py
47
52
  tests/test_normalizer.py
48
53
  tests/test_parsers.py
49
54
  tests/test_parsers_extra.py
50
55
  tests/test_pipeline.py
51
56
  tests/test_real_world.py
52
- tests/test_utils.py
57
+ tests/test_utils.py
58
+ tests/test_validation.py
@@ -0,0 +1,6 @@
1
+
2
+ [kafka]
3
+ kafka-python>=2.0
4
+
5
+ [test]
6
+ pytest>=7
@@ -31,9 +31,14 @@ def _epoch(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
31
31
  def _mon_day(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
32
32
  month = month_abbr_to_int(match.group("month"))
33
33
  day = int(match.group("day"))
34
- hour, minute, second = map(int, match.group("hms").split(":"))
35
- year = int(match.group("year")) if match.groupdict().get("year") else (now or datetime.now(timezone.utc)).year
36
- return datetime(year, month, day, hour, minute, second, tzinfo=timezone.utc)
34
+ if match.groupdict().get("year"):
35
+ hour, minute, second = map(int, match.group("hms").split(":"))
36
+ return datetime(int(match.group("year")), month, day, hour, minute, second, tzinfo=timezone.utc)
37
+ # No year: use the same rollover inference as the dedicated parsers so a
38
+ # December event parsed in January lands in the previous year.
39
+ from .parsers import _infer_timestamp
40
+
41
+ return _infer_timestamp(month, day, match.group("hms"), now=now)
37
42
 
38
43
 
39
44
  def _ymd_slash(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
@@ -62,6 +67,16 @@ TIMESTAMP_LIB: Tuple[Tuple[str, str, Callable], ...] = (
62
67
  HOST_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
63
68
  _SKIP_HOST_TOKENS = {"error", "warning", "info", "debug", "notice", "kernel"}
64
69
 
70
+
71
+ def _valid_host(candidate: Optional[str]) -> bool:
72
+ return bool(
73
+ candidate
74
+ and "=" not in candidate
75
+ and HOST_RE.match(candidate)
76
+ and candidate.lower() not in _SKIP_HOST_TOKENS
77
+ and not candidate.isdigit()
78
+ )
79
+
65
80
  _CACHE: Dict[str, Tuple[Pattern[str], str, Callable]] = {}
66
81
  _CACHE_MAX = 256
67
82
 
@@ -131,8 +146,15 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
131
146
  ts = conv(pattern_ts_match(pattern, match), now)
132
147
  except (ValueError, KeyError):
133
148
  ts = None
134
- host = match.groupdict().get("adaptive_host")
135
- msg = match.groupdict().get("adaptive_msg", "") or line
149
+ gd = match.groupdict()
150
+ host = gd.get("adaptive_host")
151
+ msg = gd.get("adaptive_msg")
152
+ msg = line if msg is None else msg
153
+ # The shape signature reduces all words to the same class, so a
154
+ # cached host slot may capture a non-host token; revalidate.
155
+ if host is not None and not _valid_host(host.rstrip(":,")):
156
+ msg = f"{host} {msg}".strip()
157
+ host = None
136
158
  return _build_event(raw, pri, ts, host, msg)
137
159
 
138
160
  # Analysis pass: find the earliest timestamp in the line head.
@@ -157,12 +179,7 @@ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
157
179
  tokens = rest.split(None, 1)
158
180
  if tokens:
159
181
  candidate = tokens[0].rstrip(":,")
160
- if (
161
- "=" not in candidate
162
- and HOST_RE.match(candidate)
163
- and candidate.lower() not in _SKIP_HOST_TOKENS
164
- and not candidate.isdigit()
165
- ):
182
+ if _valid_host(candidate):
166
183
  host = candidate
167
184
  msg = tokens[1] if len(tokens) > 1 else ""
168
185
 
@@ -74,14 +74,23 @@ def _load_mapping(mapping: Mapping[str, Any] | Path | str | None) -> Mapping[str
74
74
  def to_cef(
75
75
  event: NormalizedEvent,
76
76
  mapping: Mapping[str, Any] | Path | str | None = None,
77
+ *,
78
+ validate: bool = False,
79
+ strict: bool = False,
77
80
  ) -> str:
78
- """Convert a normalized event into a CEF string."""
81
+ """Convert a normalized event into a CEF string.
82
+
83
+ With ``validate=True``, extension values are checked against the
84
+ ArcSight dictionary and violations logged as warnings; ``strict=True``
85
+ additionally raises :class:`syslogcef.validation.CEFValidationError`
86
+ on type or length violations.
87
+ """
79
88
 
80
89
  if mapping is None:
81
90
  mapping_data = _guess_mapping(event)
82
91
  else:
83
92
  mapping_data = _load_mapping(mapping)
84
- cef_event = build_cef(event, mapping_data)
93
+ cef_event = build_cef(event, mapping_data, validate=validate, strict=strict)
85
94
  return cef_event.render()
86
95
 
87
96
 
@@ -91,13 +100,15 @@ def convert_line(
91
100
  mode: Optional[str] = None,
92
101
  mapping: Mapping[str, Any] | Path | str | None = None,
93
102
  now: Optional[datetime] = None,
103
+ validate: bool = False,
104
+ strict: bool = False,
94
105
  ) -> str:
95
106
  """Full pipeline that parses, normalizes and converts a syslog line."""
96
107
 
97
108
  parsed = parse_syslog(line, mode=mode, now=now)
98
109
  normalized = normalize_event(parsed)
99
110
 
100
- return to_cef(normalized, mapping)
111
+ return to_cef(normalized, mapping, validate=validate, strict=strict)
101
112
 
102
113
 
103
114
  def _guess_mapping(event: NormalizedEvent) -> Mapping[str, Any]:
@@ -111,9 +122,9 @@ def _guess_mapping(event: NormalizedEvent) -> Mapping[str, Any]:
111
122
  return CISCO_IOS
112
123
  if "BIG-IP" in msg_upper or "F5" in app_upper:
113
124
  return F5
114
- if "devname" in event.kv or "logid" in event.kv:
125
+ if "logid" in event.kv:
115
126
  return FORTINET
116
- if "log_id" in event.kv or event.kv.get("device") == "SFW":
127
+ if "log_id" in event.kv:
117
128
  return SOPHOS
118
129
  if "VMWARE" in msg_upper or "ESXI" in msg_upper or "VMWARE" in app_upper:
119
130
  return VMWARE
@@ -56,7 +56,12 @@ class MappingResolver:
56
56
  header = {}
57
57
  for key in ["deviceVendor", "deviceProduct", "deviceVersion", "eventClassId", "name"]:
58
58
  template = self.mapping.get(key, DEFAULT_MAPPING[key])
59
- header[key] = cef_escape_header(self._format(template, fields))
59
+ value = self._format(template, fields)
60
+ if not value:
61
+ # A template referencing missing fields must not leave an
62
+ # empty CEF header slot; fall back to the default template.
63
+ value = self._format(DEFAULT_MAPPING[key], fields)
64
+ header[key] = cef_escape_header(value)
60
65
  severity = self._resolve_severity(fields)
61
66
  header["severity"] = severity
62
67
  return header
@@ -71,15 +76,18 @@ class MappingResolver:
71
76
  return "3"
72
77
  return severity_map.get(str(value), str(value))
73
78
 
74
- def resolve_extensions(self, fields: Mapping[str, Any]) -> Dict[str, str]:
75
- extensions = {}
79
+ def resolve_raw_extensions(self, fields: Mapping[str, Any]) -> Dict[str, str]:
80
+ raw = {}
76
81
  merged = DEFAULT_MAPPING["extensions"].copy()
77
82
  merged.update(self.mapping.get("extensions", {}))
78
83
  for key, template in merged.items():
79
84
  value = self._format(template, fields)
80
85
  if value:
81
- extensions[key] = cef_escape(str(value))
82
- return extensions
86
+ raw[key] = str(value)
87
+ return raw
88
+
89
+ def resolve_extensions(self, fields: Mapping[str, Any]) -> Dict[str, str]:
90
+ return {k: cef_escape(v) for k, v in self.resolve_raw_extensions(fields).items()}
83
91
 
84
92
  def _format(self, template: str, fields: Mapping[str, Any]) -> str:
85
93
  try:
@@ -92,11 +100,27 @@ class MappingResolver:
92
100
  return ""
93
101
 
94
102
 
95
- def build_cef(event: NormalizedEvent, mapping: Mapping[str, Any]) -> CEFEvent:
103
+ def build_cef(
104
+ event: NormalizedEvent,
105
+ mapping: Mapping[str, Any],
106
+ *,
107
+ validate: bool = False,
108
+ strict: bool = False,
109
+ ) -> CEFEvent:
96
110
  resolver = MappingResolver(mapping)
97
111
  fields = event.as_field_dict()
98
112
  header = resolver.resolve_header(fields)
99
- extensions = resolver.resolve_extensions(fields)
113
+ raw = resolver.resolve_raw_extensions(fields)
114
+ if validate or strict:
115
+ from .validation import CEFValidationError, validate_extensions
116
+
117
+ findings = validate_extensions(raw)
118
+ for finding in findings:
119
+ logger.warning("CEF validation: %s", finding)
120
+ fatal = [f for f in findings if f.fatal]
121
+ if strict and fatal:
122
+ raise CEFValidationError("; ".join(str(f) for f in fatal))
123
+ extensions = {k: cef_escape(v) for k, v in raw.items()}
100
124
  return CEFEvent(header=header, extensions=extensions)
101
125
 
102
126
 
@@ -74,8 +74,8 @@ def follow_files(
74
74
  handle.close()
75
75
 
76
76
 
77
- def _convert_line_mp(line: str, mode: Optional[str], mapping: Optional[str]) -> str:
78
- return convert_line(line, mode=mode, mapping=mapping)
77
+ def _convert_line_mp(line: str, mode: Optional[str], mapping: Optional[str], validate: bool = False, strict: bool = False) -> str:
78
+ return convert_line(line, mode=mode, mapping=mapping, validate=validate, strict=strict)
79
79
 
80
80
 
81
81
  def process_lines(
@@ -85,16 +85,18 @@ def process_lines(
85
85
  mapping: Optional[str],
86
86
  use_multiprocessing: bool,
87
87
  pool_size: Optional[int],
88
+ validate: bool = False,
89
+ strict: bool = False,
88
90
  ) -> Iterator[str]:
89
91
  if use_multiprocessing:
90
92
  size = pool_size or max(1, cpu_count() - 1)
91
93
  with Pool(size) as pool:
92
- worker = partial(_convert_line_mp, mode=mode, mapping=mapping)
94
+ worker = partial(_convert_line_mp, mode=mode, mapping=mapping, validate=validate, strict=strict)
93
95
  for cef in pool.imap(worker, lines):
94
96
  yield cef
95
97
  else:
96
98
  for line in lines:
97
- yield convert_line(line, mode=mode, mapping=mapping)
99
+ yield convert_line(line, mode=mode, mapping=mapping, validate=validate, strict=strict)
98
100
 
99
101
 
100
102
  def main(argv: Optional[list[str]] = None) -> int:
@@ -105,8 +107,13 @@ def main(argv: Optional[list[str]] = None) -> int:
105
107
  parser.add_argument("--mode", choices=["rfc3164", "rfc5424", "rsyslog_json", "rsyslog_file", "journald_json", "journald_short", "journald_iso", "cisco_seq", "iso_syslog", "kv"], help="Parser mode override")
106
108
  parser.add_argument("--mapping", type=str, help="Mapping JSON file")
107
109
  parser.add_argument("--tail", action="store_true", help="Follow file like tail -f")
110
+ parser.add_argument("--listen", metavar="PROTO:PORT", help="Receive syslog over the network instead of reading files/stdin, e.g. udp:514, tcp:5514, or udp:10.0.0.5:514")
111
+ parser.add_argument("--send", metavar="URL", help="Forward CEF records to udp://HOST:PORT, tcp://HOST:PORT (newline-delimited, with reconnect), or kafka://BROKER:PORT/TOPIC (requires the kafka extra)")
112
+ parser.add_argument("--eps", type=float, help="Rate-limit --send to at most this many events per second")
108
113
  parser.add_argument("--multiprocess", action="store_true", help="Process lines using a process pool")
109
114
  parser.add_argument("--pool-size", type=int, help="Number of worker processes for --multiprocess")
115
+ parser.add_argument("--validate", action="store_true", help="Validate CEF extensions against the ArcSight dictionary; violations are logged as warnings")
116
+ parser.add_argument("--strict", action="store_true", help="Like --validate, but exit with an error on type/length violations")
110
117
  parser.add_argument("--log-level", default="WARNING", help="Logging level")
111
118
 
112
119
  args = parser.parse_args(argv)
@@ -115,20 +122,52 @@ def main(argv: Optional[list[str]] = None) -> int:
115
122
  mapping = args.mapping
116
123
 
117
124
  outputs: Iterator[str]
118
- if args.tail and args.paths:
119
- outputs = process_lines(follow_files(args.paths), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size)
125
+ if args.listen:
126
+ from .net import listen_lines, parse_endpoint
127
+
128
+ try:
129
+ proto, host, port = parse_endpoint(args.listen)
130
+ except ValueError as exc:
131
+ parser.error(str(exc))
132
+ lines = listen_lines(proto, host, port)
133
+ outputs = process_lines(lines, mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size, validate=args.validate or args.strict, strict=args.strict)
134
+ elif args.tail and args.paths:
135
+ outputs = process_lines(follow_files(args.paths), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size, validate=args.validate or args.strict, strict=args.strict)
120
136
  else:
121
- outputs = process_lines(iter_lines_from_sources(args.paths), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size)
137
+ outputs = process_lines(iter_lines_from_sources(args.paths), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size, validate=args.validate or args.strict, strict=args.strict)
122
138
 
123
- if args.output:
124
- with args.output.open("a" if args.append else "w", encoding="utf-8") as fp:
139
+ sender = None
140
+ if args.send:
141
+ from .net import create_sender
142
+
143
+ try:
144
+ sender = create_sender(args.send, eps=args.eps)
145
+ except (ValueError, RuntimeError) as exc:
146
+ parser.error(str(exc))
147
+
148
+ from .validation import CEFValidationError
149
+
150
+ try:
151
+ if args.output:
152
+ with args.output.open("a" if args.append else "w", encoding="utf-8") as fp:
153
+ for cef in outputs:
154
+ fp.write(cef + "\n")
155
+ if args.tail or args.listen:
156
+ fp.flush()
157
+ if sender:
158
+ sender.send(cef)
159
+ elif sender:
125
160
  for cef in outputs:
126
- fp.write(cef + "\n")
127
- if args.tail:
128
- fp.flush()
129
- else:
130
- for cef in outputs:
131
- print(cef)
161
+ sender.send(cef)
162
+ else:
163
+ for cef in outputs:
164
+ print(cef)
165
+ except CEFValidationError as exc:
166
+ print(f"syslogcef: strict validation failed: {exc}", file=sys.stderr)
167
+ return 1
168
+ finally:
169
+ if sender:
170
+ sender.close()
132
171
  return 0
133
172
 
134
173
 
@@ -0,0 +1,270 @@
1
+ """Network input for syslogcef: receive syslog over UDP or TCP.
2
+
3
+ The listener yields raw lines exactly as the rest of the pipeline expects
4
+ them, so ``--listen`` composes with every existing option (mappings, mode
5
+ overrides, outputs).
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import logging
11
+ import selectors
12
+ import socket
13
+ from typing import Callable, Iterator, Optional, Tuple
14
+
15
+ logger = logging.getLogger(__name__)
16
+
17
+ MAX_DATAGRAM = 65535
18
+
19
+
20
+ def parse_endpoint(spec: str) -> Tuple[str, str, int]:
21
+ """Parse ``udp:514``, ``tcp:5514``, or ``udp:10.0.0.5:514``."""
22
+
23
+ parts = spec.split(":")
24
+ proto = parts[0].lower()
25
+ if proto not in ("udp", "tcp") or len(parts) not in (2, 3):
26
+ raise ValueError(
27
+ f"Invalid endpoint {spec!r}; expected udp:PORT, tcp:PORT, or udp:HOST:PORT"
28
+ )
29
+ host = parts[1] if len(parts) == 3 else "0.0.0.0"
30
+ try:
31
+ port = int(parts[-1])
32
+ except ValueError as exc:
33
+ raise ValueError(f"Invalid port in endpoint {spec!r}") from exc
34
+ if not 0 <= port <= 65535:
35
+ raise ValueError(f"Port out of range in endpoint {spec!r}")
36
+ return proto, host, port
37
+
38
+
39
+ def listen_lines(
40
+ proto: str,
41
+ host: str,
42
+ port: int,
43
+ *,
44
+ max_messages: Optional[int] = None,
45
+ ready_callback: Optional[Callable[[int], None]] = None,
46
+ ) -> Iterator[str]:
47
+ """Yield syslog lines received on the given endpoint.
48
+
49
+ ``max_messages`` bounds the number of yielded lines (used by tests);
50
+ ``None`` listens forever. ``ready_callback`` receives the bound port
51
+ once the socket is listening (useful with port 0).
52
+ """
53
+
54
+ if proto == "udp":
55
+ yield from _listen_udp(host, port, max_messages, ready_callback)
56
+ else:
57
+ yield from _listen_tcp(host, port, max_messages, ready_callback)
58
+
59
+
60
+ def _listen_udp(host, port, max_messages, ready_callback) -> Iterator[str]:
61
+ sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
62
+ sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
63
+ sock.bind((host, port))
64
+ if ready_callback:
65
+ ready_callback(sock.getsockname()[1])
66
+ logger.info("Listening on udp:%s:%s", *sock.getsockname())
67
+ count = 0
68
+ try:
69
+ while max_messages is None or count < max_messages:
70
+ data, _addr = sock.recvfrom(MAX_DATAGRAM)
71
+ for raw in data.decode("utf-8", "replace").splitlines():
72
+ if raw.strip():
73
+ yield raw
74
+ count += 1
75
+ if max_messages is not None and count >= max_messages:
76
+ return
77
+ finally:
78
+ sock.close()
79
+
80
+
81
+ def _listen_tcp(host, port, max_messages, ready_callback) -> Iterator[str]:
82
+ server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
83
+ server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
84
+ server.bind((host, port))
85
+ server.listen(16)
86
+ server.setblocking(False)
87
+ if ready_callback:
88
+ ready_callback(server.getsockname()[1])
89
+ logger.info("Listening on tcp:%s:%s", *server.getsockname())
90
+
91
+ sel = selectors.DefaultSelector()
92
+ sel.register(server, selectors.EVENT_READ)
93
+ buffers: dict[socket.socket, bytes] = {}
94
+ count = 0
95
+ try:
96
+ while max_messages is None or count < max_messages:
97
+ for key, _events in sel.select(timeout=1):
98
+ sock = key.fileobj
99
+ if sock is server:
100
+ conn, addr = server.accept()
101
+ conn.setblocking(False)
102
+ sel.register(conn, selectors.EVENT_READ)
103
+ buffers[conn] = b""
104
+ logger.debug("Connection from %s", addr)
105
+ continue
106
+ try:
107
+ data = sock.recv(65536)
108
+ except (BlockingIOError, InterruptedError):
109
+ continue
110
+ except ConnectionError:
111
+ data = b""
112
+ if not data:
113
+ sel.unregister(sock)
114
+ sock.close()
115
+ remainder = buffers.pop(sock, b"")
116
+ line = remainder.decode("utf-8", "replace").strip()
117
+ if line:
118
+ yield line
119
+ count += 1
120
+ continue
121
+ buffers[sock] += data
122
+ while b"\n" in buffers[sock]:
123
+ raw, buffers[sock] = buffers[sock].split(b"\n", 1)
124
+ line = raw.decode("utf-8", "replace").rstrip("\r")
125
+ if line.strip():
126
+ yield line
127
+ count += 1
128
+ if max_messages is not None and count >= max_messages:
129
+ return
130
+ finally:
131
+ for sock in list(buffers):
132
+ sock.close()
133
+ sel.close()
134
+ server.close()
135
+
136
+
137
+ __all__ = ["listen_lines", "parse_endpoint"]
138
+
139
+
140
+ # --------------------------------------------------------------------------
141
+ # Network output: forward CEF records to a SIEM or pipeline.
142
+
143
+ import time as _time
144
+ from urllib.parse import urlparse
145
+
146
+
147
+ class RateLimiter:
148
+ """Simple events-per-second pacer."""
149
+
150
+ def __init__(self, eps: float, clock=_time.monotonic, sleep=_time.sleep):
151
+ if eps <= 0:
152
+ raise ValueError("eps must be positive")
153
+ self.interval = 1.0 / eps
154
+ self._clock = clock
155
+ self._sleep = sleep
156
+ self._next = self._clock()
157
+
158
+ def wait(self) -> None:
159
+ now = self._clock()
160
+ if now < self._next:
161
+ self._sleep(self._next - now)
162
+ self._next = max(self._next + self.interval, self._clock())
163
+
164
+
165
+ class SyslogSender:
166
+ """Send CEF records to a syslog destination over UDP or TCP.
167
+
168
+ TCP sends are newline-delimited and reconnect with backoff on failure.
169
+ """
170
+
171
+ RETRY_DELAYS = (0.5, 1.0, 2.0)
172
+
173
+ def __init__(self, proto: str, host: str, port: int, *, limiter: Optional[RateLimiter] = None):
174
+ self.proto = proto
175
+ self.host = host
176
+ self.port = port
177
+ self.limiter = limiter
178
+ self._sock: Optional[socket.socket] = None
179
+ if proto == "udp":
180
+ self._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
181
+
182
+ def _connect(self) -> socket.socket:
183
+ sock = socket.create_connection((self.host, self.port), timeout=10)
184
+ return sock
185
+
186
+ def send(self, record: str) -> None:
187
+ if self.limiter:
188
+ self.limiter.wait()
189
+ payload = record.encode("utf-8")
190
+ if self.proto == "udp":
191
+ self._sock.sendto(payload, (self.host, self.port))
192
+ return
193
+ data = payload + b"\n"
194
+ last_error: Optional[Exception] = None
195
+ for attempt, delay in enumerate((0.0,) + self.RETRY_DELAYS):
196
+ if delay:
197
+ _time.sleep(delay)
198
+ try:
199
+ if self._sock is None:
200
+ self._sock = self._connect()
201
+ self._sock.sendall(data)
202
+ return
203
+ except OSError as exc:
204
+ last_error = exc
205
+ logger.warning("send to %s:%s failed (%s); reconnecting", self.host, self.port, exc)
206
+ if self._sock is not None:
207
+ try:
208
+ self._sock.close()
209
+ except OSError:
210
+ pass
211
+ self._sock = None
212
+ raise ConnectionError(f"Could not deliver record to {self.host}:{self.port}") from last_error
213
+
214
+ def close(self) -> None:
215
+ if self._sock is not None:
216
+ try:
217
+ self._sock.close()
218
+ finally:
219
+ self._sock = None
220
+
221
+
222
+ class KafkaSender:
223
+ """Send CEF records to a Kafka topic (requires the 'kafka' extra)."""
224
+
225
+ def __init__(self, broker: str, topic: str, *, limiter: Optional[RateLimiter] = None, producer=None):
226
+ self.topic = topic
227
+ self.limiter = limiter
228
+ if producer is None:
229
+ try:
230
+ from kafka import KafkaProducer
231
+ except ImportError as exc:
232
+ raise RuntimeError(
233
+ "Kafka output requires the kafka extra: pip install syslog2cef[kafka]"
234
+ ) from exc
235
+ producer = KafkaProducer(bootstrap_servers=broker)
236
+ self.producer = producer
237
+
238
+ def send(self, record: str) -> None:
239
+ if self.limiter:
240
+ self.limiter.wait()
241
+ self.producer.send(self.topic, record.encode("utf-8"))
242
+
243
+ def close(self) -> None:
244
+ try:
245
+ self.producer.flush()
246
+ finally:
247
+ close = getattr(self.producer, "close", None)
248
+ if close:
249
+ close()
250
+
251
+
252
+ def create_sender(url: str, *, eps: Optional[float] = None):
253
+ """Create a sender from udp://host:port, tcp://host:port, or kafka://broker/topic."""
254
+
255
+ parsed = urlparse(url)
256
+ limiter = RateLimiter(eps) if eps else None
257
+ if parsed.scheme in ("udp", "tcp"):
258
+ if not parsed.hostname or not parsed.port:
259
+ raise ValueError(f"Invalid send target {url!r}; expected {parsed.scheme}://HOST:PORT")
260
+ return SyslogSender(parsed.scheme, parsed.hostname, parsed.port, limiter=limiter)
261
+ if parsed.scheme == "kafka":
262
+ topic = parsed.path.lstrip("/")
263
+ if not parsed.hostname or not topic:
264
+ raise ValueError(f"Invalid send target {url!r}; expected kafka://BROKER:PORT/TOPIC")
265
+ broker = f"{parsed.hostname}:{parsed.port or 9092}"
266
+ return KafkaSender(broker, topic, limiter=limiter)
267
+ raise ValueError(f"Unsupported send scheme {parsed.scheme!r}; use udp://, tcp://, or kafka://")
268
+
269
+
270
+ __all__ += ["RateLimiter", "SyslogSender", "KafkaSender", "create_sender"]
@@ -96,7 +96,7 @@ CISCO_SEQ_RE = re.compile(
96
96
  r"(?P<month>[A-Z][a-z]{2})\s+"
97
97
  r"(?P<day>\d{1,2})\s+"
98
98
  r"(?:(?P<year>\d{4})\s+)?"
99
- r"(?P<time>\d{2}:\d{2}:\d{2})(?:\.\d+)?"
99
+ r"(?P<time>\d{2}:\d{2}:\d{2})(?P<frac>\.\d+)?"
100
100
  r"(?:\s+[A-Z]{3,4})?:\s+"
101
101
  r"(?P<msg>%[A-Z].*)$"
102
102
  )
@@ -369,10 +369,11 @@ def parse_cisco_seq(line: str, *, now: Optional[datetime]) -> ParsedEvent | None
369
369
  facility, severity = convert_pri(pri)
370
370
  month = month_abbr_to_int(gd["month"])
371
371
  hour, minute, second = map(int, gd["time"].split(":"))
372
+ microsecond = int(float(gd["frac"]) * 1_000_000) if gd.get("frac") else 0
372
373
  if gd.get("year"):
373
- ts = datetime(int(gd["year"]), month, int(gd["day"]), hour, minute, second, tzinfo=timezone.utc)
374
+ ts = datetime(int(gd["year"]), month, int(gd["day"]), hour, minute, second, microsecond, tzinfo=timezone.utc)
374
375
  else:
375
- ts = _infer_timestamp(month, int(gd["day"]), gd["time"], now=now)
376
+ ts = _infer_timestamp(month, int(gd["day"]), gd["time"], now=now).replace(microsecond=microsecond)
376
377
  sd = {"cisco.sequence": gd["seq"]} if gd.get("seq") else {}
377
378
  return ParsedEvent(
378
379
  pri=pri,
@@ -444,6 +445,18 @@ def parse_kv_stream(line: str) -> ParsedEvent | None:
444
445
  ts = parse_iso8601(pairs["eventtime"][:16])
445
446
  except ValueError:
446
447
  ts = None
448
+ if ts is not None and ts.tzinfo is None:
449
+ # Apply a numeric offset supplied in the record (tz="-0500");
450
+ # named zones (timezone="CEST") cannot be resolved portably.
451
+ offset_match = re.fullmatch(
452
+ r"([+-])(\d{2}):?(\d{2})", pairs.get("tz") or pairs.get("timezone") or ""
453
+ )
454
+ if offset_match:
455
+ sign = 1 if offset_match.group(1) == "+" else -1
456
+ delta = timedelta(
457
+ hours=int(offset_match.group(2)), minutes=int(offset_match.group(3))
458
+ )
459
+ ts = ts.replace(tzinfo=timezone(sign * delta))
447
460
 
448
461
  host = (
449
462
  pairs.get("devname")
@@ -0,0 +1,164 @@
1
+ """Validation of CEF extension keys and values against the ArcSight dictionary.
2
+
3
+ The key metadata below is derived from the ArcSight Extension Dictionary
4
+ (CEF key names for event producers): data type and maximum string length
5
+ per key. Custom keys are permitted by the CEF standard, so unknown keys
6
+ produce advisory findings only; type and length violations are the
7
+ errors that ``strict`` mode raises on.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import ipaddress
13
+ import re
14
+ from dataclasses import dataclass
15
+ from typing import Dict, List, Tuple
16
+
17
+ # type tags: str (with max length), int, long, float, port, ip (v4/v6),
18
+ # ipv6, mac, ts (MMM dd yyyy HH:mm:ss or epoch milliseconds)
19
+ _S = "str"
20
+
21
+ CEF_KEYS: Dict[str, Tuple[str, int]] = {
22
+ "act": (_S, 63), "app": (_S, 31), "cat": (_S, 1023), "cnt": ("int", 0),
23
+ "cn1": ("long", 0), "cn2": ("long", 0), "cn3": ("long", 0),
24
+ "cn1Label": (_S, 1023), "cn2Label": (_S, 1023), "cn3Label": (_S, 1023),
25
+ "cfp1": ("float", 0), "cfp2": ("float", 0), "cfp3": ("float", 0), "cfp4": ("float", 0),
26
+ "cfp1Label": (_S, 1023), "cfp2Label": (_S, 1023), "cfp3Label": (_S, 1023), "cfp4Label": (_S, 1023),
27
+ "cs1": (_S, 4000), "cs2": (_S, 4000), "cs3": (_S, 4000),
28
+ "cs4": (_S, 4000), "cs5": (_S, 4000), "cs6": (_S, 4000),
29
+ "cs1Label": (_S, 1023), "cs2Label": (_S, 1023), "cs3Label": (_S, 1023),
30
+ "cs4Label": (_S, 1023), "cs5Label": (_S, 1023), "cs6Label": (_S, 1023),
31
+ "c6a1": ("ipv6", 0), "c6a2": ("ipv6", 0), "c6a3": ("ipv6", 0), "c6a4": ("ipv6", 0),
32
+ "c6a1Label": (_S, 1023), "c6a2Label": (_S, 1023), "c6a3Label": (_S, 1023), "c6a4Label": (_S, 1023),
33
+ "destinationDnsDomain": (_S, 255), "destinationServiceName": (_S, 1023),
34
+ "destinationTranslatedAddress": ("ip", 0), "destinationTranslatedPort": ("port", 0),
35
+ "deviceDirection": ("int", 0), "deviceDnsDomain": (_S, 255),
36
+ "deviceExternalId": (_S, 255), "deviceFacility": (_S, 1023),
37
+ "deviceInboundInterface": (_S, 128), "deviceOutboundInterface": (_S, 128),
38
+ "deviceNtDomain": (_S, 255), "devicePayloadId": (_S, 128),
39
+ "deviceProcessName": (_S, 1023), "deviceTranslatedAddress": ("ip", 0),
40
+ "deviceCustomDate1": ("ts", 0), "deviceCustomDate2": ("ts", 0),
41
+ "deviceCustomDate1Label": (_S, 1023), "deviceCustomDate2Label": (_S, 1023),
42
+ "dhost": (_S, 1023), "dmac": ("mac", 0), "dntdom": (_S, 255),
43
+ "dpid": ("int", 0), "dpriv": (_S, 1023), "dproc": (_S, 1023),
44
+ "dpt": ("port", 0), "dst": ("ip", 0), "dtz": (_S, 255),
45
+ "duid": (_S, 1023), "duser": (_S, 1023),
46
+ "dvc": ("ip", 0), "dvchost": (_S, 100), "dvcpid": ("int", 0),
47
+ "end": ("ts", 0), "externalId": (_S, 40),
48
+ "fileCreateTime": ("ts", 0), "fileHash": (_S, 255), "fileId": (_S, 1023),
49
+ "fileModificationTime": ("ts", 0), "filePath": (_S, 1023),
50
+ "filePermission": (_S, 1023), "fileType": (_S, 1023),
51
+ "flexDate1": ("ts", 0), "flexDate1Label": (_S, 128),
52
+ "flexString1": (_S, 1023), "flexString1Label": (_S, 128),
53
+ "flexString2": (_S, 1023), "flexString2Label": (_S, 128),
54
+ "fname": (_S, 1023), "fsize": ("int", 0),
55
+ "in": ("long", 0), "out": ("long", 0),
56
+ "msg": (_S, 1023), "outcome": (_S, 63), "proto": (_S, 31),
57
+ "reason": (_S, 1023), "request": (_S, 1023),
58
+ "requestClientApplication": (_S, 1023), "requestContext": (_S, 2048),
59
+ "requestCookies": (_S, 1023), "requestMethod": (_S, 1023),
60
+ "rt": ("ts", 0), "start": ("ts", 0),
61
+ "shost": (_S, 1023), "smac": ("mac", 0), "sntdom": (_S, 255),
62
+ "sourceDnsDomain": (_S, 255), "sourceServiceName": (_S, 1023),
63
+ "sourceTranslatedAddress": ("ip", 0), "sourceTranslatedPort": ("port", 0),
64
+ "spid": ("int", 0), "spriv": (_S, 1023), "sproc": (_S, 1023),
65
+ "spt": ("port", 0), "src": ("ip", 0),
66
+ "suid": (_S, 1023), "suser": (_S, 1023),
67
+ "type": ("int", 0),
68
+ "rawEvent": (_S, 4000),
69
+ }
70
+
71
+ MAC_RE = re.compile(r"^[0-9a-fA-F]{2}(:[0-9a-fA-F]{2}){5}$")
72
+ TS_RE = re.compile(r"^[A-Z][a-z]{2} \d{2} \d{4} \d{2}:\d{2}:\d{2}$")
73
+
74
+
75
+ class CEFValidationError(ValueError):
76
+ """Raised in strict mode when an extension value violates the dictionary."""
77
+
78
+
79
+ @dataclass
80
+ class Finding:
81
+ key: str
82
+ problem: str
83
+ fatal: bool
84
+
85
+ def __str__(self) -> str:
86
+ return f"{self.key}: {self.problem}"
87
+
88
+
89
+ def _is_ip(value: str, version: int | None = None) -> bool:
90
+ try:
91
+ addr = ipaddress.ip_address(value)
92
+ except ValueError:
93
+ return False
94
+ return version is None or addr.version == version
95
+
96
+
97
+ def _check(key: str, value: str) -> str | None:
98
+ kind, maxlen = CEF_KEYS[key]
99
+ if kind == _S:
100
+ if maxlen and len(value) > maxlen:
101
+ return f"string exceeds maximum length {maxlen} ({len(value)} characters)"
102
+ return None
103
+ if kind in ("int", "long"):
104
+ try:
105
+ int(value)
106
+ except ValueError:
107
+ return f"expected an integer, got {value!r}"
108
+ return None
109
+ if kind == "float":
110
+ try:
111
+ float(value)
112
+ except ValueError:
113
+ return f"expected a number, got {value!r}"
114
+ return None
115
+ if kind == "port":
116
+ try:
117
+ port = int(value)
118
+ except ValueError:
119
+ return f"expected a port number, got {value!r}"
120
+ if not 0 <= port <= 65535:
121
+ return f"port {port} outside 0-65535"
122
+ return None
123
+ if kind == "ip":
124
+ if not _is_ip(value):
125
+ return f"expected an IP address, got {value!r}"
126
+ return None
127
+ if kind == "ipv6":
128
+ if not _is_ip(value, 6):
129
+ return f"expected an IPv6 address, got {value!r}"
130
+ return None
131
+ if kind == "mac":
132
+ if not MAC_RE.match(value):
133
+ return f"expected a MAC address (aa:bb:cc:dd:ee:ff), got {value!r}"
134
+ return None
135
+ if kind == "ts":
136
+ if value.isdigit():
137
+ return None # epoch milliseconds
138
+ if TS_RE.match(value):
139
+ return None
140
+ return f"expected 'MMM dd yyyy HH:mm:ss' or epoch milliseconds, got {value!r}"
141
+ return None
142
+
143
+
144
+ def validate_extensions(extensions: Dict[str, str]) -> List[Finding]:
145
+ """Validate raw (pre-escaping) extension values against the dictionary.
146
+
147
+ Returns findings; type/length violations are fatal, unknown keys are
148
+ advisory (the CEF standard allows custom keys).
149
+ """
150
+
151
+ findings: List[Finding] = []
152
+ for key, value in extensions.items():
153
+ if key not in CEF_KEYS:
154
+ findings.append(
155
+ Finding(key, "not in the ArcSight dictionary (custom keys are allowed)", fatal=False)
156
+ )
157
+ continue
158
+ problem = _check(key, str(value))
159
+ if problem:
160
+ findings.append(Finding(key, problem, fatal=True))
161
+ return findings
162
+
163
+
164
+ __all__ = ["CEF_KEYS", "CEFValidationError", "Finding", "validate_extensions"]
@@ -135,3 +135,57 @@ def test_garbage_still_falls_back_to_unknown():
135
135
  ev2 = parse_syslog("<30>still garbage but with a pri")
136
136
  assert ev2.source_hint == "unknown"
137
137
  assert ev2.pri == 30
138
+
139
+
140
+ # --- fixes from code review of the adaptive parsing PR -------------------------
141
+
142
+ def test_kv_without_id_falls_back_to_nonempty_event_class():
143
+ line = '<30>device="SFW" date=2020-05-18 time=14:38:48 device_name="XG230" reason="x" a=1'
144
+ cef = convert_line(line)
145
+ header_fields = fields(cef)
146
+ assert header_fields[4] != "" # eventClassId never empty
147
+
148
+
149
+ def test_kv_numeric_timezone_applied():
150
+ line = 'date=2020-04-23 time=12:32:48 devname="fw1" logid="0100000001" tz="-0500" x=1'
151
+ ev = parse_syslog(line)
152
+ assert ev.ts.utcoffset().total_seconds() == -5 * 3600
153
+
154
+
155
+ def test_cisco_milliseconds_preserved():
156
+ line = "000123: Feb 8 04:00:47.272: %SEC-6-IPACCESSLOGP: list 100 denied tcp"
157
+ ev = parse_syslog(line)
158
+ assert ev.ts.microsecond == 272000
159
+
160
+
161
+ def test_adaptive_mon_day_year_rollover():
162
+ import re
163
+ from datetime import datetime, timezone
164
+
165
+ from syslogcef.adaptive import TIMESTAMP_LIB
166
+
167
+ src, conv = next((s, c) for n, s, c in TIMESTAMP_LIB if n == "mon_day")
168
+ match = re.compile(src).match("Dec 31 23:59:59")
169
+ ts = conv(match, datetime(2026, 1, 2, tzinfo=timezone.utc))
170
+ assert ts.year == 2025
171
+
172
+
173
+ def test_adaptive_cached_host_revalidated():
174
+ clear_cache()
175
+ good = "2021/05/01 10:00:00 fw-edge-01 session established"
176
+ bad = "2021/05/02 11:00:00 ERROR link down"
177
+ ev1 = parse_syslog(good)
178
+ assert ev1.host == "fw-edge-01"
179
+ ev2 = parse_syslog(bad)
180
+ assert ev2.source_hint == "adaptive"
181
+ assert ev2.host is None
182
+ assert ev2.msg.startswith("ERROR")
183
+
184
+
185
+ def test_adaptive_cached_empty_message_preserved():
186
+ clear_cache()
187
+ ev1 = parse_syslog("2021/05/01 10:00:00 fw-edge-01 something happened")
188
+ assert ev1.msg == "something happened"
189
+ ev2 = parse_syslog("2021/05/02 11:00:00 fw-edge-02")
190
+ assert ev2.host == "fw-edge-02"
191
+ assert ev2.msg == ""
@@ -0,0 +1,180 @@
1
+ from __future__ import annotations
2
+
3
+ import socket
4
+ import threading
5
+
6
+ import pytest
7
+
8
+ from syslogcef.net import listen_lines, parse_endpoint
9
+
10
+
11
+ def test_parse_endpoint_forms():
12
+ assert parse_endpoint("udp:514") == ("udp", "0.0.0.0", 514)
13
+ assert parse_endpoint("tcp:5514") == ("tcp", "0.0.0.0", 5514)
14
+ assert parse_endpoint("udp:10.0.0.5:514") == ("udp", "10.0.0.5", 514)
15
+ for bad in ("http:80", "udp:notaport", "udp:99999", "514"):
16
+ with pytest.raises(ValueError):
17
+ parse_endpoint(bad)
18
+
19
+
20
+ def _run_when_ready(send_fn):
21
+ """Return (ready_callback, thread) that fires send_fn(port) once bound."""
22
+ started = threading.Event()
23
+ bound = {}
24
+
25
+ def ready(port):
26
+ bound["port"] = port
27
+ started.set()
28
+
29
+ def runner():
30
+ started.wait(5)
31
+ send_fn(bound["port"])
32
+
33
+ thread = threading.Thread(target=runner, daemon=True)
34
+ thread.start()
35
+ return ready, thread
36
+
37
+
38
+ def test_udp_listener_receives_lines():
39
+ def send(port):
40
+ sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
41
+ sock.sendto(b"<166>Jan 1 12:34:56 fw1 app: one", ("127.0.0.1", port))
42
+ sock.sendto(b"line two\nline three\n", ("127.0.0.1", port))
43
+ sock.close()
44
+
45
+ ready, thread = _run_when_ready(send)
46
+ lines = list(
47
+ listen_lines("udp", "127.0.0.1", 0, max_messages=3, ready_callback=ready)
48
+ )
49
+ thread.join(5)
50
+ assert lines == ["<166>Jan 1 12:34:56 fw1 app: one", "line two", "line three"]
51
+
52
+
53
+ def test_tcp_listener_receives_lines_across_connections():
54
+ def send(port):
55
+ c1 = socket.create_connection(("127.0.0.1", port), timeout=5)
56
+ c1.sendall(b"alpha\nbeta\n")
57
+ c1.close()
58
+ c2 = socket.create_connection(("127.0.0.1", port), timeout=5)
59
+ c2.sendall(b"gamma without newline")
60
+ c2.close()
61
+
62
+ ready, thread = _run_when_ready(send)
63
+ lines = list(
64
+ listen_lines("tcp", "127.0.0.1", 0, max_messages=3, ready_callback=ready)
65
+ )
66
+ thread.join(5)
67
+ assert sorted(lines) == ["alpha", "beta", "gamma without newline"]
68
+
69
+
70
+ def test_listener_lines_convert_end_to_end():
71
+ from syslogcef import convert_line
72
+
73
+ def send(port):
74
+ sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
75
+ sock.sendto(
76
+ b"<166>Jan 1 12:34:56 fw1 %ASA-6-302013: Built inbound src=1.2.3.4",
77
+ ("127.0.0.1", port),
78
+ )
79
+ sock.close()
80
+
81
+ ready, thread = _run_when_ready(send)
82
+ lines = list(
83
+ listen_lines("udp", "127.0.0.1", 0, max_messages=1, ready_callback=ready)
84
+ )
85
+ thread.join(5)
86
+ cef = convert_line(lines[0])
87
+ assert cef.startswith("CEF:0|Cisco|ASA|")
88
+
89
+
90
+ # --- forwarding ---------------------------------------------------------------
91
+
92
+ def test_create_sender_parses_targets():
93
+ from syslogcef.net import SyslogSender, create_sender
94
+
95
+ s = create_sender("udp://siem.example.com:514")
96
+ assert isinstance(s, SyslogSender) and s.proto == "udp" and s.port == 514
97
+ s.close()
98
+ for bad in ("udp://nohost", "ftp://x:1", "kafka://broker:9092"):
99
+ with pytest.raises(ValueError):
100
+ create_sender(bad)
101
+
102
+
103
+ def test_udp_sender_delivers_to_listener():
104
+ from syslogcef.net import create_sender
105
+
106
+ rx = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
107
+ rx.bind(("127.0.0.1", 0))
108
+ rx.settimeout(5)
109
+ port = rx.getsockname()[1]
110
+ sender = create_sender(f"udp://127.0.0.1:{port}")
111
+ sender.send("CEF:0|Cisco|ASA|auto|x|y|2|src=1.2.3.4")
112
+ data, _ = rx.recvfrom(65535)
113
+ sender.close()
114
+ rx.close()
115
+ assert data.decode() == "CEF:0|Cisco|ASA|auto|x|y|2|src=1.2.3.4"
116
+
117
+
118
+ def test_tcp_sender_delivers_newline_delimited():
119
+ from syslogcef.net import create_sender
120
+
121
+ srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
122
+ srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
123
+ srv.bind(("127.0.0.1", 0))
124
+ srv.listen(1)
125
+ srv.settimeout(5)
126
+ port = srv.getsockname()[1]
127
+
128
+ sender = create_sender(f"tcp://127.0.0.1:{port}")
129
+ sender.send("record one")
130
+ conn, _ = srv.accept()
131
+ conn.settimeout(5)
132
+ sender.send("record two")
133
+ sender.close()
134
+ received = b""
135
+ while b"two\n" not in received:
136
+ received += conn.recv(1024)
137
+ conn.close()
138
+ srv.close()
139
+ assert received == b"record one\nrecord two\n"
140
+
141
+
142
+ def test_rate_limiter_paces_sends():
143
+ from syslogcef.net import RateLimiter
144
+
145
+ sleeps = []
146
+ clock_value = [0.0]
147
+
148
+ def clock():
149
+ return clock_value[0]
150
+
151
+ def sleep(seconds):
152
+ sleeps.append(seconds)
153
+ clock_value[0] += seconds
154
+
155
+ limiter = RateLimiter(10, clock=clock, sleep=sleep) # 0.1s interval
156
+ for _ in range(3):
157
+ limiter.wait()
158
+ assert sum(sleeps) == pytest.approx(0.2, abs=0.01)
159
+
160
+
161
+ def test_kafka_sender_uses_injected_producer():
162
+ from syslogcef.net import KafkaSender
163
+
164
+ class FakeProducer:
165
+ def __init__(self):
166
+ self.sent = []
167
+ self.flushed = False
168
+
169
+ def send(self, topic, payload):
170
+ self.sent.append((topic, payload))
171
+
172
+ def flush(self):
173
+ self.flushed = True
174
+
175
+ producer = FakeProducer()
176
+ sender = KafkaSender("broker:9092", "cef-events", producer=producer)
177
+ sender.send("CEF:0|a|b|1|c|d|5|")
178
+ sender.close()
179
+ assert producer.sent == [("cef-events", b"CEF:0|a|b|1|c|d|5|")]
180
+ assert producer.flushed
@@ -0,0 +1,66 @@
1
+ from __future__ import annotations
2
+
3
+ import pytest
4
+
5
+ from syslogcef import convert_line, normalize_event, parse_syslog, to_cef
6
+ from syslogcef.validation import CEFValidationError, validate_extensions
7
+
8
+
9
+ def test_valid_extensions_produce_no_fatal_findings():
10
+ findings = validate_extensions(
11
+ {"src": "10.0.0.1", "dst": "2001:db8::1", "spt": "443", "msg": "hello", "smac": "00:0d:60:af:1b:61"}
12
+ )
13
+ assert [f for f in findings if f.fatal] == []
14
+
15
+
16
+ def test_type_violations_are_fatal():
17
+ findings = validate_extensions({"src": "not-an-ip", "spt": "99999", "fsize": "big"})
18
+ assert sorted(f.key for f in findings if f.fatal) == ["fsize", "spt", "src"]
19
+
20
+
21
+ def test_unknown_keys_are_advisory_only():
22
+ findings = validate_extensions({"myCustomKey": "anything"})
23
+ assert len(findings) == 1
24
+ assert not findings[0].fatal
25
+
26
+
27
+ def test_length_violation_detected():
28
+ findings = validate_extensions({"outcome": "x" * 100}) # max 63
29
+ assert findings and findings[0].fatal
30
+
31
+
32
+ def test_timestamp_formats():
33
+ ok = validate_extensions({"rt": "Jan 17 2020 08:52:10", "start": "1587231168000"})
34
+ assert [f for f in ok if f.fatal] == []
35
+ bad = validate_extensions({"rt": "yesterday at noon"})
36
+ assert bad and bad[0].fatal
37
+
38
+
39
+ def test_strict_mode_raises_through_pipeline():
40
+ # The ASA mapping maps src= from the message; a bogus IP violates the
41
+ # dictionary in strict mode.
42
+ line = "<166>Jan 1 12:34:56 fw1 asa[1]: %ASA-6-302013: Built inbound src=notanip dst=10.0.0.2"
43
+ with pytest.raises(CEFValidationError):
44
+ convert_line(line, validate=True, strict=True)
45
+ # Non-strict validation still renders the record.
46
+ assert convert_line(line, validate=True).startswith("CEF:0|")
47
+
48
+
49
+ def test_strict_mode_passes_clean_lines():
50
+ line = "<166>Jan 1 12:34:56 fw1 asa[1]: %ASA-6-302013: Built inbound src=10.0.0.1 dst=10.0.0.2"
51
+ cef = convert_line(line, validate=True, strict=True)
52
+ assert "src=10.0.0.1" in cef
53
+
54
+
55
+ def test_cli_strict_exits_nonzero(tmp_path):
56
+ import subprocess
57
+ import sys
58
+
59
+ bad = tmp_path / "bad.log"
60
+ bad.write_text("<166>Jan 1 12:34:56 fw1 asa[1]: %ASA-6-302013: x src=notanip\n")
61
+ result = subprocess.run(
62
+ [sys.executable, "-m", "syslogcef", str(bad), "--strict"],
63
+ capture_output=True,
64
+ )
65
+ assert result.returncode == 1
66
+ assert b"strict validation failed" in result.stderr
@@ -1,3 +0,0 @@
1
-
2
- [test]
3
- pytest>=7
File without changes
File without changes
File without changes
File without changes
File without changes