syslog2cef 0.1.1__tar.gz → 0.1.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. {syslog2cef-0.1.1/syslog2cef.egg-info → syslog2cef-0.1.3}/PKG-INFO +7 -1
  2. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/README.md +4 -0
  3. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/pyproject.toml +4 -1
  4. {syslog2cef-0.1.1 → syslog2cef-0.1.3/syslog2cef.egg-info}/PKG-INFO +7 -1
  5. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslog2cef.egg-info/SOURCES.txt +15 -1
  6. syslog2cef-0.1.3/syslog2cef.egg-info/requires.txt +6 -0
  7. syslog2cef-0.1.3/syslogcef/adaptive.py +214 -0
  8. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslogcef/api.py +24 -5
  9. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslogcef/cef.py +31 -7
  10. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslogcef/cli.py +55 -16
  11. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslogcef/mappings/__init__.py +6 -0
  12. syslog2cef-0.1.3/syslogcef/mappings/fortinet.json +19 -0
  13. syslog2cef-0.1.3/syslogcef/mappings/sophos.json +18 -0
  14. syslog2cef-0.1.3/syslogcef/net.py +270 -0
  15. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslogcef/normalizer.py +29 -5
  16. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslogcef/parsers.py +205 -9
  17. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslogcef/utils.py +17 -2
  18. syslog2cef-0.1.3/syslogcef/validation.py +164 -0
  19. syslog2cef-0.1.3/tests/test_adaptive.py +191 -0
  20. syslog2cef-0.1.3/tests/test_net.py +180 -0
  21. syslog2cef-0.1.3/tests/test_real_world.py +93 -0
  22. syslog2cef-0.1.3/tests/test_validation.py +66 -0
  23. syslog2cef-0.1.1/syslog2cef.egg-info/requires.txt +0 -3
  24. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/LICENSE +0 -0
  25. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/setup.cfg +0 -0
  26. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslog2cef.egg-info/dependency_links.txt +0 -0
  27. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslog2cef.egg-info/entry_points.txt +0 -0
  28. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslog2cef.egg-info/top_level.txt +0 -0
  29. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslogcef/__init__.py +0 -0
  30. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslogcef/__main__.py +0 -0
  31. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslogcef/mappings/cisco_asa.json +0 -0
  32. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslogcef/mappings/cisco_ios.json +0 -0
  33. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslogcef/mappings/f5.json +0 -0
  34. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslogcef/mappings/linux.json +0 -0
  35. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslogcef/mappings/vmware.json +0 -0
  36. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/tests/test_cef.py +0 -0
  37. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/tests/test_cli.py +0 -0
  38. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/tests/test_normalizer.py +0 -0
  39. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/tests/test_parsers.py +0 -0
  40. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/tests/test_parsers_extra.py +0 -0
  41. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/tests/test_pipeline.py +0 -0
  42. {syslog2cef-0.1.1 → syslog2cef-0.1.3}/tests/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: syslog2cef
3
- Version: 0.1.1
3
+ Version: 0.1.3
4
4
  Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
5
  Author-email: Tamir Suliman <allamiro@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/allamiro/syslogcef
@@ -15,6 +15,8 @@ Description-Content-Type: text/markdown
15
15
  License-File: LICENSE
16
16
  Provides-Extra: test
17
17
  Requires-Dist: pytest>=7; extra == "test"
18
+ Provides-Extra: kafka
19
+ Requires-Dist: kafka-python>=2.0; extra == "kafka"
18
20
  Dynamic: license-file
19
21
 
20
22
  <p align="center">
@@ -99,6 +101,10 @@ signatures. The public key is committed at
99
101
 
100
102
  ## Command Line Usage
101
103
 
104
+ <p align="center">
105
+ <img src="https://raw.githubusercontent.com/allamiro/syslogcef/main/docs/demo.svg" alt="Animated demo of syslogcef converting different formats" width="820">
106
+ </p>
107
+
102
108
  ```bash
103
109
  # Read from stdin, write CEF to stdout
104
110
  syslogcef < /var/log/syslog
@@ -80,6 +80,10 @@ signatures. The public key is committed at
80
80
 
81
81
  ## Command Line Usage
82
82
 
83
+ <p align="center">
84
+ <img src="https://raw.githubusercontent.com/allamiro/syslogcef/main/docs/demo.svg" alt="Animated demo of syslogcef converting different formats" width="820">
85
+ </p>
86
+
83
87
  ```bash
84
88
  # Read from stdin, write CEF to stdout
85
89
  syslogcef < /var/log/syslog
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "syslog2cef"
7
- version = "0.1.1"
7
+ version = "0.1.3"
8
8
  description = "Convert RFC3164/RFC5424 syslog lines into ArcSight CEF"
9
9
  authors = [{name = "Tamir Suliman", email = "allamiro@gmail.com"}]
10
10
  readme = "README.md"
@@ -26,6 +26,9 @@ Issues = "https://github.com/allamiro/syslogcef/issues"
26
26
  test = [
27
27
  "pytest>=7",
28
28
  ]
29
+ kafka = [
30
+ "kafka-python>=2.0",
31
+ ]
29
32
 
30
33
  [project.scripts]
31
34
  syslogcef = "syslogcef.cli:main"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: syslog2cef
3
- Version: 0.1.1
3
+ Version: 0.1.3
4
4
  Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
5
  Author-email: Tamir Suliman <allamiro@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/allamiro/syslogcef
@@ -15,6 +15,8 @@ Description-Content-Type: text/markdown
15
15
  License-File: LICENSE
16
16
  Provides-Extra: test
17
17
  Requires-Dist: pytest>=7; extra == "test"
18
+ Provides-Extra: kafka
19
+ Requires-Dist: kafka-python>=2.0; extra == "kafka"
18
20
  Dynamic: license-file
19
21
 
20
22
  <p align="center">
@@ -99,6 +101,10 @@ signatures. The public key is committed at
99
101
 
100
102
  ## Command Line Usage
101
103
 
104
+ <p align="center">
105
+ <img src="https://raw.githubusercontent.com/allamiro/syslogcef/main/docs/demo.svg" alt="Animated demo of syslogcef converting different formats" width="820">
106
+ </p>
107
+
102
108
  ```bash
103
109
  # Read from stdin, write CEF to stdout
104
110
  syslogcef < /var/log/syslog
@@ -3,17 +3,22 @@ README.md
3
3
  pyproject.toml
4
4
  ./syslogcef/__init__.py
5
5
  ./syslogcef/__main__.py
6
+ ./syslogcef/adaptive.py
6
7
  ./syslogcef/api.py
7
8
  ./syslogcef/cef.py
8
9
  ./syslogcef/cli.py
10
+ ./syslogcef/net.py
9
11
  ./syslogcef/normalizer.py
10
12
  ./syslogcef/parsers.py
11
13
  ./syslogcef/utils.py
14
+ ./syslogcef/validation.py
12
15
  ./syslogcef/mappings/__init__.py
13
16
  ./syslogcef/mappings/cisco_asa.json
14
17
  ./syslogcef/mappings/cisco_ios.json
15
18
  ./syslogcef/mappings/f5.json
19
+ ./syslogcef/mappings/fortinet.json
16
20
  ./syslogcef/mappings/linux.json
21
+ ./syslogcef/mappings/sophos.json
17
22
  ./syslogcef/mappings/vmware.json
18
23
  syslog2cef.egg-info/PKG-INFO
19
24
  syslog2cef.egg-info/SOURCES.txt
@@ -23,22 +28,31 @@ syslog2cef.egg-info/requires.txt
23
28
  syslog2cef.egg-info/top_level.txt
24
29
  syslogcef/__init__.py
25
30
  syslogcef/__main__.py
31
+ syslogcef/adaptive.py
26
32
  syslogcef/api.py
27
33
  syslogcef/cef.py
28
34
  syslogcef/cli.py
35
+ syslogcef/net.py
29
36
  syslogcef/normalizer.py
30
37
  syslogcef/parsers.py
31
38
  syslogcef/utils.py
39
+ syslogcef/validation.py
32
40
  syslogcef/mappings/__init__.py
33
41
  syslogcef/mappings/cisco_asa.json
34
42
  syslogcef/mappings/cisco_ios.json
35
43
  syslogcef/mappings/f5.json
44
+ syslogcef/mappings/fortinet.json
36
45
  syslogcef/mappings/linux.json
46
+ syslogcef/mappings/sophos.json
37
47
  syslogcef/mappings/vmware.json
48
+ tests/test_adaptive.py
38
49
  tests/test_cef.py
39
50
  tests/test_cli.py
51
+ tests/test_net.py
40
52
  tests/test_normalizer.py
41
53
  tests/test_parsers.py
42
54
  tests/test_parsers_extra.py
43
55
  tests/test_pipeline.py
44
- tests/test_utils.py
56
+ tests/test_real_world.py
57
+ tests/test_utils.py
58
+ tests/test_validation.py
@@ -0,0 +1,6 @@
1
+
2
+ [kafka]
3
+ kafka-python>=2.0
4
+
5
+ [test]
6
+ pytest>=7
@@ -0,0 +1,214 @@
1
+ """Adaptive pattern detection for lines no known parser matches.
2
+
3
+ When every dedicated parser fails, this module analyzes the line: it
4
+ locates a timestamp using a library of datetime shapes, identifies a
5
+ plausible host token, and synthesizes a compiled regex for that line
6
+ layout. The synthesized pattern is cached by the line's shape signature,
7
+ so subsequent lines with the same layout parse with a single regex match
8
+ instead of a fresh analysis. Events parsed this way report
9
+ ``source_hint="adaptive"``.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import re
15
+ from datetime import datetime, timezone
16
+ from typing import Callable, Dict, Optional, Pattern, Tuple
17
+
18
+ from .utils import convert_pri, month_abbr_to_int, parse_iso8601, sanitize_message
19
+
20
+ PRI_RE = re.compile(r"^<(\d{1,3})>")
21
+
22
+ # Each entry: (name, regex source, converter(match, now) -> datetime)
23
+ def _iso(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
24
+ return parse_iso8601(match.group(0).replace(" ", "T", 1) if "T" not in match.group(0) else match.group(0))
25
+
26
+
27
+ def _epoch(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
28
+ return parse_iso8601(match.group(0))
29
+
30
+
31
+ def _mon_day(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
32
+ month = month_abbr_to_int(match.group("month"))
33
+ day = int(match.group("day"))
34
+ if match.groupdict().get("year"):
35
+ hour, minute, second = map(int, match.group("hms").split(":"))
36
+ return datetime(int(match.group("year")), month, day, hour, minute, second, tzinfo=timezone.utc)
37
+ # No year: use the same rollover inference as the dedicated parsers so a
38
+ # December event parsed in January lands in the previous year.
39
+ from .parsers import _infer_timestamp
40
+
41
+ return _infer_timestamp(month, day, match.group("hms"), now=now)
42
+
43
+
44
+ def _ymd_slash(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
45
+ y, m, d = int(match.group("y")), int(match.group("m")), int(match.group("d"))
46
+ hour, minute, second = map(int, match.group("hms").split(":"))
47
+ return datetime(y, m, d, hour, minute, second, tzinfo=timezone.utc)
48
+
49
+
50
+ def _dmy_dash(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
51
+ d = int(match.group("d"))
52
+ month = month_abbr_to_int(match.group("month").title())
53
+ y = int(match.group("y"))
54
+ hour, minute, second = map(int, match.group("hms").split(":"))
55
+ return datetime(y, month, d, hour, minute, second, tzinfo=timezone.utc)
56
+
57
+
58
+ TIMESTAMP_LIB: Tuple[Tuple[str, str, Callable], ...] = (
59
+ ("iso", r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?", _iso),
60
+ ("mon_day_year", r"(?P<month>[A-Z][a-z]{2})\s+(?P<day>\d{1,2})\s+(?P<year>\d{4})\s+(?P<hms>\d{2}:\d{2}:\d{2})(?:\.\d+)?", _mon_day),
61
+ ("mon_day", r"(?P<month>[A-Z][a-z]{2})\s+(?P<day>\d{1,2})\s+(?P<hms>\d{2}:\d{2}:\d{2})(?:\.\d+)?", _mon_day),
62
+ ("ymd_slash", r"(?P<y>\d{4})/(?P<m>\d{2})/(?P<d>\d{2})[ T](?P<hms>\d{2}:\d{2}:\d{2})", _ymd_slash),
63
+ ("dmy_dash", r"(?P<d>\d{1,2})-(?P<month>[A-Za-z]{3})-(?P<y>\d{4})\s+(?P<hms>\d{2}:\d{2}:\d{2})", _dmy_dash),
64
+ ("epoch", r"\b1\d{9}(?:\d{6})?\b", _epoch),
65
+ )
66
+
67
+ HOST_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
68
+ _SKIP_HOST_TOKENS = {"error", "warning", "info", "debug", "notice", "kernel"}
69
+
70
+
71
+ def _valid_host(candidate: Optional[str]) -> bool:
72
+ return bool(
73
+ candidate
74
+ and "=" not in candidate
75
+ and HOST_RE.match(candidate)
76
+ and candidate.lower() not in _SKIP_HOST_TOKENS
77
+ and not candidate.isdigit()
78
+ )
79
+
80
+ _CACHE: Dict[str, Tuple[Pattern[str], str, Callable]] = {}
81
+ _CACHE_MAX = 256
82
+
83
+
84
+ def _signature(line: str) -> str:
85
+ """Reduce a line to a layout signature: token classes of the prefix.
86
+
87
+ Only the first few tokens participate — the layout (timestamp + host)
88
+ lives there, while later tokens are free-form message words that must
89
+ not fragment the cache.
90
+ """
91
+ tokens = line[:96].split()[:4]
92
+ classes = []
93
+ for token in tokens:
94
+ cls = re.sub(r"[A-Za-z]+", "A", token)
95
+ cls = re.sub(r"\d+", "9", cls)
96
+ classes.append(cls)
97
+ return " ".join(classes)
98
+
99
+
100
+ def clear_cache() -> None:
101
+ _CACHE.clear()
102
+
103
+
104
+ def cache_size() -> int:
105
+ return len(_CACHE)
106
+
107
+
108
+ def _build_event(line, pri, ts, host, msg):
109
+ from .parsers import ParsedEvent
110
+
111
+ facility, severity = convert_pri(pri)
112
+ return ParsedEvent(
113
+ pri=pri,
114
+ facility=facility,
115
+ severity=severity,
116
+ ts=ts,
117
+ ts_orig="",
118
+ host=host,
119
+ app=None,
120
+ pid=None,
121
+ msgid=None,
122
+ sd={},
123
+ msg=sanitize_message(msg),
124
+ raw=line,
125
+ source_hint="adaptive",
126
+ )
127
+
128
+
129
+ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
130
+ """Parse an unknown-format line, learning its layout on first sight."""
131
+
132
+ raw = line
133
+ pri = None
134
+ pri_match = PRI_RE.match(line)
135
+ if pri_match:
136
+ pri = int(pri_match.group(1))
137
+ line = line[pri_match.end():]
138
+
139
+ sig = _signature(line)
140
+ cached = _CACHE.get(sig)
141
+ if cached is not None:
142
+ pattern, _name, conv = cached
143
+ match = pattern.match(line)
144
+ if match is not None:
145
+ try:
146
+ ts = conv(pattern_ts_match(pattern, match), now)
147
+ except (ValueError, KeyError):
148
+ ts = None
149
+ gd = match.groupdict()
150
+ host = gd.get("adaptive_host")
151
+ msg = gd.get("adaptive_msg")
152
+ msg = line if msg is None else msg
153
+ # The shape signature reduces all words to the same class, so a
154
+ # cached host slot may capture a non-host token; revalidate.
155
+ if host is not None and not _valid_host(host.rstrip(":,")):
156
+ msg = f"{host} {msg}".strip()
157
+ host = None
158
+ return _build_event(raw, pri, ts, host, msg)
159
+
160
+ # Analysis pass: find the earliest timestamp in the line head.
161
+ best = None
162
+ for name, src, conv in TIMESTAMP_LIB:
163
+ match = re.compile(src).search(line[:96])
164
+ if match and (best is None or match.start() < best[0].start()):
165
+ best = (match, name, src, conv)
166
+ if best is None:
167
+ return None
168
+ ts_match, name, src, conv = best
169
+ try:
170
+ ts = conv(ts_match, now)
171
+ except (ValueError, KeyError):
172
+ return None
173
+
174
+ # Host candidate: the first plausible token after the timestamp that is
175
+ # not a key=value pair or severity word.
176
+ rest = line[ts_match.end():].lstrip(" :,\t")
177
+ host = None
178
+ msg = rest
179
+ tokens = rest.split(None, 1)
180
+ if tokens:
181
+ candidate = tokens[0].rstrip(":,")
182
+ if _valid_host(candidate):
183
+ host = candidate
184
+ msg = tokens[1] if len(tokens) > 1 else ""
185
+
186
+ # Synthesize and cache a pattern for this layout.
187
+ prefix = line[: ts_match.start()]
188
+ parts = [re.escape(prefix), f"(?P<adaptive_ts>{src})", r"[\s:,]*"]
189
+ if host is not None:
190
+ parts.append(r"(?P<adaptive_host>\S+?):?(?:\s+|$)")
191
+ parts.append(r"(?P<adaptive_msg>.*)$")
192
+ try:
193
+ pattern = re.compile("".join(parts))
194
+ except re.error:
195
+ pattern = None
196
+ if pattern is not None and pattern.match(line):
197
+ if len(_CACHE) >= _CACHE_MAX:
198
+ _CACHE.pop(next(iter(_CACHE)))
199
+ _CACHE[sig] = (pattern, name, conv)
200
+
201
+ return _build_event(raw, pri, ts, host, msg)
202
+
203
+
204
+ def pattern_ts_match(pattern: Pattern[str], match: "re.Match[str]"):
205
+ """Re-match the timestamp portion so converters see their own groups."""
206
+ ts_text = match.group("adaptive_ts")
207
+ for _name, src, _conv in TIMESTAMP_LIB:
208
+ m = re.compile(src).fullmatch(ts_text)
209
+ if m:
210
+ return m
211
+ raise ValueError(f"cached timestamp no longer matches: {ts_text!r}")
212
+
213
+
214
+ __all__ = ["adaptive_parse", "cache_size", "clear_cache"]
@@ -9,7 +9,7 @@ from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional
9
9
 
10
10
  from .parsers import ParsedEvent, autodetect_and_parse
11
11
  from .normalizer import NormalizedEvent, normalize
12
- from .mappings import CISCO_ASA, CISCO_IOS, F5, LINUX, VMWARE
12
+ from .mappings import CISCO_ASA, CISCO_IOS, F5, FORTINET, LINUX, SOPHOS, VMWARE
13
13
  from .cef import CEFEvent, build_cef
14
14
 
15
15
  logger = logging.getLogger(__name__)
@@ -74,14 +74,23 @@ def _load_mapping(mapping: Mapping[str, Any] | Path | str | None) -> Mapping[str
74
74
  def to_cef(
75
75
  event: NormalizedEvent,
76
76
  mapping: Mapping[str, Any] | Path | str | None = None,
77
+ *,
78
+ validate: bool = False,
79
+ strict: bool = False,
77
80
  ) -> str:
78
- """Convert a normalized event into a CEF string."""
81
+ """Convert a normalized event into a CEF string.
82
+
83
+ With ``validate=True``, extension values are checked against the
84
+ ArcSight dictionary and violations logged as warnings; ``strict=True``
85
+ additionally raises :class:`syslogcef.validation.CEFValidationError`
86
+ on type or length violations.
87
+ """
79
88
 
80
89
  if mapping is None:
81
90
  mapping_data = _guess_mapping(event)
82
91
  else:
83
92
  mapping_data = _load_mapping(mapping)
84
- cef_event = build_cef(event, mapping_data)
93
+ cef_event = build_cef(event, mapping_data, validate=validate, strict=strict)
85
94
  return cef_event.render()
86
95
 
87
96
 
@@ -91,27 +100,37 @@ def convert_line(
91
100
  mode: Optional[str] = None,
92
101
  mapping: Mapping[str, Any] | Path | str | None = None,
93
102
  now: Optional[datetime] = None,
103
+ validate: bool = False,
104
+ strict: bool = False,
94
105
  ) -> str:
95
106
  """Full pipeline that parses, normalizes and converts a syslog line."""
96
107
 
97
108
  parsed = parse_syslog(line, mode=mode, now=now)
98
109
  normalized = normalize_event(parsed)
99
110
 
100
- return to_cef(normalized, mapping)
111
+ return to_cef(normalized, mapping, validate=validate, strict=strict)
101
112
 
102
113
 
103
114
  def _guess_mapping(event: NormalizedEvent) -> Mapping[str, Any]:
104
115
  msg_upper = event.msg.upper()
105
116
  app_upper = (event.app or "").upper()
117
+ event_code = event.kv.get("event_code", "")
106
118
 
107
- if "%ASA-" in msg_upper or "ASA" in app_upper:
119
+ if "%ASA-" in msg_upper or "ASA" in app_upper or event_code.startswith(("ASA-", "FTD-")):
108
120
  return CISCO_ASA
109
121
  if "IOS" in app_upper or "%IOS-" in msg_upper:
110
122
  return CISCO_IOS
111
123
  if "BIG-IP" in msg_upper or "F5" in app_upper:
112
124
  return F5
125
+ if "logid" in event.kv:
126
+ return FORTINET
127
+ if "log_id" in event.kv:
128
+ return SOPHOS
113
129
  if "VMWARE" in msg_upper or "ESXI" in msg_upper or "VMWARE" in app_upper:
114
130
  return VMWARE
131
+ if event_code:
132
+ # Other %FAC-SEV-MNEMONIC codes are Cisco IOS style.
133
+ return CISCO_IOS
115
134
  return LINUX
116
135
 
117
136
 
@@ -56,7 +56,12 @@ class MappingResolver:
56
56
  header = {}
57
57
  for key in ["deviceVendor", "deviceProduct", "deviceVersion", "eventClassId", "name"]:
58
58
  template = self.mapping.get(key, DEFAULT_MAPPING[key])
59
- header[key] = cef_escape_header(self._format(template, fields))
59
+ value = self._format(template, fields)
60
+ if not value:
61
+ # A template referencing missing fields must not leave an
62
+ # empty CEF header slot; fall back to the default template.
63
+ value = self._format(DEFAULT_MAPPING[key], fields)
64
+ header[key] = cef_escape_header(value)
60
65
  severity = self._resolve_severity(fields)
61
66
  header["severity"] = severity
62
67
  return header
@@ -71,15 +76,18 @@ class MappingResolver:
71
76
  return "3"
72
77
  return severity_map.get(str(value), str(value))
73
78
 
74
- def resolve_extensions(self, fields: Mapping[str, Any]) -> Dict[str, str]:
75
- extensions = {}
79
+ def resolve_raw_extensions(self, fields: Mapping[str, Any]) -> Dict[str, str]:
80
+ raw = {}
76
81
  merged = DEFAULT_MAPPING["extensions"].copy()
77
82
  merged.update(self.mapping.get("extensions", {}))
78
83
  for key, template in merged.items():
79
84
  value = self._format(template, fields)
80
85
  if value:
81
- extensions[key] = cef_escape(str(value))
82
- return extensions
86
+ raw[key] = str(value)
87
+ return raw
88
+
89
+ def resolve_extensions(self, fields: Mapping[str, Any]) -> Dict[str, str]:
90
+ return {k: cef_escape(v) for k, v in self.resolve_raw_extensions(fields).items()}
83
91
 
84
92
  def _format(self, template: str, fields: Mapping[str, Any]) -> str:
85
93
  try:
@@ -92,11 +100,27 @@ class MappingResolver:
92
100
  return ""
93
101
 
94
102
 
95
- def build_cef(event: NormalizedEvent, mapping: Mapping[str, Any]) -> CEFEvent:
103
+ def build_cef(
104
+ event: NormalizedEvent,
105
+ mapping: Mapping[str, Any],
106
+ *,
107
+ validate: bool = False,
108
+ strict: bool = False,
109
+ ) -> CEFEvent:
96
110
  resolver = MappingResolver(mapping)
97
111
  fields = event.as_field_dict()
98
112
  header = resolver.resolve_header(fields)
99
- extensions = resolver.resolve_extensions(fields)
113
+ raw = resolver.resolve_raw_extensions(fields)
114
+ if validate or strict:
115
+ from .validation import CEFValidationError, validate_extensions
116
+
117
+ findings = validate_extensions(raw)
118
+ for finding in findings:
119
+ logger.warning("CEF validation: %s", finding)
120
+ fatal = [f for f in findings if f.fatal]
121
+ if strict and fatal:
122
+ raise CEFValidationError("; ".join(str(f) for f in fatal))
123
+ extensions = {k: cef_escape(v) for k, v in raw.items()}
100
124
  return CEFEvent(header=header, extensions=extensions)
101
125
 
102
126
 
@@ -74,8 +74,8 @@ def follow_files(
74
74
  handle.close()
75
75
 
76
76
 
77
- def _convert_line_mp(line: str, mode: Optional[str], mapping: Optional[str]) -> str:
78
- return convert_line(line, mode=mode, mapping=mapping)
77
+ def _convert_line_mp(line: str, mode: Optional[str], mapping: Optional[str], validate: bool = False, strict: bool = False) -> str:
78
+ return convert_line(line, mode=mode, mapping=mapping, validate=validate, strict=strict)
79
79
 
80
80
 
81
81
  def process_lines(
@@ -85,16 +85,18 @@ def process_lines(
85
85
  mapping: Optional[str],
86
86
  use_multiprocessing: bool,
87
87
  pool_size: Optional[int],
88
+ validate: bool = False,
89
+ strict: bool = False,
88
90
  ) -> Iterator[str]:
89
91
  if use_multiprocessing:
90
92
  size = pool_size or max(1, cpu_count() - 1)
91
93
  with Pool(size) as pool:
92
- worker = partial(_convert_line_mp, mode=mode, mapping=mapping)
94
+ worker = partial(_convert_line_mp, mode=mode, mapping=mapping, validate=validate, strict=strict)
93
95
  for cef in pool.imap(worker, lines):
94
96
  yield cef
95
97
  else:
96
98
  for line in lines:
97
- yield convert_line(line, mode=mode, mapping=mapping)
99
+ yield convert_line(line, mode=mode, mapping=mapping, validate=validate, strict=strict)
98
100
 
99
101
 
100
102
  def main(argv: Optional[list[str]] = None) -> int:
@@ -102,11 +104,16 @@ def main(argv: Optional[list[str]] = None) -> int:
102
104
  parser.add_argument("paths", nargs="*", type=Path, help="Input files (defaults to stdin)")
103
105
  parser.add_argument("-o", "--output", type=Path, help="Output file")
104
106
  parser.add_argument("-a", "--append", action="store_true", help="Append to --output instead of truncating it")
105
- parser.add_argument("--mode", choices=["rfc3164", "rfc5424", "rsyslog_json", "rsyslog_file", "journald_json", "journald_short", "journald_iso"], help="Parser mode override")
107
+ parser.add_argument("--mode", choices=["rfc3164", "rfc5424", "rsyslog_json", "rsyslog_file", "journald_json", "journald_short", "journald_iso", "cisco_seq", "iso_syslog", "kv"], help="Parser mode override")
106
108
  parser.add_argument("--mapping", type=str, help="Mapping JSON file")
107
109
  parser.add_argument("--tail", action="store_true", help="Follow file like tail -f")
110
+ parser.add_argument("--listen", metavar="PROTO:PORT", help="Receive syslog over the network instead of reading files/stdin, e.g. udp:514, tcp:5514, or udp:10.0.0.5:514")
111
+ parser.add_argument("--send", metavar="URL", help="Forward CEF records to udp://HOST:PORT, tcp://HOST:PORT (newline-delimited, with reconnect), or kafka://BROKER:PORT/TOPIC (requires the kafka extra)")
112
+ parser.add_argument("--eps", type=float, help="Rate-limit --send to at most this many events per second")
108
113
  parser.add_argument("--multiprocess", action="store_true", help="Process lines using a process pool")
109
114
  parser.add_argument("--pool-size", type=int, help="Number of worker processes for --multiprocess")
115
+ parser.add_argument("--validate", action="store_true", help="Validate CEF extensions against the ArcSight dictionary; violations are logged as warnings")
116
+ parser.add_argument("--strict", action="store_true", help="Like --validate, but exit with an error on type/length violations")
110
117
  parser.add_argument("--log-level", default="WARNING", help="Logging level")
111
118
 
112
119
  args = parser.parse_args(argv)
@@ -115,20 +122,52 @@ def main(argv: Optional[list[str]] = None) -> int:
115
122
  mapping = args.mapping
116
123
 
117
124
  outputs: Iterator[str]
118
- if args.tail and args.paths:
119
- outputs = process_lines(follow_files(args.paths), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size)
125
+ if args.listen:
126
+ from .net import listen_lines, parse_endpoint
127
+
128
+ try:
129
+ proto, host, port = parse_endpoint(args.listen)
130
+ except ValueError as exc:
131
+ parser.error(str(exc))
132
+ lines = listen_lines(proto, host, port)
133
+ outputs = process_lines(lines, mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size, validate=args.validate or args.strict, strict=args.strict)
134
+ elif args.tail and args.paths:
135
+ outputs = process_lines(follow_files(args.paths), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size, validate=args.validate or args.strict, strict=args.strict)
120
136
  else:
121
- outputs = process_lines(iter_lines_from_sources(args.paths), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size)
137
+ outputs = process_lines(iter_lines_from_sources(args.paths), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size, validate=args.validate or args.strict, strict=args.strict)
122
138
 
123
- if args.output:
124
- with args.output.open("a" if args.append else "w", encoding="utf-8") as fp:
139
+ sender = None
140
+ if args.send:
141
+ from .net import create_sender
142
+
143
+ try:
144
+ sender = create_sender(args.send, eps=args.eps)
145
+ except (ValueError, RuntimeError) as exc:
146
+ parser.error(str(exc))
147
+
148
+ from .validation import CEFValidationError
149
+
150
+ try:
151
+ if args.output:
152
+ with args.output.open("a" if args.append else "w", encoding="utf-8") as fp:
153
+ for cef in outputs:
154
+ fp.write(cef + "\n")
155
+ if args.tail or args.listen:
156
+ fp.flush()
157
+ if sender:
158
+ sender.send(cef)
159
+ elif sender:
125
160
  for cef in outputs:
126
- fp.write(cef + "\n")
127
- if args.tail:
128
- fp.flush()
129
- else:
130
- for cef in outputs:
131
- print(cef)
161
+ sender.send(cef)
162
+ else:
163
+ for cef in outputs:
164
+ print(cef)
165
+ except CEFValidationError as exc:
166
+ print(f"syslogcef: strict validation failed: {exc}", file=sys.stderr)
167
+ return 1
168
+ finally:
169
+ if sender:
170
+ sender.close()
132
171
  return 0
133
172
 
134
173
 
@@ -9,7 +9,9 @@ __all__ = [
9
9
  "CISCO_ASA",
10
10
  "CISCO_IOS",
11
11
  "F5",
12
+ "FORTINET",
12
13
  "LINUX",
14
+ "SOPHOS",
13
15
  "VMWARE",
14
16
  ]
15
17
 
@@ -22,7 +24,9 @@ def _load(name: str) -> Mapping[str, Any]:
22
24
  CISCO_ASA = _load("cisco_asa")
23
25
  CISCO_IOS = _load("cisco_ios")
24
26
  F5 = _load("f5")
27
+ FORTINET = _load("fortinet")
25
28
  LINUX = _load("linux")
29
+ SOPHOS = _load("sophos")
26
30
  VMWARE = _load("vmware")
27
31
 
28
32
 
@@ -31,7 +35,9 @@ def load_mapping(name: str) -> Mapping[str, Any]:
31
35
  "cisco_asa": CISCO_ASA,
32
36
  "cisco_ios": CISCO_IOS,
33
37
  "f5": F5,
38
+ "fortinet": FORTINET,
34
39
  "linux": LINUX,
40
+ "sophos": SOPHOS,
35
41
  "vmware": VMWARE,
36
42
  }
37
43
  try:
@@ -0,0 +1,19 @@
1
+ {
2
+ "deviceVendor": "Fortinet",
3
+ "deviceProduct": "FortiGate",
4
+ "deviceVersion": "auto",
5
+ "eventClassId": "fortinet.%(logid)s",
6
+ "name": "%(message_short)s",
7
+ "extensions": {
8
+ "dvchost": "%(devname)s",
9
+ "src": "%(srcip)s",
10
+ "dst": "%(dstip)s",
11
+ "spt": "%(srcport)s",
12
+ "dpt": "%(dstport)s",
13
+ "act": "%(action)s",
14
+ "app": "%(service)s",
15
+ "cat": "%(type)s",
16
+ "cs1Label": "rawEvent",
17
+ "cs1": "%(raw)s"
18
+ }
19
+ }