syslog2cef 0.1.1__tar.gz → 0.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {syslog2cef-0.1.1/syslog2cef.egg-info → syslog2cef-0.1.3}/PKG-INFO +7 -1
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/README.md +4 -0
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/pyproject.toml +4 -1
- {syslog2cef-0.1.1 → syslog2cef-0.1.3/syslog2cef.egg-info}/PKG-INFO +7 -1
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslog2cef.egg-info/SOURCES.txt +15 -1
- syslog2cef-0.1.3/syslog2cef.egg-info/requires.txt +6 -0
- syslog2cef-0.1.3/syslogcef/adaptive.py +214 -0
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslogcef/api.py +24 -5
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslogcef/cef.py +31 -7
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslogcef/cli.py +55 -16
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslogcef/mappings/__init__.py +6 -0
- syslog2cef-0.1.3/syslogcef/mappings/fortinet.json +19 -0
- syslog2cef-0.1.3/syslogcef/mappings/sophos.json +18 -0
- syslog2cef-0.1.3/syslogcef/net.py +270 -0
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslogcef/normalizer.py +29 -5
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslogcef/parsers.py +205 -9
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslogcef/utils.py +17 -2
- syslog2cef-0.1.3/syslogcef/validation.py +164 -0
- syslog2cef-0.1.3/tests/test_adaptive.py +191 -0
- syslog2cef-0.1.3/tests/test_net.py +180 -0
- syslog2cef-0.1.3/tests/test_real_world.py +93 -0
- syslog2cef-0.1.3/tests/test_validation.py +66 -0
- syslog2cef-0.1.1/syslog2cef.egg-info/requires.txt +0 -3
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/LICENSE +0 -0
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/setup.cfg +0 -0
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslog2cef.egg-info/dependency_links.txt +0 -0
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslog2cef.egg-info/entry_points.txt +0 -0
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslog2cef.egg-info/top_level.txt +0 -0
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslogcef/__init__.py +0 -0
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslogcef/__main__.py +0 -0
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslogcef/mappings/cisco_asa.json +0 -0
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslogcef/mappings/cisco_ios.json +0 -0
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslogcef/mappings/f5.json +0 -0
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslogcef/mappings/linux.json +0 -0
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/syslogcef/mappings/vmware.json +0 -0
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/tests/test_cef.py +0 -0
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/tests/test_cli.py +0 -0
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/tests/test_normalizer.py +0 -0
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/tests/test_parsers.py +0 -0
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/tests/test_parsers_extra.py +0 -0
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/tests/test_pipeline.py +0 -0
- {syslog2cef-0.1.1 → syslog2cef-0.1.3}/tests/test_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: syslog2cef
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.3
|
|
4
4
|
Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
|
|
5
5
|
Author-email: Tamir Suliman <allamiro@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/allamiro/syslogcef
|
|
@@ -15,6 +15,8 @@ Description-Content-Type: text/markdown
|
|
|
15
15
|
License-File: LICENSE
|
|
16
16
|
Provides-Extra: test
|
|
17
17
|
Requires-Dist: pytest>=7; extra == "test"
|
|
18
|
+
Provides-Extra: kafka
|
|
19
|
+
Requires-Dist: kafka-python>=2.0; extra == "kafka"
|
|
18
20
|
Dynamic: license-file
|
|
19
21
|
|
|
20
22
|
<p align="center">
|
|
@@ -99,6 +101,10 @@ signatures. The public key is committed at
|
|
|
99
101
|
|
|
100
102
|
## Command Line Usage
|
|
101
103
|
|
|
104
|
+
<p align="center">
|
|
105
|
+
<img src="https://raw.githubusercontent.com/allamiro/syslogcef/main/docs/demo.svg" alt="Animated demo of syslogcef converting different formats" width="820">
|
|
106
|
+
</p>
|
|
107
|
+
|
|
102
108
|
```bash
|
|
103
109
|
# Read from stdin, write CEF to stdout
|
|
104
110
|
syslogcef < /var/log/syslog
|
|
@@ -80,6 +80,10 @@ signatures. The public key is committed at
|
|
|
80
80
|
|
|
81
81
|
## Command Line Usage
|
|
82
82
|
|
|
83
|
+
<p align="center">
|
|
84
|
+
<img src="https://raw.githubusercontent.com/allamiro/syslogcef/main/docs/demo.svg" alt="Animated demo of syslogcef converting different formats" width="820">
|
|
85
|
+
</p>
|
|
86
|
+
|
|
83
87
|
```bash
|
|
84
88
|
# Read from stdin, write CEF to stdout
|
|
85
89
|
syslogcef < /var/log/syslog
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "syslog2cef"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.3"
|
|
8
8
|
description = "Convert RFC3164/RFC5424 syslog lines into ArcSight CEF"
|
|
9
9
|
authors = [{name = "Tamir Suliman", email = "allamiro@gmail.com"}]
|
|
10
10
|
readme = "README.md"
|
|
@@ -26,6 +26,9 @@ Issues = "https://github.com/allamiro/syslogcef/issues"
|
|
|
26
26
|
test = [
|
|
27
27
|
"pytest>=7",
|
|
28
28
|
]
|
|
29
|
+
kafka = [
|
|
30
|
+
"kafka-python>=2.0",
|
|
31
|
+
]
|
|
29
32
|
|
|
30
33
|
[project.scripts]
|
|
31
34
|
syslogcef = "syslogcef.cli:main"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: syslog2cef
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.3
|
|
4
4
|
Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
|
|
5
5
|
Author-email: Tamir Suliman <allamiro@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/allamiro/syslogcef
|
|
@@ -15,6 +15,8 @@ Description-Content-Type: text/markdown
|
|
|
15
15
|
License-File: LICENSE
|
|
16
16
|
Provides-Extra: test
|
|
17
17
|
Requires-Dist: pytest>=7; extra == "test"
|
|
18
|
+
Provides-Extra: kafka
|
|
19
|
+
Requires-Dist: kafka-python>=2.0; extra == "kafka"
|
|
18
20
|
Dynamic: license-file
|
|
19
21
|
|
|
20
22
|
<p align="center">
|
|
@@ -99,6 +101,10 @@ signatures. The public key is committed at
|
|
|
99
101
|
|
|
100
102
|
## Command Line Usage
|
|
101
103
|
|
|
104
|
+
<p align="center">
|
|
105
|
+
<img src="https://raw.githubusercontent.com/allamiro/syslogcef/main/docs/demo.svg" alt="Animated demo of syslogcef converting different formats" width="820">
|
|
106
|
+
</p>
|
|
107
|
+
|
|
102
108
|
```bash
|
|
103
109
|
# Read from stdin, write CEF to stdout
|
|
104
110
|
syslogcef < /var/log/syslog
|
|
@@ -3,17 +3,22 @@ README.md
|
|
|
3
3
|
pyproject.toml
|
|
4
4
|
./syslogcef/__init__.py
|
|
5
5
|
./syslogcef/__main__.py
|
|
6
|
+
./syslogcef/adaptive.py
|
|
6
7
|
./syslogcef/api.py
|
|
7
8
|
./syslogcef/cef.py
|
|
8
9
|
./syslogcef/cli.py
|
|
10
|
+
./syslogcef/net.py
|
|
9
11
|
./syslogcef/normalizer.py
|
|
10
12
|
./syslogcef/parsers.py
|
|
11
13
|
./syslogcef/utils.py
|
|
14
|
+
./syslogcef/validation.py
|
|
12
15
|
./syslogcef/mappings/__init__.py
|
|
13
16
|
./syslogcef/mappings/cisco_asa.json
|
|
14
17
|
./syslogcef/mappings/cisco_ios.json
|
|
15
18
|
./syslogcef/mappings/f5.json
|
|
19
|
+
./syslogcef/mappings/fortinet.json
|
|
16
20
|
./syslogcef/mappings/linux.json
|
|
21
|
+
./syslogcef/mappings/sophos.json
|
|
17
22
|
./syslogcef/mappings/vmware.json
|
|
18
23
|
syslog2cef.egg-info/PKG-INFO
|
|
19
24
|
syslog2cef.egg-info/SOURCES.txt
|
|
@@ -23,22 +28,31 @@ syslog2cef.egg-info/requires.txt
|
|
|
23
28
|
syslog2cef.egg-info/top_level.txt
|
|
24
29
|
syslogcef/__init__.py
|
|
25
30
|
syslogcef/__main__.py
|
|
31
|
+
syslogcef/adaptive.py
|
|
26
32
|
syslogcef/api.py
|
|
27
33
|
syslogcef/cef.py
|
|
28
34
|
syslogcef/cli.py
|
|
35
|
+
syslogcef/net.py
|
|
29
36
|
syslogcef/normalizer.py
|
|
30
37
|
syslogcef/parsers.py
|
|
31
38
|
syslogcef/utils.py
|
|
39
|
+
syslogcef/validation.py
|
|
32
40
|
syslogcef/mappings/__init__.py
|
|
33
41
|
syslogcef/mappings/cisco_asa.json
|
|
34
42
|
syslogcef/mappings/cisco_ios.json
|
|
35
43
|
syslogcef/mappings/f5.json
|
|
44
|
+
syslogcef/mappings/fortinet.json
|
|
36
45
|
syslogcef/mappings/linux.json
|
|
46
|
+
syslogcef/mappings/sophos.json
|
|
37
47
|
syslogcef/mappings/vmware.json
|
|
48
|
+
tests/test_adaptive.py
|
|
38
49
|
tests/test_cef.py
|
|
39
50
|
tests/test_cli.py
|
|
51
|
+
tests/test_net.py
|
|
40
52
|
tests/test_normalizer.py
|
|
41
53
|
tests/test_parsers.py
|
|
42
54
|
tests/test_parsers_extra.py
|
|
43
55
|
tests/test_pipeline.py
|
|
44
|
-
tests/
|
|
56
|
+
tests/test_real_world.py
|
|
57
|
+
tests/test_utils.py
|
|
58
|
+
tests/test_validation.py
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
"""Adaptive pattern detection for lines no known parser matches.
|
|
2
|
+
|
|
3
|
+
When every dedicated parser fails, this module analyzes the line: it
|
|
4
|
+
locates a timestamp using a library of datetime shapes, identifies a
|
|
5
|
+
plausible host token, and synthesizes a compiled regex for that line
|
|
6
|
+
layout. The synthesized pattern is cached by the line's shape signature,
|
|
7
|
+
so subsequent lines with the same layout parse with a single regex match
|
|
8
|
+
instead of a fresh analysis. Events parsed this way report
|
|
9
|
+
``source_hint="adaptive"``.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import re
|
|
15
|
+
from datetime import datetime, timezone
|
|
16
|
+
from typing import Callable, Dict, Optional, Pattern, Tuple
|
|
17
|
+
|
|
18
|
+
from .utils import convert_pri, month_abbr_to_int, parse_iso8601, sanitize_message
|
|
19
|
+
|
|
20
|
+
PRI_RE = re.compile(r"^<(\d{1,3})>")
|
|
21
|
+
|
|
22
|
+
# Each entry: (name, regex source, converter(match, now) -> datetime)
|
|
23
|
+
def _iso(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
|
|
24
|
+
return parse_iso8601(match.group(0).replace(" ", "T", 1) if "T" not in match.group(0) else match.group(0))
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _epoch(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
|
|
28
|
+
return parse_iso8601(match.group(0))
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _mon_day(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
|
|
32
|
+
month = month_abbr_to_int(match.group("month"))
|
|
33
|
+
day = int(match.group("day"))
|
|
34
|
+
if match.groupdict().get("year"):
|
|
35
|
+
hour, minute, second = map(int, match.group("hms").split(":"))
|
|
36
|
+
return datetime(int(match.group("year")), month, day, hour, minute, second, tzinfo=timezone.utc)
|
|
37
|
+
# No year: use the same rollover inference as the dedicated parsers so a
|
|
38
|
+
# December event parsed in January lands in the previous year.
|
|
39
|
+
from .parsers import _infer_timestamp
|
|
40
|
+
|
|
41
|
+
return _infer_timestamp(month, day, match.group("hms"), now=now)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _ymd_slash(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
|
|
45
|
+
y, m, d = int(match.group("y")), int(match.group("m")), int(match.group("d"))
|
|
46
|
+
hour, minute, second = map(int, match.group("hms").split(":"))
|
|
47
|
+
return datetime(y, m, d, hour, minute, second, tzinfo=timezone.utc)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _dmy_dash(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
|
|
51
|
+
d = int(match.group("d"))
|
|
52
|
+
month = month_abbr_to_int(match.group("month").title())
|
|
53
|
+
y = int(match.group("y"))
|
|
54
|
+
hour, minute, second = map(int, match.group("hms").split(":"))
|
|
55
|
+
return datetime(y, month, d, hour, minute, second, tzinfo=timezone.utc)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
TIMESTAMP_LIB: Tuple[Tuple[str, str, Callable], ...] = (
|
|
59
|
+
("iso", r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?", _iso),
|
|
60
|
+
("mon_day_year", r"(?P<month>[A-Z][a-z]{2})\s+(?P<day>\d{1,2})\s+(?P<year>\d{4})\s+(?P<hms>\d{2}:\d{2}:\d{2})(?:\.\d+)?", _mon_day),
|
|
61
|
+
("mon_day", r"(?P<month>[A-Z][a-z]{2})\s+(?P<day>\d{1,2})\s+(?P<hms>\d{2}:\d{2}:\d{2})(?:\.\d+)?", _mon_day),
|
|
62
|
+
("ymd_slash", r"(?P<y>\d{4})/(?P<m>\d{2})/(?P<d>\d{2})[ T](?P<hms>\d{2}:\d{2}:\d{2})", _ymd_slash),
|
|
63
|
+
("dmy_dash", r"(?P<d>\d{1,2})-(?P<month>[A-Za-z]{3})-(?P<y>\d{4})\s+(?P<hms>\d{2}:\d{2}:\d{2})", _dmy_dash),
|
|
64
|
+
("epoch", r"\b1\d{9}(?:\d{6})?\b", _epoch),
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
HOST_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
|
|
68
|
+
_SKIP_HOST_TOKENS = {"error", "warning", "info", "debug", "notice", "kernel"}
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _valid_host(candidate: Optional[str]) -> bool:
|
|
72
|
+
return bool(
|
|
73
|
+
candidate
|
|
74
|
+
and "=" not in candidate
|
|
75
|
+
and HOST_RE.match(candidate)
|
|
76
|
+
and candidate.lower() not in _SKIP_HOST_TOKENS
|
|
77
|
+
and not candidate.isdigit()
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
_CACHE: Dict[str, Tuple[Pattern[str], str, Callable]] = {}
|
|
81
|
+
_CACHE_MAX = 256
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _signature(line: str) -> str:
|
|
85
|
+
"""Reduce a line to a layout signature: token classes of the prefix.
|
|
86
|
+
|
|
87
|
+
Only the first few tokens participate — the layout (timestamp + host)
|
|
88
|
+
lives there, while later tokens are free-form message words that must
|
|
89
|
+
not fragment the cache.
|
|
90
|
+
"""
|
|
91
|
+
tokens = line[:96].split()[:4]
|
|
92
|
+
classes = []
|
|
93
|
+
for token in tokens:
|
|
94
|
+
cls = re.sub(r"[A-Za-z]+", "A", token)
|
|
95
|
+
cls = re.sub(r"\d+", "9", cls)
|
|
96
|
+
classes.append(cls)
|
|
97
|
+
return " ".join(classes)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def clear_cache() -> None:
|
|
101
|
+
_CACHE.clear()
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def cache_size() -> int:
|
|
105
|
+
return len(_CACHE)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _build_event(line, pri, ts, host, msg):
|
|
109
|
+
from .parsers import ParsedEvent
|
|
110
|
+
|
|
111
|
+
facility, severity = convert_pri(pri)
|
|
112
|
+
return ParsedEvent(
|
|
113
|
+
pri=pri,
|
|
114
|
+
facility=facility,
|
|
115
|
+
severity=severity,
|
|
116
|
+
ts=ts,
|
|
117
|
+
ts_orig="",
|
|
118
|
+
host=host,
|
|
119
|
+
app=None,
|
|
120
|
+
pid=None,
|
|
121
|
+
msgid=None,
|
|
122
|
+
sd={},
|
|
123
|
+
msg=sanitize_message(msg),
|
|
124
|
+
raw=line,
|
|
125
|
+
source_hint="adaptive",
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def adaptive_parse(line: str, *, now: Optional[datetime] = None):
|
|
130
|
+
"""Parse an unknown-format line, learning its layout on first sight."""
|
|
131
|
+
|
|
132
|
+
raw = line
|
|
133
|
+
pri = None
|
|
134
|
+
pri_match = PRI_RE.match(line)
|
|
135
|
+
if pri_match:
|
|
136
|
+
pri = int(pri_match.group(1))
|
|
137
|
+
line = line[pri_match.end():]
|
|
138
|
+
|
|
139
|
+
sig = _signature(line)
|
|
140
|
+
cached = _CACHE.get(sig)
|
|
141
|
+
if cached is not None:
|
|
142
|
+
pattern, _name, conv = cached
|
|
143
|
+
match = pattern.match(line)
|
|
144
|
+
if match is not None:
|
|
145
|
+
try:
|
|
146
|
+
ts = conv(pattern_ts_match(pattern, match), now)
|
|
147
|
+
except (ValueError, KeyError):
|
|
148
|
+
ts = None
|
|
149
|
+
gd = match.groupdict()
|
|
150
|
+
host = gd.get("adaptive_host")
|
|
151
|
+
msg = gd.get("adaptive_msg")
|
|
152
|
+
msg = line if msg is None else msg
|
|
153
|
+
# The shape signature reduces all words to the same class, so a
|
|
154
|
+
# cached host slot may capture a non-host token; revalidate.
|
|
155
|
+
if host is not None and not _valid_host(host.rstrip(":,")):
|
|
156
|
+
msg = f"{host} {msg}".strip()
|
|
157
|
+
host = None
|
|
158
|
+
return _build_event(raw, pri, ts, host, msg)
|
|
159
|
+
|
|
160
|
+
# Analysis pass: find the earliest timestamp in the line head.
|
|
161
|
+
best = None
|
|
162
|
+
for name, src, conv in TIMESTAMP_LIB:
|
|
163
|
+
match = re.compile(src).search(line[:96])
|
|
164
|
+
if match and (best is None or match.start() < best[0].start()):
|
|
165
|
+
best = (match, name, src, conv)
|
|
166
|
+
if best is None:
|
|
167
|
+
return None
|
|
168
|
+
ts_match, name, src, conv = best
|
|
169
|
+
try:
|
|
170
|
+
ts = conv(ts_match, now)
|
|
171
|
+
except (ValueError, KeyError):
|
|
172
|
+
return None
|
|
173
|
+
|
|
174
|
+
# Host candidate: the first plausible token after the timestamp that is
|
|
175
|
+
# not a key=value pair or severity word.
|
|
176
|
+
rest = line[ts_match.end():].lstrip(" :,\t")
|
|
177
|
+
host = None
|
|
178
|
+
msg = rest
|
|
179
|
+
tokens = rest.split(None, 1)
|
|
180
|
+
if tokens:
|
|
181
|
+
candidate = tokens[0].rstrip(":,")
|
|
182
|
+
if _valid_host(candidate):
|
|
183
|
+
host = candidate
|
|
184
|
+
msg = tokens[1] if len(tokens) > 1 else ""
|
|
185
|
+
|
|
186
|
+
# Synthesize and cache a pattern for this layout.
|
|
187
|
+
prefix = line[: ts_match.start()]
|
|
188
|
+
parts = [re.escape(prefix), f"(?P<adaptive_ts>{src})", r"[\s:,]*"]
|
|
189
|
+
if host is not None:
|
|
190
|
+
parts.append(r"(?P<adaptive_host>\S+?):?(?:\s+|$)")
|
|
191
|
+
parts.append(r"(?P<adaptive_msg>.*)$")
|
|
192
|
+
try:
|
|
193
|
+
pattern = re.compile("".join(parts))
|
|
194
|
+
except re.error:
|
|
195
|
+
pattern = None
|
|
196
|
+
if pattern is not None and pattern.match(line):
|
|
197
|
+
if len(_CACHE) >= _CACHE_MAX:
|
|
198
|
+
_CACHE.pop(next(iter(_CACHE)))
|
|
199
|
+
_CACHE[sig] = (pattern, name, conv)
|
|
200
|
+
|
|
201
|
+
return _build_event(raw, pri, ts, host, msg)
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def pattern_ts_match(pattern: Pattern[str], match: "re.Match[str]"):
|
|
205
|
+
"""Re-match the timestamp portion so converters see their own groups."""
|
|
206
|
+
ts_text = match.group("adaptive_ts")
|
|
207
|
+
for _name, src, _conv in TIMESTAMP_LIB:
|
|
208
|
+
m = re.compile(src).fullmatch(ts_text)
|
|
209
|
+
if m:
|
|
210
|
+
return m
|
|
211
|
+
raise ValueError(f"cached timestamp no longer matches: {ts_text!r}")
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
__all__ = ["adaptive_parse", "cache_size", "clear_cache"]
|
|
@@ -9,7 +9,7 @@ from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional
|
|
|
9
9
|
|
|
10
10
|
from .parsers import ParsedEvent, autodetect_and_parse
|
|
11
11
|
from .normalizer import NormalizedEvent, normalize
|
|
12
|
-
from .mappings import CISCO_ASA, CISCO_IOS, F5, LINUX, VMWARE
|
|
12
|
+
from .mappings import CISCO_ASA, CISCO_IOS, F5, FORTINET, LINUX, SOPHOS, VMWARE
|
|
13
13
|
from .cef import CEFEvent, build_cef
|
|
14
14
|
|
|
15
15
|
logger = logging.getLogger(__name__)
|
|
@@ -74,14 +74,23 @@ def _load_mapping(mapping: Mapping[str, Any] | Path | str | None) -> Mapping[str
|
|
|
74
74
|
def to_cef(
|
|
75
75
|
event: NormalizedEvent,
|
|
76
76
|
mapping: Mapping[str, Any] | Path | str | None = None,
|
|
77
|
+
*,
|
|
78
|
+
validate: bool = False,
|
|
79
|
+
strict: bool = False,
|
|
77
80
|
) -> str:
|
|
78
|
-
"""Convert a normalized event into a CEF string.
|
|
81
|
+
"""Convert a normalized event into a CEF string.
|
|
82
|
+
|
|
83
|
+
With ``validate=True``, extension values are checked against the
|
|
84
|
+
ArcSight dictionary and violations logged as warnings; ``strict=True``
|
|
85
|
+
additionally raises :class:`syslogcef.validation.CEFValidationError`
|
|
86
|
+
on type or length violations.
|
|
87
|
+
"""
|
|
79
88
|
|
|
80
89
|
if mapping is None:
|
|
81
90
|
mapping_data = _guess_mapping(event)
|
|
82
91
|
else:
|
|
83
92
|
mapping_data = _load_mapping(mapping)
|
|
84
|
-
cef_event = build_cef(event, mapping_data)
|
|
93
|
+
cef_event = build_cef(event, mapping_data, validate=validate, strict=strict)
|
|
85
94
|
return cef_event.render()
|
|
86
95
|
|
|
87
96
|
|
|
@@ -91,27 +100,37 @@ def convert_line(
|
|
|
91
100
|
mode: Optional[str] = None,
|
|
92
101
|
mapping: Mapping[str, Any] | Path | str | None = None,
|
|
93
102
|
now: Optional[datetime] = None,
|
|
103
|
+
validate: bool = False,
|
|
104
|
+
strict: bool = False,
|
|
94
105
|
) -> str:
|
|
95
106
|
"""Full pipeline that parses, normalizes and converts a syslog line."""
|
|
96
107
|
|
|
97
108
|
parsed = parse_syslog(line, mode=mode, now=now)
|
|
98
109
|
normalized = normalize_event(parsed)
|
|
99
110
|
|
|
100
|
-
return to_cef(normalized, mapping)
|
|
111
|
+
return to_cef(normalized, mapping, validate=validate, strict=strict)
|
|
101
112
|
|
|
102
113
|
|
|
103
114
|
def _guess_mapping(event: NormalizedEvent) -> Mapping[str, Any]:
|
|
104
115
|
msg_upper = event.msg.upper()
|
|
105
116
|
app_upper = (event.app or "").upper()
|
|
117
|
+
event_code = event.kv.get("event_code", "")
|
|
106
118
|
|
|
107
|
-
if "%ASA-" in msg_upper or "ASA" in app_upper:
|
|
119
|
+
if "%ASA-" in msg_upper or "ASA" in app_upper or event_code.startswith(("ASA-", "FTD-")):
|
|
108
120
|
return CISCO_ASA
|
|
109
121
|
if "IOS" in app_upper or "%IOS-" in msg_upper:
|
|
110
122
|
return CISCO_IOS
|
|
111
123
|
if "BIG-IP" in msg_upper or "F5" in app_upper:
|
|
112
124
|
return F5
|
|
125
|
+
if "logid" in event.kv:
|
|
126
|
+
return FORTINET
|
|
127
|
+
if "log_id" in event.kv:
|
|
128
|
+
return SOPHOS
|
|
113
129
|
if "VMWARE" in msg_upper or "ESXI" in msg_upper or "VMWARE" in app_upper:
|
|
114
130
|
return VMWARE
|
|
131
|
+
if event_code:
|
|
132
|
+
# Other %FAC-SEV-MNEMONIC codes are Cisco IOS style.
|
|
133
|
+
return CISCO_IOS
|
|
115
134
|
return LINUX
|
|
116
135
|
|
|
117
136
|
|
|
@@ -56,7 +56,12 @@ class MappingResolver:
|
|
|
56
56
|
header = {}
|
|
57
57
|
for key in ["deviceVendor", "deviceProduct", "deviceVersion", "eventClassId", "name"]:
|
|
58
58
|
template = self.mapping.get(key, DEFAULT_MAPPING[key])
|
|
59
|
-
|
|
59
|
+
value = self._format(template, fields)
|
|
60
|
+
if not value:
|
|
61
|
+
# A template referencing missing fields must not leave an
|
|
62
|
+
# empty CEF header slot; fall back to the default template.
|
|
63
|
+
value = self._format(DEFAULT_MAPPING[key], fields)
|
|
64
|
+
header[key] = cef_escape_header(value)
|
|
60
65
|
severity = self._resolve_severity(fields)
|
|
61
66
|
header["severity"] = severity
|
|
62
67
|
return header
|
|
@@ -71,15 +76,18 @@ class MappingResolver:
|
|
|
71
76
|
return "3"
|
|
72
77
|
return severity_map.get(str(value), str(value))
|
|
73
78
|
|
|
74
|
-
def
|
|
75
|
-
|
|
79
|
+
def resolve_raw_extensions(self, fields: Mapping[str, Any]) -> Dict[str, str]:
|
|
80
|
+
raw = {}
|
|
76
81
|
merged = DEFAULT_MAPPING["extensions"].copy()
|
|
77
82
|
merged.update(self.mapping.get("extensions", {}))
|
|
78
83
|
for key, template in merged.items():
|
|
79
84
|
value = self._format(template, fields)
|
|
80
85
|
if value:
|
|
81
|
-
|
|
82
|
-
return
|
|
86
|
+
raw[key] = str(value)
|
|
87
|
+
return raw
|
|
88
|
+
|
|
89
|
+
def resolve_extensions(self, fields: Mapping[str, Any]) -> Dict[str, str]:
|
|
90
|
+
return {k: cef_escape(v) for k, v in self.resolve_raw_extensions(fields).items()}
|
|
83
91
|
|
|
84
92
|
def _format(self, template: str, fields: Mapping[str, Any]) -> str:
|
|
85
93
|
try:
|
|
@@ -92,11 +100,27 @@ class MappingResolver:
|
|
|
92
100
|
return ""
|
|
93
101
|
|
|
94
102
|
|
|
95
|
-
def build_cef(
|
|
103
|
+
def build_cef(
|
|
104
|
+
event: NormalizedEvent,
|
|
105
|
+
mapping: Mapping[str, Any],
|
|
106
|
+
*,
|
|
107
|
+
validate: bool = False,
|
|
108
|
+
strict: bool = False,
|
|
109
|
+
) -> CEFEvent:
|
|
96
110
|
resolver = MappingResolver(mapping)
|
|
97
111
|
fields = event.as_field_dict()
|
|
98
112
|
header = resolver.resolve_header(fields)
|
|
99
|
-
|
|
113
|
+
raw = resolver.resolve_raw_extensions(fields)
|
|
114
|
+
if validate or strict:
|
|
115
|
+
from .validation import CEFValidationError, validate_extensions
|
|
116
|
+
|
|
117
|
+
findings = validate_extensions(raw)
|
|
118
|
+
for finding in findings:
|
|
119
|
+
logger.warning("CEF validation: %s", finding)
|
|
120
|
+
fatal = [f for f in findings if f.fatal]
|
|
121
|
+
if strict and fatal:
|
|
122
|
+
raise CEFValidationError("; ".join(str(f) for f in fatal))
|
|
123
|
+
extensions = {k: cef_escape(v) for k, v in raw.items()}
|
|
100
124
|
return CEFEvent(header=header, extensions=extensions)
|
|
101
125
|
|
|
102
126
|
|
|
@@ -74,8 +74,8 @@ def follow_files(
|
|
|
74
74
|
handle.close()
|
|
75
75
|
|
|
76
76
|
|
|
77
|
-
def _convert_line_mp(line: str, mode: Optional[str], mapping: Optional[str]) -> str:
|
|
78
|
-
return convert_line(line, mode=mode, mapping=mapping)
|
|
77
|
+
def _convert_line_mp(line: str, mode: Optional[str], mapping: Optional[str], validate: bool = False, strict: bool = False) -> str:
|
|
78
|
+
return convert_line(line, mode=mode, mapping=mapping, validate=validate, strict=strict)
|
|
79
79
|
|
|
80
80
|
|
|
81
81
|
def process_lines(
|
|
@@ -85,16 +85,18 @@ def process_lines(
|
|
|
85
85
|
mapping: Optional[str],
|
|
86
86
|
use_multiprocessing: bool,
|
|
87
87
|
pool_size: Optional[int],
|
|
88
|
+
validate: bool = False,
|
|
89
|
+
strict: bool = False,
|
|
88
90
|
) -> Iterator[str]:
|
|
89
91
|
if use_multiprocessing:
|
|
90
92
|
size = pool_size or max(1, cpu_count() - 1)
|
|
91
93
|
with Pool(size) as pool:
|
|
92
|
-
worker = partial(_convert_line_mp, mode=mode, mapping=mapping)
|
|
94
|
+
worker = partial(_convert_line_mp, mode=mode, mapping=mapping, validate=validate, strict=strict)
|
|
93
95
|
for cef in pool.imap(worker, lines):
|
|
94
96
|
yield cef
|
|
95
97
|
else:
|
|
96
98
|
for line in lines:
|
|
97
|
-
yield convert_line(line, mode=mode, mapping=mapping)
|
|
99
|
+
yield convert_line(line, mode=mode, mapping=mapping, validate=validate, strict=strict)
|
|
98
100
|
|
|
99
101
|
|
|
100
102
|
def main(argv: Optional[list[str]] = None) -> int:
|
|
@@ -102,11 +104,16 @@ def main(argv: Optional[list[str]] = None) -> int:
|
|
|
102
104
|
parser.add_argument("paths", nargs="*", type=Path, help="Input files (defaults to stdin)")
|
|
103
105
|
parser.add_argument("-o", "--output", type=Path, help="Output file")
|
|
104
106
|
parser.add_argument("-a", "--append", action="store_true", help="Append to --output instead of truncating it")
|
|
105
|
-
parser.add_argument("--mode", choices=["rfc3164", "rfc5424", "rsyslog_json", "rsyslog_file", "journald_json", "journald_short", "journald_iso"], help="Parser mode override")
|
|
107
|
+
parser.add_argument("--mode", choices=["rfc3164", "rfc5424", "rsyslog_json", "rsyslog_file", "journald_json", "journald_short", "journald_iso", "cisco_seq", "iso_syslog", "kv"], help="Parser mode override")
|
|
106
108
|
parser.add_argument("--mapping", type=str, help="Mapping JSON file")
|
|
107
109
|
parser.add_argument("--tail", action="store_true", help="Follow file like tail -f")
|
|
110
|
+
parser.add_argument("--listen", metavar="PROTO:PORT", help="Receive syslog over the network instead of reading files/stdin, e.g. udp:514, tcp:5514, or udp:10.0.0.5:514")
|
|
111
|
+
parser.add_argument("--send", metavar="URL", help="Forward CEF records to udp://HOST:PORT, tcp://HOST:PORT (newline-delimited, with reconnect), or kafka://BROKER:PORT/TOPIC (requires the kafka extra)")
|
|
112
|
+
parser.add_argument("--eps", type=float, help="Rate-limit --send to at most this many events per second")
|
|
108
113
|
parser.add_argument("--multiprocess", action="store_true", help="Process lines using a process pool")
|
|
109
114
|
parser.add_argument("--pool-size", type=int, help="Number of worker processes for --multiprocess")
|
|
115
|
+
parser.add_argument("--validate", action="store_true", help="Validate CEF extensions against the ArcSight dictionary; violations are logged as warnings")
|
|
116
|
+
parser.add_argument("--strict", action="store_true", help="Like --validate, but exit with an error on type/length violations")
|
|
110
117
|
parser.add_argument("--log-level", default="WARNING", help="Logging level")
|
|
111
118
|
|
|
112
119
|
args = parser.parse_args(argv)
|
|
@@ -115,20 +122,52 @@ def main(argv: Optional[list[str]] = None) -> int:
|
|
|
115
122
|
mapping = args.mapping
|
|
116
123
|
|
|
117
124
|
outputs: Iterator[str]
|
|
118
|
-
if args.
|
|
119
|
-
|
|
125
|
+
if args.listen:
|
|
126
|
+
from .net import listen_lines, parse_endpoint
|
|
127
|
+
|
|
128
|
+
try:
|
|
129
|
+
proto, host, port = parse_endpoint(args.listen)
|
|
130
|
+
except ValueError as exc:
|
|
131
|
+
parser.error(str(exc))
|
|
132
|
+
lines = listen_lines(proto, host, port)
|
|
133
|
+
outputs = process_lines(lines, mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size, validate=args.validate or args.strict, strict=args.strict)
|
|
134
|
+
elif args.tail and args.paths:
|
|
135
|
+
outputs = process_lines(follow_files(args.paths), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size, validate=args.validate or args.strict, strict=args.strict)
|
|
120
136
|
else:
|
|
121
|
-
outputs = process_lines(iter_lines_from_sources(args.paths), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size)
|
|
137
|
+
outputs = process_lines(iter_lines_from_sources(args.paths), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size, validate=args.validate or args.strict, strict=args.strict)
|
|
122
138
|
|
|
123
|
-
|
|
124
|
-
|
|
139
|
+
sender = None
|
|
140
|
+
if args.send:
|
|
141
|
+
from .net import create_sender
|
|
142
|
+
|
|
143
|
+
try:
|
|
144
|
+
sender = create_sender(args.send, eps=args.eps)
|
|
145
|
+
except (ValueError, RuntimeError) as exc:
|
|
146
|
+
parser.error(str(exc))
|
|
147
|
+
|
|
148
|
+
from .validation import CEFValidationError
|
|
149
|
+
|
|
150
|
+
try:
|
|
151
|
+
if args.output:
|
|
152
|
+
with args.output.open("a" if args.append else "w", encoding="utf-8") as fp:
|
|
153
|
+
for cef in outputs:
|
|
154
|
+
fp.write(cef + "\n")
|
|
155
|
+
if args.tail or args.listen:
|
|
156
|
+
fp.flush()
|
|
157
|
+
if sender:
|
|
158
|
+
sender.send(cef)
|
|
159
|
+
elif sender:
|
|
125
160
|
for cef in outputs:
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
161
|
+
sender.send(cef)
|
|
162
|
+
else:
|
|
163
|
+
for cef in outputs:
|
|
164
|
+
print(cef)
|
|
165
|
+
except CEFValidationError as exc:
|
|
166
|
+
print(f"syslogcef: strict validation failed: {exc}", file=sys.stderr)
|
|
167
|
+
return 1
|
|
168
|
+
finally:
|
|
169
|
+
if sender:
|
|
170
|
+
sender.close()
|
|
132
171
|
return 0
|
|
133
172
|
|
|
134
173
|
|
|
@@ -9,7 +9,9 @@ __all__ = [
|
|
|
9
9
|
"CISCO_ASA",
|
|
10
10
|
"CISCO_IOS",
|
|
11
11
|
"F5",
|
|
12
|
+
"FORTINET",
|
|
12
13
|
"LINUX",
|
|
14
|
+
"SOPHOS",
|
|
13
15
|
"VMWARE",
|
|
14
16
|
]
|
|
15
17
|
|
|
@@ -22,7 +24,9 @@ def _load(name: str) -> Mapping[str, Any]:
|
|
|
22
24
|
CISCO_ASA = _load("cisco_asa")
|
|
23
25
|
CISCO_IOS = _load("cisco_ios")
|
|
24
26
|
F5 = _load("f5")
|
|
27
|
+
FORTINET = _load("fortinet")
|
|
25
28
|
LINUX = _load("linux")
|
|
29
|
+
SOPHOS = _load("sophos")
|
|
26
30
|
VMWARE = _load("vmware")
|
|
27
31
|
|
|
28
32
|
|
|
@@ -31,7 +35,9 @@ def load_mapping(name: str) -> Mapping[str, Any]:
|
|
|
31
35
|
"cisco_asa": CISCO_ASA,
|
|
32
36
|
"cisco_ios": CISCO_IOS,
|
|
33
37
|
"f5": F5,
|
|
38
|
+
"fortinet": FORTINET,
|
|
34
39
|
"linux": LINUX,
|
|
40
|
+
"sophos": SOPHOS,
|
|
35
41
|
"vmware": VMWARE,
|
|
36
42
|
}
|
|
37
43
|
try:
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
{
|
|
2
|
+
"deviceVendor": "Fortinet",
|
|
3
|
+
"deviceProduct": "FortiGate",
|
|
4
|
+
"deviceVersion": "auto",
|
|
5
|
+
"eventClassId": "fortinet.%(logid)s",
|
|
6
|
+
"name": "%(message_short)s",
|
|
7
|
+
"extensions": {
|
|
8
|
+
"dvchost": "%(devname)s",
|
|
9
|
+
"src": "%(srcip)s",
|
|
10
|
+
"dst": "%(dstip)s",
|
|
11
|
+
"spt": "%(srcport)s",
|
|
12
|
+
"dpt": "%(dstport)s",
|
|
13
|
+
"act": "%(action)s",
|
|
14
|
+
"app": "%(service)s",
|
|
15
|
+
"cat": "%(type)s",
|
|
16
|
+
"cs1Label": "rawEvent",
|
|
17
|
+
"cs1": "%(raw)s"
|
|
18
|
+
}
|
|
19
|
+
}
|