syslog2cef 0.1.0__tar.gz → 0.1.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. {syslog2cef-0.1.0/syslog2cef.egg-info → syslog2cef-0.1.2}/PKG-INFO +46 -4
  2. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/README.md +45 -3
  3. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/pyproject.toml +1 -1
  4. {syslog2cef-0.1.0 → syslog2cef-0.1.2/syslog2cef.egg-info}/PKG-INFO +46 -4
  5. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslog2cef.egg-info/SOURCES.txt +8 -0
  6. syslog2cef-0.1.2/syslogcef/adaptive.py +197 -0
  7. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslogcef/api.py +11 -3
  8. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslogcef/cef.py +1 -1
  9. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslogcef/cli.py +1 -1
  10. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslogcef/mappings/__init__.py +6 -0
  11. syslog2cef-0.1.2/syslogcef/mappings/fortinet.json +19 -0
  12. syslog2cef-0.1.2/syslogcef/mappings/sophos.json +18 -0
  13. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslogcef/normalizer.py +30 -6
  14. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslogcef/parsers.py +193 -10
  15. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslogcef/utils.py +17 -2
  16. syslog2cef-0.1.2/tests/test_adaptive.py +137 -0
  17. syslog2cef-0.1.2/tests/test_real_world.py +93 -0
  18. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/LICENSE +0 -0
  19. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/setup.cfg +0 -0
  20. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslog2cef.egg-info/dependency_links.txt +0 -0
  21. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslog2cef.egg-info/entry_points.txt +0 -0
  22. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslog2cef.egg-info/requires.txt +0 -0
  23. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslog2cef.egg-info/top_level.txt +0 -0
  24. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslogcef/__init__.py +0 -0
  25. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslogcef/__main__.py +0 -0
  26. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslogcef/mappings/cisco_asa.json +0 -0
  27. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslogcef/mappings/cisco_ios.json +0 -0
  28. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslogcef/mappings/f5.json +0 -0
  29. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslogcef/mappings/linux.json +0 -0
  30. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslogcef/mappings/vmware.json +0 -0
  31. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/tests/test_cef.py +0 -0
  32. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/tests/test_cli.py +0 -0
  33. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/tests/test_normalizer.py +0 -0
  34. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/tests/test_parsers.py +0 -0
  35. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/tests/test_parsers_extra.py +0 -0
  36. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/tests/test_pipeline.py +0 -0
  37. {syslog2cef-0.1.0 → syslog2cef-0.1.2}/tests/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: syslog2cef
3
- Version: 0.1.0
3
+ Version: 0.1.2
4
4
  Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
5
  Author-email: Tamir Suliman <allamiro@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/allamiro/syslogcef
@@ -17,6 +17,10 @@ Provides-Extra: test
17
17
  Requires-Dist: pytest>=7; extra == "test"
18
18
  Dynamic: license-file
19
19
 
20
+ <p align="center">
21
+ <img src="https://raw.githubusercontent.com/allamiro/syslogcef/main/docs/logo.svg" alt="syslogcef logo" width="540">
22
+ </p>
23
+
20
24
  # syslogcef
21
25
 
22
26
  syslogcef converts syslog events into ArcSight Common Event Format (CEF). It
@@ -58,11 +62,47 @@ cd syslogcef
58
62
  pip install .
59
63
  ```
60
64
 
61
- An RPM package with a systemd service is also available from the GitHub
62
- releases page; see "Running as a service" below.
65
+ The GitHub releases page also provides, for every version:
66
+
67
+ - An RPM package (Fedora/RHEL) with a systemd service — see "Running as a
68
+ Service" below.
69
+ - A Debian package (`syslogcef_X.Y.Z-1_all.deb`) with the same systemd
70
+ service: `sudo apt install ./syslogcef_X.Y.Z-1_all.deb`.
71
+ - An Alpine APK with an OpenRC service:
72
+ `apk add --allow-untrusted syslogcef-X.Y.Z-r0.apk` (or install the
73
+ signing key from [packaging/apk/syslogcef.rsa.pub](packaging/apk/syslogcef.rsa.pub)
74
+ into `/etc/apk/keys/` first to verify).
75
+ - A standalone executable (`syslogcef-X.Y.Z.pyz`) that runs on any system
76
+ with Python 3.9+: `chmod +x syslogcef-X.Y.Z.pyz && ./syslogcef-X.Y.Z.pyz`.
77
+ - A source zip and sdist/wheel files.
78
+
79
+ A multi-arch (amd64/arm64) container image is published to GitHub
80
+ Container Registry on each release:
81
+
82
+ ```bash
83
+ docker run -i ghcr.io/allamiro/syslogcef < /var/log/syslog
84
+ ```
85
+
86
+ Fedora, RHEL/Alma/Rocky 9 and 10, and CentOS Stream users can install
87
+ from the [COPR repository](https://copr.fedorainfracloud.org/coprs/allamiro/syslogcef/),
88
+ which rebuilds automatically from every commit:
89
+
90
+ ```bash
91
+ sudo dnf copr enable allamiro/syslogcef
92
+ sudo dnf install syslogcef
93
+ ```
94
+
95
+ Every asset ships with a detached GPG signature (`.asc`) and is listed in
96
+ a signed `SHA256SUMS` file; RPMs additionally carry embedded `rpmsign`
97
+ signatures. The public key is committed at
98
+ [packaging/rpm/RPM-GPG-KEY-syslogcef](packaging/rpm/RPM-GPG-KEY-syslogcef).
63
99
 
64
100
  ## Command Line Usage
65
101
 
102
+ <p align="center">
103
+ <img src="https://raw.githubusercontent.com/allamiro/syslogcef/main/docs/demo.svg" alt="Animated demo of syslogcef converting different formats" width="820">
104
+ </p>
105
+
66
106
  ```bash
67
107
  # Read from stdin, write CEF to stdout
68
108
  syslogcef < /var/log/syslog
@@ -159,7 +199,9 @@ for the full CEF extension dictionary.
159
199
 
160
200
  ## Running as a Service
161
201
 
162
- The RPM package installs a systemd unit and an environment file:
202
+ The RPM and Debian packages install a systemd unit and an environment
203
+ file (the Alpine APK installs the equivalent OpenRC service with its
204
+ configuration in `/etc/conf.d/syslogcef`):
163
205
 
164
206
  - `/etc/syslogcef/syslogcef.conf` — input file, output file, and extra
165
207
  arguments for the converter.
@@ -1,3 +1,7 @@
1
+ <p align="center">
2
+ <img src="https://raw.githubusercontent.com/allamiro/syslogcef/main/docs/logo.svg" alt="syslogcef logo" width="540">
3
+ </p>
4
+
1
5
  # syslogcef
2
6
 
3
7
  syslogcef converts syslog events into ArcSight Common Event Format (CEF). It
@@ -39,11 +43,47 @@ cd syslogcef
39
43
  pip install .
40
44
  ```
41
45
 
42
- An RPM package with a systemd service is also available from the GitHub
43
- releases page; see "Running as a service" below.
46
+ The GitHub releases page also provides, for every version:
47
+
48
+ - An RPM package (Fedora/RHEL) with a systemd service — see "Running as a
49
+ Service" below.
50
+ - A Debian package (`syslogcef_X.Y.Z-1_all.deb`) with the same systemd
51
+ service: `sudo apt install ./syslogcef_X.Y.Z-1_all.deb`.
52
+ - An Alpine APK with an OpenRC service:
53
+ `apk add --allow-untrusted syslogcef-X.Y.Z-r0.apk` (or install the
54
+ signing key from [packaging/apk/syslogcef.rsa.pub](packaging/apk/syslogcef.rsa.pub)
55
+ into `/etc/apk/keys/` first to verify).
56
+ - A standalone executable (`syslogcef-X.Y.Z.pyz`) that runs on any system
57
+ with Python 3.9+: `chmod +x syslogcef-X.Y.Z.pyz && ./syslogcef-X.Y.Z.pyz`.
58
+ - A source zip and sdist/wheel files.
59
+
60
+ A multi-arch (amd64/arm64) container image is published to GitHub
61
+ Container Registry on each release:
62
+
63
+ ```bash
64
+ docker run -i ghcr.io/allamiro/syslogcef < /var/log/syslog
65
+ ```
66
+
67
+ Fedora, RHEL/Alma/Rocky 9 and 10, and CentOS Stream users can install
68
+ from the [COPR repository](https://copr.fedorainfracloud.org/coprs/allamiro/syslogcef/),
69
+ which rebuilds automatically from every commit:
70
+
71
+ ```bash
72
+ sudo dnf copr enable allamiro/syslogcef
73
+ sudo dnf install syslogcef
74
+ ```
75
+
76
+ Every asset ships with a detached GPG signature (`.asc`) and is listed in
77
+ a signed `SHA256SUMS` file; RPMs additionally carry embedded `rpmsign`
78
+ signatures. The public key is committed at
79
+ [packaging/rpm/RPM-GPG-KEY-syslogcef](packaging/rpm/RPM-GPG-KEY-syslogcef).
44
80
 
45
81
  ## Command Line Usage
46
82
 
83
+ <p align="center">
84
+ <img src="https://raw.githubusercontent.com/allamiro/syslogcef/main/docs/demo.svg" alt="Animated demo of syslogcef converting different formats" width="820">
85
+ </p>
86
+
47
87
  ```bash
48
88
  # Read from stdin, write CEF to stdout
49
89
  syslogcef < /var/log/syslog
@@ -140,7 +180,9 @@ for the full CEF extension dictionary.
140
180
 
141
181
  ## Running as a Service
142
182
 
143
- The RPM package installs a systemd unit and an environment file:
183
+ The RPM and Debian packages install a systemd unit and an environment
184
+ file (the Alpine APK installs the equivalent OpenRC service with its
185
+ configuration in `/etc/conf.d/syslogcef`):
144
186
 
145
187
  - `/etc/syslogcef/syslogcef.conf` — input file, output file, and extra
146
188
  arguments for the converter.
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "syslog2cef"
7
- version = "0.1.0"
7
+ version = "0.1.2"
8
8
  description = "Convert RFC3164/RFC5424 syslog lines into ArcSight CEF"
9
9
  authors = [{name = "Tamir Suliman", email = "allamiro@gmail.com"}]
10
10
  readme = "README.md"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: syslog2cef
3
- Version: 0.1.0
3
+ Version: 0.1.2
4
4
  Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
5
  Author-email: Tamir Suliman <allamiro@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/allamiro/syslogcef
@@ -17,6 +17,10 @@ Provides-Extra: test
17
17
  Requires-Dist: pytest>=7; extra == "test"
18
18
  Dynamic: license-file
19
19
 
20
+ <p align="center">
21
+ <img src="https://raw.githubusercontent.com/allamiro/syslogcef/main/docs/logo.svg" alt="syslogcef logo" width="540">
22
+ </p>
23
+
20
24
  # syslogcef
21
25
 
22
26
  syslogcef converts syslog events into ArcSight Common Event Format (CEF). It
@@ -58,11 +62,47 @@ cd syslogcef
58
62
  pip install .
59
63
  ```
60
64
 
61
- An RPM package with a systemd service is also available from the GitHub
62
- releases page; see "Running as a service" below.
65
+ The GitHub releases page also provides, for every version:
66
+
67
+ - An RPM package (Fedora/RHEL) with a systemd service — see "Running as a
68
+ Service" below.
69
+ - A Debian package (`syslogcef_X.Y.Z-1_all.deb`) with the same systemd
70
+ service: `sudo apt install ./syslogcef_X.Y.Z-1_all.deb`.
71
+ - An Alpine APK with an OpenRC service:
72
+ `apk add --allow-untrusted syslogcef-X.Y.Z-r0.apk` (or install the
73
+ signing key from [packaging/apk/syslogcef.rsa.pub](packaging/apk/syslogcef.rsa.pub)
74
+ into `/etc/apk/keys/` first to verify).
75
+ - A standalone executable (`syslogcef-X.Y.Z.pyz`) that runs on any system
76
+ with Python 3.9+: `chmod +x syslogcef-X.Y.Z.pyz && ./syslogcef-X.Y.Z.pyz`.
77
+ - A source zip and sdist/wheel files.
78
+
79
+ A multi-arch (amd64/arm64) container image is published to GitHub
80
+ Container Registry on each release:
81
+
82
+ ```bash
83
+ docker run -i ghcr.io/allamiro/syslogcef < /var/log/syslog
84
+ ```
85
+
86
+ Fedora, RHEL/Alma/Rocky 9 and 10, and CentOS Stream users can install
87
+ from the [COPR repository](https://copr.fedorainfracloud.org/coprs/allamiro/syslogcef/),
88
+ which rebuilds automatically from every commit:
89
+
90
+ ```bash
91
+ sudo dnf copr enable allamiro/syslogcef
92
+ sudo dnf install syslogcef
93
+ ```
94
+
95
+ Every asset ships with a detached GPG signature (`.asc`) and is listed in
96
+ a signed `SHA256SUMS` file; RPMs additionally carry embedded `rpmsign`
97
+ signatures. The public key is committed at
98
+ [packaging/rpm/RPM-GPG-KEY-syslogcef](packaging/rpm/RPM-GPG-KEY-syslogcef).
63
99
 
64
100
  ## Command Line Usage
65
101
 
102
+ <p align="center">
103
+ <img src="https://raw.githubusercontent.com/allamiro/syslogcef/main/docs/demo.svg" alt="Animated demo of syslogcef converting different formats" width="820">
104
+ </p>
105
+
66
106
  ```bash
67
107
  # Read from stdin, write CEF to stdout
68
108
  syslogcef < /var/log/syslog
@@ -159,7 +199,9 @@ for the full CEF extension dictionary.
159
199
 
160
200
  ## Running as a Service
161
201
 
162
- The RPM package installs a systemd unit and an environment file:
202
+ The RPM and Debian packages install a systemd unit and an environment
203
+ file (the Alpine APK installs the equivalent OpenRC service with its
204
+ configuration in `/etc/conf.d/syslogcef`):
163
205
 
164
206
  - `/etc/syslogcef/syslogcef.conf` — input file, output file, and extra
165
207
  arguments for the converter.
@@ -3,6 +3,7 @@ README.md
3
3
  pyproject.toml
4
4
  ./syslogcef/__init__.py
5
5
  ./syslogcef/__main__.py
6
+ ./syslogcef/adaptive.py
6
7
  ./syslogcef/api.py
7
8
  ./syslogcef/cef.py
8
9
  ./syslogcef/cli.py
@@ -13,7 +14,9 @@ pyproject.toml
13
14
  ./syslogcef/mappings/cisco_asa.json
14
15
  ./syslogcef/mappings/cisco_ios.json
15
16
  ./syslogcef/mappings/f5.json
17
+ ./syslogcef/mappings/fortinet.json
16
18
  ./syslogcef/mappings/linux.json
19
+ ./syslogcef/mappings/sophos.json
17
20
  ./syslogcef/mappings/vmware.json
18
21
  syslog2cef.egg-info/PKG-INFO
19
22
  syslog2cef.egg-info/SOURCES.txt
@@ -23,6 +26,7 @@ syslog2cef.egg-info/requires.txt
23
26
  syslog2cef.egg-info/top_level.txt
24
27
  syslogcef/__init__.py
25
28
  syslogcef/__main__.py
29
+ syslogcef/adaptive.py
26
30
  syslogcef/api.py
27
31
  syslogcef/cef.py
28
32
  syslogcef/cli.py
@@ -33,12 +37,16 @@ syslogcef/mappings/__init__.py
33
37
  syslogcef/mappings/cisco_asa.json
34
38
  syslogcef/mappings/cisco_ios.json
35
39
  syslogcef/mappings/f5.json
40
+ syslogcef/mappings/fortinet.json
36
41
  syslogcef/mappings/linux.json
42
+ syslogcef/mappings/sophos.json
37
43
  syslogcef/mappings/vmware.json
44
+ tests/test_adaptive.py
38
45
  tests/test_cef.py
39
46
  tests/test_cli.py
40
47
  tests/test_normalizer.py
41
48
  tests/test_parsers.py
42
49
  tests/test_parsers_extra.py
43
50
  tests/test_pipeline.py
51
+ tests/test_real_world.py
44
52
  tests/test_utils.py
@@ -0,0 +1,197 @@
1
+ """Adaptive pattern detection for lines no known parser matches.
2
+
3
+ When every dedicated parser fails, this module analyzes the line: it
4
+ locates a timestamp using a library of datetime shapes, identifies a
5
+ plausible host token, and synthesizes a compiled regex for that line
6
+ layout. The synthesized pattern is cached by the line's shape signature,
7
+ so subsequent lines with the same layout parse with a single regex match
8
+ instead of a fresh analysis. Events parsed this way report
9
+ ``source_hint="adaptive"``.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import re
15
+ from datetime import datetime, timezone
16
+ from typing import Callable, Dict, Optional, Pattern, Tuple
17
+
18
+ from .utils import convert_pri, month_abbr_to_int, parse_iso8601, sanitize_message
19
+
20
+ PRI_RE = re.compile(r"^<(\d{1,3})>")
21
+
22
+ # Each entry: (name, regex source, converter(match, now) -> datetime)
23
+ def _iso(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
24
+ return parse_iso8601(match.group(0).replace(" ", "T", 1) if "T" not in match.group(0) else match.group(0))
25
+
26
+
27
+ def _epoch(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
28
+ return parse_iso8601(match.group(0))
29
+
30
+
31
+ def _mon_day(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
32
+ month = month_abbr_to_int(match.group("month"))
33
+ day = int(match.group("day"))
34
+ hour, minute, second = map(int, match.group("hms").split(":"))
35
+ year = int(match.group("year")) if match.groupdict().get("year") else (now or datetime.now(timezone.utc)).year
36
+ return datetime(year, month, day, hour, minute, second, tzinfo=timezone.utc)
37
+
38
+
39
+ def _ymd_slash(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
40
+ y, m, d = int(match.group("y")), int(match.group("m")), int(match.group("d"))
41
+ hour, minute, second = map(int, match.group("hms").split(":"))
42
+ return datetime(y, m, d, hour, minute, second, tzinfo=timezone.utc)
43
+
44
+
45
+ def _dmy_dash(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
46
+ d = int(match.group("d"))
47
+ month = month_abbr_to_int(match.group("month").title())
48
+ y = int(match.group("y"))
49
+ hour, minute, second = map(int, match.group("hms").split(":"))
50
+ return datetime(y, month, d, hour, minute, second, tzinfo=timezone.utc)
51
+
52
+
53
+ TIMESTAMP_LIB: Tuple[Tuple[str, str, Callable], ...] = (
54
+ ("iso", r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?", _iso),
55
+ ("mon_day_year", r"(?P<month>[A-Z][a-z]{2})\s+(?P<day>\d{1,2})\s+(?P<year>\d{4})\s+(?P<hms>\d{2}:\d{2}:\d{2})(?:\.\d+)?", _mon_day),
56
+ ("mon_day", r"(?P<month>[A-Z][a-z]{2})\s+(?P<day>\d{1,2})\s+(?P<hms>\d{2}:\d{2}:\d{2})(?:\.\d+)?", _mon_day),
57
+ ("ymd_slash", r"(?P<y>\d{4})/(?P<m>\d{2})/(?P<d>\d{2})[ T](?P<hms>\d{2}:\d{2}:\d{2})", _ymd_slash),
58
+ ("dmy_dash", r"(?P<d>\d{1,2})-(?P<month>[A-Za-z]{3})-(?P<y>\d{4})\s+(?P<hms>\d{2}:\d{2}:\d{2})", _dmy_dash),
59
+ ("epoch", r"\b1\d{9}(?:\d{6})?\b", _epoch),
60
+ )
61
+
62
+ HOST_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
63
+ _SKIP_HOST_TOKENS = {"error", "warning", "info", "debug", "notice", "kernel"}
64
+
65
+ _CACHE: Dict[str, Tuple[Pattern[str], str, Callable]] = {}
66
+ _CACHE_MAX = 256
67
+
68
+
69
+ def _signature(line: str) -> str:
70
+ """Reduce a line to a layout signature: token classes of the prefix.
71
+
72
+ Only the first few tokens participate — the layout (timestamp + host)
73
+ lives there, while later tokens are free-form message words that must
74
+ not fragment the cache.
75
+ """
76
+ tokens = line[:96].split()[:4]
77
+ classes = []
78
+ for token in tokens:
79
+ cls = re.sub(r"[A-Za-z]+", "A", token)
80
+ cls = re.sub(r"\d+", "9", cls)
81
+ classes.append(cls)
82
+ return " ".join(classes)
83
+
84
+
85
+ def clear_cache() -> None:
86
+ _CACHE.clear()
87
+
88
+
89
+ def cache_size() -> int:
90
+ return len(_CACHE)
91
+
92
+
93
+ def _build_event(line, pri, ts, host, msg):
94
+ from .parsers import ParsedEvent
95
+
96
+ facility, severity = convert_pri(pri)
97
+ return ParsedEvent(
98
+ pri=pri,
99
+ facility=facility,
100
+ severity=severity,
101
+ ts=ts,
102
+ ts_orig="",
103
+ host=host,
104
+ app=None,
105
+ pid=None,
106
+ msgid=None,
107
+ sd={},
108
+ msg=sanitize_message(msg),
109
+ raw=line,
110
+ source_hint="adaptive",
111
+ )
112
+
113
+
114
+ def adaptive_parse(line: str, *, now: Optional[datetime] = None):
115
+ """Parse an unknown-format line, learning its layout on first sight."""
116
+
117
+ raw = line
118
+ pri = None
119
+ pri_match = PRI_RE.match(line)
120
+ if pri_match:
121
+ pri = int(pri_match.group(1))
122
+ line = line[pri_match.end():]
123
+
124
+ sig = _signature(line)
125
+ cached = _CACHE.get(sig)
126
+ if cached is not None:
127
+ pattern, _name, conv = cached
128
+ match = pattern.match(line)
129
+ if match is not None:
130
+ try:
131
+ ts = conv(pattern_ts_match(pattern, match), now)
132
+ except (ValueError, KeyError):
133
+ ts = None
134
+ host = match.groupdict().get("adaptive_host")
135
+ msg = match.groupdict().get("adaptive_msg", "") or line
136
+ return _build_event(raw, pri, ts, host, msg)
137
+
138
+ # Analysis pass: find the earliest timestamp in the line head.
139
+ best = None
140
+ for name, src, conv in TIMESTAMP_LIB:
141
+ match = re.compile(src).search(line[:96])
142
+ if match and (best is None or match.start() < best[0].start()):
143
+ best = (match, name, src, conv)
144
+ if best is None:
145
+ return None
146
+ ts_match, name, src, conv = best
147
+ try:
148
+ ts = conv(ts_match, now)
149
+ except (ValueError, KeyError):
150
+ return None
151
+
152
+ # Host candidate: the first plausible token after the timestamp that is
153
+ # not a key=value pair or severity word.
154
+ rest = line[ts_match.end():].lstrip(" :,\t")
155
+ host = None
156
+ msg = rest
157
+ tokens = rest.split(None, 1)
158
+ if tokens:
159
+ candidate = tokens[0].rstrip(":,")
160
+ if (
161
+ "=" not in candidate
162
+ and HOST_RE.match(candidate)
163
+ and candidate.lower() not in _SKIP_HOST_TOKENS
164
+ and not candidate.isdigit()
165
+ ):
166
+ host = candidate
167
+ msg = tokens[1] if len(tokens) > 1 else ""
168
+
169
+ # Synthesize and cache a pattern for this layout.
170
+ prefix = line[: ts_match.start()]
171
+ parts = [re.escape(prefix), f"(?P<adaptive_ts>{src})", r"[\s:,]*"]
172
+ if host is not None:
173
+ parts.append(r"(?P<adaptive_host>\S+?):?(?:\s+|$)")
174
+ parts.append(r"(?P<adaptive_msg>.*)$")
175
+ try:
176
+ pattern = re.compile("".join(parts))
177
+ except re.error:
178
+ pattern = None
179
+ if pattern is not None and pattern.match(line):
180
+ if len(_CACHE) >= _CACHE_MAX:
181
+ _CACHE.pop(next(iter(_CACHE)))
182
+ _CACHE[sig] = (pattern, name, conv)
183
+
184
+ return _build_event(raw, pri, ts, host, msg)
185
+
186
+
187
+ def pattern_ts_match(pattern: Pattern[str], match: "re.Match[str]"):
188
+ """Re-match the timestamp portion so converters see their own groups."""
189
+ ts_text = match.group("adaptive_ts")
190
+ for _name, src, _conv in TIMESTAMP_LIB:
191
+ m = re.compile(src).fullmatch(ts_text)
192
+ if m:
193
+ return m
194
+ raise ValueError(f"cached timestamp no longer matches: {ts_text!r}")
195
+
196
+
197
+ __all__ = ["adaptive_parse", "cache_size", "clear_cache"]
@@ -9,13 +9,13 @@ from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional
9
9
 
10
10
  from .parsers import ParsedEvent, autodetect_and_parse
11
11
  from .normalizer import NormalizedEvent, normalize
12
- from .mappings import CISCO_ASA, CISCO_IOS, F5, LINUX, VMWARE
12
+ from .mappings import CISCO_ASA, CISCO_IOS, F5, FORTINET, LINUX, SOPHOS, VMWARE
13
13
  from .cef import CEFEvent, build_cef
14
14
 
15
15
  logger = logging.getLogger(__name__)
16
16
 
17
17
 
18
- @dataclass(slots=True)
18
+ @dataclass
19
19
  class ParseResult:
20
20
  """Container returned by :func:`parse_syslog`.
21
21
 
@@ -103,15 +103,23 @@ def convert_line(
103
103
  def _guess_mapping(event: NormalizedEvent) -> Mapping[str, Any]:
104
104
  msg_upper = event.msg.upper()
105
105
  app_upper = (event.app or "").upper()
106
+ event_code = event.kv.get("event_code", "")
106
107
 
107
- if "%ASA-" in msg_upper or "ASA" in app_upper:
108
+ if "%ASA-" in msg_upper or "ASA" in app_upper or event_code.startswith(("ASA-", "FTD-")):
108
109
  return CISCO_ASA
109
110
  if "IOS" in app_upper or "%IOS-" in msg_upper:
110
111
  return CISCO_IOS
111
112
  if "BIG-IP" in msg_upper or "F5" in app_upper:
112
113
  return F5
114
+ if "devname" in event.kv or "logid" in event.kv:
115
+ return FORTINET
116
+ if "log_id" in event.kv or event.kv.get("device") == "SFW":
117
+ return SOPHOS
113
118
  if "VMWARE" in msg_upper or "ESXI" in msg_upper or "VMWARE" in app_upper:
114
119
  return VMWARE
120
+ if event_code:
121
+ # Other %FAC-SEV-MNEMONIC codes are Cisco IOS style.
122
+ return CISCO_IOS
115
123
  return LINUX
116
124
 
117
125
 
@@ -34,7 +34,7 @@ DEFAULT_MAPPING = {
34
34
  }
35
35
 
36
36
 
37
- @dataclass(slots=True)
37
+ @dataclass
38
38
  class CEFEvent:
39
39
  header: Dict[str, str]
40
40
  extensions: Dict[str, str]
@@ -102,7 +102,7 @@ def main(argv: Optional[list[str]] = None) -> int:
102
102
  parser.add_argument("paths", nargs="*", type=Path, help="Input files (defaults to stdin)")
103
103
  parser.add_argument("-o", "--output", type=Path, help="Output file")
104
104
  parser.add_argument("-a", "--append", action="store_true", help="Append to --output instead of truncating it")
105
- parser.add_argument("--mode", choices=["rfc3164", "rfc5424", "rsyslog_json", "rsyslog_file", "journald_json", "journald_short", "journald_iso"], help="Parser mode override")
105
+ parser.add_argument("--mode", choices=["rfc3164", "rfc5424", "rsyslog_json", "rsyslog_file", "journald_json", "journald_short", "journald_iso", "cisco_seq", "iso_syslog", "kv"], help="Parser mode override")
106
106
  parser.add_argument("--mapping", type=str, help="Mapping JSON file")
107
107
  parser.add_argument("--tail", action="store_true", help="Follow file like tail -f")
108
108
  parser.add_argument("--multiprocess", action="store_true", help="Process lines using a process pool")
@@ -9,7 +9,9 @@ __all__ = [
9
9
  "CISCO_ASA",
10
10
  "CISCO_IOS",
11
11
  "F5",
12
+ "FORTINET",
12
13
  "LINUX",
14
+ "SOPHOS",
13
15
  "VMWARE",
14
16
  ]
15
17
 
@@ -22,7 +24,9 @@ def _load(name: str) -> Mapping[str, Any]:
22
24
  CISCO_ASA = _load("cisco_asa")
23
25
  CISCO_IOS = _load("cisco_ios")
24
26
  F5 = _load("f5")
27
+ FORTINET = _load("fortinet")
25
28
  LINUX = _load("linux")
29
+ SOPHOS = _load("sophos")
26
30
  VMWARE = _load("vmware")
27
31
 
28
32
 
@@ -31,7 +35,9 @@ def load_mapping(name: str) -> Mapping[str, Any]:
31
35
  "cisco_asa": CISCO_ASA,
32
36
  "cisco_ios": CISCO_IOS,
33
37
  "f5": F5,
38
+ "fortinet": FORTINET,
34
39
  "linux": LINUX,
40
+ "sophos": SOPHOS,
35
41
  "vmware": VMWARE,
36
42
  }
37
43
  try:
@@ -0,0 +1,19 @@
1
+ {
2
+ "deviceVendor": "Fortinet",
3
+ "deviceProduct": "FortiGate",
4
+ "deviceVersion": "auto",
5
+ "eventClassId": "fortinet.%(logid)s",
6
+ "name": "%(message_short)s",
7
+ "extensions": {
8
+ "dvchost": "%(devname)s",
9
+ "src": "%(srcip)s",
10
+ "dst": "%(dstip)s",
11
+ "spt": "%(srcport)s",
12
+ "dpt": "%(dstport)s",
13
+ "act": "%(action)s",
14
+ "app": "%(service)s",
15
+ "cat": "%(type)s",
16
+ "cs1Label": "rawEvent",
17
+ "cs1": "%(raw)s"
18
+ }
19
+ }
@@ -0,0 +1,18 @@
1
+ {
2
+ "deviceVendor": "Sophos",
3
+ "deviceProduct": "XG Firewall",
4
+ "deviceVersion": "auto",
5
+ "eventClassId": "sophos.%(log_id)s",
6
+ "name": "%(message_short)s",
7
+ "extensions": {
8
+ "dvchost": "%(device_name)s",
9
+ "src": "%(src_ip)s",
10
+ "dst": "%(dst_ip)s",
11
+ "spt": "%(src_port)s",
12
+ "dpt": "%(dst_port)s",
13
+ "suser": "%(user_name)s",
14
+ "cat": "%(log_type)s",
15
+ "cs1Label": "rawEvent",
16
+ "cs1": "%(raw)s"
17
+ }
18
+ }
@@ -1,17 +1,18 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  import logging
4
+ import re
4
5
  from dataclasses import dataclass, replace
5
6
  from datetime import datetime, timezone
6
7
  from typing import Any, Dict, Mapping, Optional
7
8
 
8
9
  from .parsers import ParsedEvent
9
- from .utils import ensure_tzaware, parse_key_value_pairs, sanitize_message
10
+ from .utils import ensure_tzaware, parse_key_value_pairs, sanitize_message, severity_from_word
10
11
 
11
12
  logger = logging.getLogger(__name__)
12
13
 
13
14
 
14
- @dataclass(slots=True)
15
+ @dataclass
15
16
  class NormalizedEvent(ParsedEvent):
16
17
  kv: Dict[str, str] = None
17
18
  extras: Dict[str, Any] = None
@@ -91,12 +92,35 @@ def _derive_common_fields(event: NormalizedEvent) -> None:
91
92
  if code:
92
93
  event.kv["event_code"] = code
93
94
 
95
+ # Cisco codes embed the severity digit (%SEC-6-IPACCESSLOGRP -> 6);
96
+ # use it when the line carried no PRI.
97
+ if event.severity is None:
98
+ code = event.kv.get("event_code", "")
99
+ sev_match = re.search(r"-(\d)-", code)
100
+ if sev_match:
101
+ event.severity = int(sev_match.group(1))
102
+
103
+ # Key=value formats often carry a textual level (level="notice",
104
+ # priority=Information); use it when no PRI was present.
105
+ if event.severity is None:
106
+ for key in ("level", "priority", "severity"):
107
+ word = event.kv.get(key)
108
+ if word:
109
+ sev = severity_from_word(word)
110
+ if sev is not None:
111
+ event.severity = sev
112
+ break
113
+
114
+
115
+ CISCO_CODE_RE = re.compile(r"%([A-Z][A-Z0-9_]*(?:-[A-Z0-9_]+)*-\d-[A-Z0-9_]+)")
116
+
94
117
 
95
118
  def _extract_event_code(msg: str) -> Optional[str]:
96
- # Cisco ASA style: %ASA-6-106100: message
97
- if msg.startswith("%") and ":" in msg:
98
- head = msg.split(":", 1)[0]
99
- return head.strip("%")
119
+ # Cisco style codes (%ASA-6-106100, %SEC-6-IPACCESSLOGRP, %LINK-3-UPDOWN)
120
+ # may appear at the start of the message or after an inline timestamp.
121
+ match = CISCO_CODE_RE.search(msg)
122
+ if match:
123
+ return match.group(1)
100
124
  return None
101
125
 
102
126
 
@@ -8,18 +8,20 @@ from datetime import datetime, timezone, timedelta
8
8
  from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional, Tuple
9
9
 
10
10
  from .utils import (
11
+ KEY_VALUE_RE,
11
12
  JournalJSONMapping,
12
13
  convert_pri,
13
14
  guess_hostname,
14
15
  month_abbr_to_int,
15
16
  parse_iso8601,
17
+ parse_key_value_pairs,
16
18
  sanitize_message,
17
19
  )
18
20
 
19
21
  logger = logging.getLogger(__name__)
20
22
 
21
23
 
22
- @dataclass(slots=True)
24
+ @dataclass
23
25
  class ParsedEvent:
24
26
  pri: Optional[int]
25
27
  facility: Optional[int]
@@ -39,9 +41,11 @@ class ParsedEvent:
39
41
  RFC3164_RE = re.compile(
40
42
  r"^(?:<(?P<pri>\d+)>)?"
41
43
  r"(?P<month>[A-Z][a-z]{2})\s+"
42
- r"(?P<day>\d{1,2})\s"
44
+ r"(?P<day>\d{1,2})\s+"
45
+ r"(?:(?P<year>\d{4})\s+)?"
43
46
  r"(?P<time>\d{2}:\d{2}:\d{2})\s"
44
- r"(?P<host>[^\s]+)\s"
47
+ r"(?P<host>[^\s]+?):?\s+"
48
+ r"(?::\s+)?"
45
49
  r"(?:(?P<tag>[\w\-/\.]+)(?:\[(?P<pid>[^\]]+)\])?:\s?)?"
46
50
  r"(?P<msg>.*)$"
47
51
  )
@@ -82,6 +86,44 @@ JOURNALCTL_ISO_RE = re.compile(
82
86
 
83
87
  KV_SPLIT_RE = re.compile(r"\s+|,|")
84
88
 
89
+ # Native Cisco console/buffer format: optional sequence number, optional
90
+ # clock-state marker (* = unsynced, . = drifting), timestamp with optional
91
+ # year/milliseconds/timezone, then the %CODE message.
92
+ CISCO_SEQ_RE = re.compile(
93
+ r"^(?:<(?P<pri>\d+)>)?"
94
+ r"(?:(?P<seq>\d+):\s+)?"
95
+ r"[*.]?"
96
+ r"(?P<month>[A-Z][a-z]{2})\s+"
97
+ r"(?P<day>\d{1,2})\s+"
98
+ r"(?:(?P<year>\d{4})\s+)?"
99
+ r"(?P<time>\d{2}:\d{2}:\d{2})(?:\.\d+)?"
100
+ r"(?:\s+[A-Z]{3,4})?:\s+"
101
+ r"(?P<msg>%[A-Z].*)$"
102
+ )
103
+
104
+ # ISO timestamp syslog without RFC5424 framing: <PRI>ISO host [tag[pid]:] msg
105
+ ISO_SYSLOG_RE = re.compile(
106
+ r"^(?:<(?P<pri>\d+)>)?"
107
+ r"(?P<timestamp>\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?)\s+"
108
+ r"(?P<host>[^\s]+?):?\s+"
109
+ r"(?:(?P<tag>[\w\-/\.]+)(?:\[(?P<pid>[^\]]+)\])?:\s?)?"
110
+ r"(?P<msg>.*)$"
111
+ )
112
+
113
+ # Key=value stream format (Fortinet FortiGate, Sophos XG, ...): the whole
114
+ # line is key=value pairs, optionally preceded by a PRI. Detection is done
115
+ # in _looks_like_kv_line (linear scan) rather than a repeated-group regex,
116
+ # which backtracks catastrophically on long non-matching lines.
117
+ PRI_PREFIX_RE = re.compile(r"^<(?P<pri>\d{1,3})>")
118
+
119
+
120
+ def _looks_like_kv_line(body: str) -> bool:
121
+ matches = list(KEY_VALUE_RE.finditer(body))
122
+ if len(matches) < 3:
123
+ return False
124
+ covered = sum(m.end() - m.start() for m in matches)
125
+ return covered >= 0.7 * len(body.strip())
126
+
85
127
 
86
128
  def _infer_timestamp(month: int, day: int, time_str: str, *, now: Optional[datetime]) -> datetime:
87
129
  if now is None:
@@ -129,7 +171,13 @@ def parse_rfc3164(line: str, *, now: Optional[datetime]) -> ParsedEvent | None:
129
171
  pri = int(gd["pri"]) if gd.get("pri") else None
130
172
  facility, severity = convert_pri(pri)
131
173
  month = month_abbr_to_int(gd["month"])
132
- ts = _infer_timestamp(month, int(gd["day"]), gd["time"], now=now)
174
+ hour, minute, second = map(int, gd["time"].split(":"))
175
+ if gd.get("year"):
176
+ ts = datetime(int(gd["year"]), month, int(gd["day"]), hour, minute, second, tzinfo=timezone.utc)
177
+ ts_orig = f"{gd['month']} {gd['day']} {gd['year']} {gd['time']}"
178
+ else:
179
+ ts = _infer_timestamp(month, int(gd["day"]), gd["time"], now=now)
180
+ ts_orig = f"{gd['month']} {gd['day']} {gd['time']}"
133
181
  app = gd.get("tag")
134
182
  msg = gd.get("msg", "")
135
183
  return ParsedEvent(
@@ -137,7 +185,7 @@ def parse_rfc3164(line: str, *, now: Optional[datetime]) -> ParsedEvent | None:
137
185
  facility=facility,
138
186
  severity=severity,
139
187
  ts=ts,
140
- ts_orig=f"{gd['month']} {gd['day']} {gd['time']}",
188
+ ts_orig=ts_orig,
141
189
  host=gd.get("host"),
142
190
  app=app,
143
191
  pid=gd.get("pid"),
@@ -312,6 +360,115 @@ def parse_journalctl_iso(line: str) -> ParsedEvent | None:
312
360
  )
313
361
 
314
362
 
363
+ def parse_cisco_seq(line: str, *, now: Optional[datetime]) -> ParsedEvent | None:
364
+ match = CISCO_SEQ_RE.match(line)
365
+ if not match:
366
+ return None
367
+ gd = match.groupdict()
368
+ pri = int(gd["pri"]) if gd.get("pri") else None
369
+ facility, severity = convert_pri(pri)
370
+ month = month_abbr_to_int(gd["month"])
371
+ hour, minute, second = map(int, gd["time"].split(":"))
372
+ if gd.get("year"):
373
+ ts = datetime(int(gd["year"]), month, int(gd["day"]), hour, minute, second, tzinfo=timezone.utc)
374
+ else:
375
+ ts = _infer_timestamp(month, int(gd["day"]), gd["time"], now=now)
376
+ sd = {"cisco.sequence": gd["seq"]} if gd.get("seq") else {}
377
+ return ParsedEvent(
378
+ pri=pri,
379
+ facility=facility,
380
+ severity=severity,
381
+ ts=ts,
382
+ ts_orig=f"{gd['month']} {gd['day']} {gd['time']}",
383
+ host=None,
384
+ app=None,
385
+ pid=None,
386
+ msgid=None,
387
+ sd=sd,
388
+ msg=gd.get("msg", ""),
389
+ raw=line,
390
+ source_hint="cisco",
391
+ )
392
+
393
+
394
+ def parse_iso_syslog(line: str) -> ParsedEvent | None:
395
+ match = ISO_SYSLOG_RE.match(line)
396
+ if not match:
397
+ return None
398
+ gd = match.groupdict()
399
+ pri = int(gd["pri"]) if gd.get("pri") else None
400
+ facility, severity = convert_pri(pri)
401
+ try:
402
+ ts = parse_iso8601(gd["timestamp"])
403
+ except ValueError:
404
+ return None
405
+ return ParsedEvent(
406
+ pri=pri,
407
+ facility=facility,
408
+ severity=severity,
409
+ ts=ts,
410
+ ts_orig=gd["timestamp"],
411
+ host=gd.get("host"),
412
+ app=gd.get("tag"),
413
+ pid=gd.get("pid"),
414
+ msgid=None,
415
+ sd={},
416
+ msg=gd.get("msg", ""),
417
+ raw=line,
418
+ source_hint="iso_syslog",
419
+ )
420
+
421
+
422
+ def parse_kv_stream(line: str) -> ParsedEvent | None:
423
+ pri = None
424
+ body = line
425
+ pri_match = PRI_PREFIX_RE.match(line)
426
+ if pri_match:
427
+ pri = int(pri_match.group("pri"))
428
+ body = line[pri_match.end():]
429
+ if not _looks_like_kv_line(body):
430
+ return None
431
+ facility, severity = convert_pri(pri)
432
+ pairs = parse_key_value_pairs(body)
433
+ if len(pairs) < 3:
434
+ return None
435
+
436
+ ts = None
437
+ if pairs.get("date") and pairs.get("time"):
438
+ try:
439
+ ts = parse_iso8601(f"{pairs['date']}T{pairs['time']}")
440
+ except ValueError:
441
+ ts = None
442
+ elif pairs.get("eventtime", "").isdigit():
443
+ try:
444
+ ts = parse_iso8601(pairs["eventtime"][:16])
445
+ except ValueError:
446
+ ts = None
447
+
448
+ host = (
449
+ pairs.get("devname")
450
+ or pairs.get("device_name")
451
+ or pairs.get("dvchost")
452
+ or pairs.get("devid")
453
+ or pairs.get("device_id")
454
+ )
455
+ return ParsedEvent(
456
+ pri=pri,
457
+ facility=facility,
458
+ severity=severity,
459
+ ts=ts,
460
+ ts_orig=f"{pairs.get('date', '')} {pairs.get('time', '')}".strip(),
461
+ host=host,
462
+ app=pairs.get("type") or pairs.get("log_type"),
463
+ pid=None,
464
+ msgid=pairs.get("logid") or pairs.get("log_id"),
465
+ sd={},
466
+ msg=body.strip(),
467
+ raw=line,
468
+ source_hint="kv",
469
+ )
470
+
471
+
315
472
  PARSERS = {
316
473
  "rfc3164": parse_rfc3164,
317
474
  "rfc5424": parse_rfc5424,
@@ -320,6 +477,9 @@ PARSERS = {
320
477
  "journald_json": parse_journalctl_json,
321
478
  "journald_short": parse_journalctl_short,
322
479
  "journald_iso": parse_journalctl_iso,
480
+ "cisco_seq": parse_cisco_seq,
481
+ "iso_syslog": parse_iso_syslog,
482
+ "kv": parse_kv_stream,
323
483
  }
324
484
 
325
485
 
@@ -330,6 +490,9 @@ def autodetect_and_parse(line: str, *, mode: Optional[str] = None, now: Optional
330
490
  ("rfc5424", lambda s: s.startswith("<") and ">" in s[:10]),
331
491
  ("rsyslog_file", lambda s: bool(RSYSLOG_FILE_RE.match(s))),
332
492
  ("journald_iso", lambda s: bool(JOURNALCTL_ISO_RE.match(s))),
493
+ ("iso_syslog", lambda s: bool(ISO_SYSLOG_RE.match(s))),
494
+ ("kv", lambda s: "=" in s[:80]),
495
+ ("cisco_seq", lambda s: bool(CISCO_SEQ_RE.match(s))),
333
496
  ("journald_short", lambda s: bool(JOURNALCTL_SHORT_RE.match(s))),
334
497
  ("rfc3164", lambda s: bool(RFC3164_RE.match(s))),
335
498
  ]
@@ -354,12 +517,32 @@ def autodetect_and_parse(line: str, *, mode: Optional[str] = None, now: Optional
354
517
  logger.exception("Parser %s failed for line", key)
355
518
  continue
356
519
 
357
- # Fallback to raw message if nothing matches.
520
+ # Nothing known matched: let the adaptive detector analyze the line and
521
+ # synthesize a reusable pattern before giving up.
522
+ from .adaptive import adaptive_parse
523
+
524
+ try:
525
+ adaptive_event = adaptive_parse(line, now=now)
526
+ except Exception:
527
+ logger.exception("Adaptive parser failed for line")
528
+ adaptive_event = None
529
+ if adaptive_event is not None:
530
+ return adaptive_event
531
+
532
+ # Fallback to raw message if nothing matches. A leading <PRI> is still
533
+ # honored so facility/severity survive for non-standard formats.
358
534
  logger.debug("Falling back to raw parser for line: %s", line)
535
+ pri = None
536
+ body = line
537
+ pri_match = re.match(r"^<(\d{1,3})>", line)
538
+ if pri_match:
539
+ pri = int(pri_match.group(1))
540
+ body = line[pri_match.end():]
541
+ facility, severity = convert_pri(pri)
359
542
  return ParsedEvent(
360
- pri=None,
361
- facility=None,
362
- severity=None,
543
+ pri=pri,
544
+ facility=facility,
545
+ severity=severity,
363
546
  ts=None,
364
547
  ts_orig="",
365
548
  host=guess_hostname(),
@@ -367,7 +550,7 @@ def autodetect_and_parse(line: str, *, mode: Optional[str] = None, now: Optional
367
550
  pid=None,
368
551
  msgid=None,
369
552
  sd={},
370
- msg=sanitize_message(line),
553
+ msg=sanitize_message(body),
371
554
  raw=line,
372
555
  source_hint="unknown",
373
556
  )
@@ -26,7 +26,18 @@ MONTHS = {
26
26
  "Dec": 12,
27
27
  }
28
28
 
29
- KEY_VALUE_RE = re.compile(r"(?P<key>[A-Za-z0-9_.-]+)=(?P<value>\S+)")
29
+ KEY_VALUE_RE = re.compile(r"(?P<key>[A-Za-z0-9_.-]+)=(?P<value>\"[^\"]*\"|'[^']*'|\S+)")
30
+
31
+ SEVERITY_WORDS = {
32
+ "emergency": 0, "emerg": 0, "panic": 0,
33
+ "alert": 1,
34
+ "critical": 2, "crit": 2,
35
+ "error": 3, "err": 3,
36
+ "warning": 4, "warn": 4,
37
+ "notice": 5,
38
+ "information": 6, "informational": 6, "info": 6,
39
+ "debug": 7,
40
+ }
30
41
  UTF8_REPLACEMENT_CHAR = "\uFFFD"
31
42
 
32
43
 
@@ -80,12 +91,16 @@ def parse_key_value_pairs(msg: str) -> Dict[str, str]:
80
91
  for match in KEY_VALUE_RE.finditer(msg):
81
92
  key = match.group("key")
82
93
  value = match.group("value")
83
- if value.startswith(('"', "'")) and value.endswith(('"', "'")):
94
+ if len(value) >= 2 and value[0] in "\"'" and value[-1] == value[0]:
84
95
  value = value[1:-1]
85
96
  pairs[key] = value
86
97
  return pairs
87
98
 
88
99
 
100
+ def severity_from_word(word: str) -> Optional[int]:
101
+ return SEVERITY_WORDS.get(word.strip().lower())
102
+
103
+
89
104
  def guess_hostname() -> str:
90
105
  try:
91
106
  return socket.gethostname()
@@ -0,0 +1,137 @@
1
+ from __future__ import annotations
2
+
3
+ """Tests for the new format parsers and the adaptive pattern detector."""
4
+
5
+ import pytest
6
+
7
+ from syslogcef import convert_line, normalize_event, parse_syslog
8
+ from syslogcef.adaptive import cache_size, clear_cache
9
+
10
+
11
+ def fields(cef: str) -> list[str]:
12
+ return cef.split("|")
13
+
14
+
15
+ # --- key=value stream formats -------------------------------------------------
16
+
17
+ FORTINET_LINE = '<189>date=2020-04-23 time=12:32:48 devname="testswitch3" devid="someotherrouteridagain" logid="0102043014" type="event" subtype="user" level="notice" vd="root" srcip=10.1.1.1 dstip=10.2.2.2 action="login"'
18
+ SOPHOS_LINE = '<30>device="SFW" date=2020-05-18 time=14:38:48 timezone="CEST" device_name="XG230" device_id=1234567890123456 log_id=041101618035 log_type="Anti-Spam" log_component="SMTP" log_subtype="Allowed" priority=Information src_ip="10.9.9.9" user_name="alice"'
19
+
20
+
21
+ def test_fortinet_kv_line():
22
+ ev = parse_syslog(FORTINET_LINE)
23
+ assert ev.source_hint == "kv"
24
+ assert ev.host == "testswitch3"
25
+ assert ev.pri == 189
26
+ assert ev.ts.year == 2020
27
+ cef = convert_line(FORTINET_LINE)
28
+ assert fields(cef)[1] == "Fortinet"
29
+ assert fields(cef)[4] == "fortinet.0102043014"
30
+ assert "src=10.1.1.1" in cef
31
+ assert "dvchost=testswitch3" in cef
32
+
33
+
34
+ def test_sophos_kv_line():
35
+ ev = parse_syslog(SOPHOS_LINE)
36
+ assert ev.source_hint == "kv"
37
+ assert ev.host == "XG230"
38
+ cef = convert_line(SOPHOS_LINE)
39
+ assert fields(cef)[1] == "Sophos"
40
+ assert fields(cef)[4] == "sophos.041101618035"
41
+ assert "suser=alice" in cef
42
+
43
+
44
+ def test_kv_severity_from_level_word_without_pri():
45
+ line = 'date=2020-04-23 time=12:32:48 devname="fw1" logid="0100000001" level="warning" msgtext="x"'
46
+ ev = normalize_event(parse_syslog(line))
47
+ assert ev.severity == 4
48
+
49
+
50
+ def test_quoted_kv_values_with_spaces():
51
+ line = '<30>device="SFW" date=2020-05-18 time=14:38:48 device_name="XG230" log_id=1 reason="Email has been accepted" x=1'
52
+ ev = normalize_event(parse_syslog(line))
53
+ assert ev.kv["reason"] == "Email has been accepted"
54
+
55
+
56
+ # --- native Cisco console formats ---------------------------------------------
57
+
58
+ def test_cisco_native_sequence_line():
59
+ line = "000123: Feb 8 04:00:47.272: %SEC-6-IPACCESSLOGP: list 100 denied tcp 10.1.1.1(1024) -> 10.2.2.2(80), 1 packet"
60
+ ev = parse_syslog(line)
61
+ assert ev.source_hint == "cisco"
62
+ assert ev.sd["cisco.sequence"] == "000123"
63
+ norm = normalize_event(ev)
64
+ assert norm.kv["event_code"] == "SEC-6-IPACCESSLOGP"
65
+ assert norm.severity == 6
66
+
67
+
68
+ def test_cisco_star_clock_line():
69
+ line = "*Mar 1 18:46:11.012: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36)"
70
+ ev = parse_syslog(line)
71
+ assert ev.source_hint == "cisco"
72
+ norm = normalize_event(ev)
73
+ assert norm.kv["event_code"] == "SYS-5-CONFIG_I"
74
+ assert norm.severity == 5
75
+ assert fields(convert_line(line))[2] == "IOS"
76
+
77
+
78
+ def test_cisco_native_with_year_and_timezone():
79
+ line = "Feb 8 2024 04:00:47 UTC: %LINK-3-UPDOWN: Interface GigabitEthernet0/1, changed state to down"
80
+ ev = parse_syslog(line)
81
+ assert ev.source_hint == "cisco"
82
+ assert ev.ts.year == 2024
83
+ assert normalize_event(ev).severity == 3
84
+
85
+
86
+ # --- ISO syslog without RFC5424 framing ---------------------------------------
87
+
88
+ def test_iso_timestamp_with_pri_and_tag():
89
+ line = "<13>2023-07-01T12:00:00Z web01 nginx[42]: GET /index.html"
90
+ ev = parse_syslog(line)
91
+ assert ev.source_hint == "iso_syslog"
92
+ assert ev.pri == 13
93
+ assert ev.host == "web01"
94
+ assert ev.app == "nginx"
95
+ assert ev.pid == "42"
96
+ assert ev.msg == "GET /index.html"
97
+
98
+
99
+ # --- adaptive detection -------------------------------------------------------
100
+
101
+ def test_adaptive_learns_unknown_layout_and_caches():
102
+ clear_cache()
103
+ line1 = "2020/04/23 12:32:48 fw-edge-01 session established peer=10.0.0.1"
104
+ line2 = "2020/04/24 09:10:11 fw-edge-02 session torn down peer=10.0.0.2"
105
+ ev1 = parse_syslog(line1)
106
+ assert ev1.source_hint == "adaptive"
107
+ assert ev1.host == "fw-edge-01"
108
+ assert ev1.ts.year == 2020 and ev1.ts.month == 4 and ev1.ts.day == 23
109
+ assert cache_size() == 1
110
+ ev2 = parse_syslog(line2)
111
+ assert ev2.source_hint == "adaptive"
112
+ assert ev2.host == "fw-edge-02"
113
+ assert cache_size() == 1 # reused the learned pattern
114
+
115
+
116
+ def test_adaptive_dmy_dash_timestamp():
117
+ line = "17-Apr-2020 14:08:08 core01 link flap detected"
118
+ ev = parse_syslog(line)
119
+ assert ev.source_hint == "adaptive"
120
+ assert ev.ts.year == 2020 and ev.ts.month == 4
121
+ assert ev.host == "core01"
122
+
123
+
124
+ def test_adaptive_preserves_pri():
125
+ line = "<86>2020/04/23 12:32:48 bastion sudo session opened"
126
+ ev = parse_syslog(line)
127
+ assert ev.source_hint == "adaptive"
128
+ assert ev.pri == 86
129
+ assert ev.severity == 6
130
+
131
+
132
+ def test_garbage_still_falls_back_to_unknown():
133
+ ev = parse_syslog("complete garbage with no timestamp at all ###")
134
+ assert ev.source_hint == "unknown"
135
+ ev2 = parse_syslog("<30>still garbage but with a pri")
136
+ assert ev2.source_hint == "unknown"
137
+ assert ev2.pri == 30
@@ -0,0 +1,93 @@
1
+ from __future__ import annotations
2
+
3
+ """Regression tests built from real sample lines in the Elastic Beats
4
+ filebeat module test corpus (x-pack/filebeat/module/*/test)."""
5
+
6
+ from syslogcef import convert_line, normalize_event, parse_syslog
7
+
8
+
9
+ def fields(cef: str) -> list[str]:
10
+ return cef.split("|")
11
+
12
+
13
+ def test_asa_with_year_and_tag():
14
+ line = "Oct 10 2018 12:34:56 localhost CiscoASA[999]: %ASA-6-305011: Built dynamic TCP translation from inside:172.31.98.44/1772 to outside:100.66.98.44/8256"
15
+ ev = parse_syslog(line)
16
+ assert ev.source_hint == "rfc3164"
17
+ assert ev.host == "localhost"
18
+ assert ev.app == "CiscoASA"
19
+ assert ev.pid == "999"
20
+ assert ev.ts.year == 2018
21
+ cef = convert_line(line)
22
+ assert fields(cef)[1] == "Cisco"
23
+ assert fields(cef)[4] == "asa.ASA-6-305011"
24
+
25
+
26
+ def test_asa_with_year_and_standalone_colon_host():
27
+ line = "Apr 17 2020 14:08:08 SNL-ASA-VPN-A01 : %ASA-6-302016: Teardown UDP connection 110577675 for Outside:10.123.123.123/53723(LOCAL\\Elastic) to Inside:10.233.123.123/53 duration 0:00:00 bytes 148"
28
+ ev = parse_syslog(line)
29
+ assert ev.source_hint == "rfc3164"
30
+ assert ev.host == "SNL-ASA-VPN-A01"
31
+ assert ev.ts.year == 2020
32
+ cef = convert_line(line)
33
+ assert fields(cef)[4] == "asa.ASA-6-302016"
34
+
35
+
36
+ def test_asa_host_with_trailing_colon():
37
+ line = "Oct 10 2019 10:21:36 localhost: %ASA-6-302021: Teardown ICMP connection for faddr 192.0.2.15/0 gaddr 192.0.2.134/57808 laddr 192.0.2.134/57808"
38
+ ev = parse_syslog(line)
39
+ assert ev.source_hint == "rfc3164"
40
+ assert ev.host == "localhost"
41
+ assert normalize_event(ev).kv["event_code"] == "ASA-6-302021"
42
+
43
+
44
+ def test_ios_sequence_number_line_gets_code_and_severity():
45
+ line = "Feb 8 04:00:48 198.51.100.2 585917: Feb 8 04:00:47.272: %SEC-6-IPACCESSLOGRP: list 177 denied igmp 198.51.100.197 -> 224.0.0.22, 1 packet"
46
+ ev = normalize_event(parse_syslog(line))
47
+ assert ev.kv["event_code"] == "SEC-6-IPACCESSLOGRP"
48
+ assert ev.severity == 6
49
+ cef = convert_line(line)
50
+ assert fields(cef)[1] == "Cisco"
51
+ assert fields(cef)[2] == "IOS"
52
+ assert fields(cef)[4] == "ios.SEC-6-IPACCESSLOGRP"
53
+ # syslog severity 6 maps to CEF severity 2 via the default map
54
+ assert fields(cef)[6] == "2"
55
+
56
+
57
+ def test_ftd_event_code_routes_to_asa_mapping():
58
+ line = "Apr 17 2020 14:00:00 firepower : %FTD-6-305012: Teardown dynamic TCP translation from Inside:10.1.1.1/54000 to Outside:198.51.100.1/54000 duration 0:00:30"
59
+ cef = convert_line(line)
60
+ assert fields(cef)[1] == "Cisco"
61
+ assert fields(cef)[4] == "asa.FTD-6-305012"
62
+
63
+
64
+ def test_pri_survives_sophos_kv_line():
65
+ # Sophos XG style: PRI prefix followed by key=value pairs, no syslog
66
+ # header. Parsed by the dedicated kv parser; PRI is preserved.
67
+ line = '<30>device="SFW" date=2020-05-18 time=14:38:48 timezone="CEST" device_name="XG230" log_type="Anti-Spam" src_ip="" dst_ip=""'
68
+ ev = parse_syslog(line)
69
+ assert ev.source_hint == "kv"
70
+ assert ev.pri == 30
71
+ assert ev.facility == 3
72
+ assert ev.severity == 6
73
+ assert not ev.msg.startswith("<30>")
74
+ cef = convert_line(line)
75
+ assert fields(cef)[6] == "2"
76
+
77
+
78
+ def test_pri_survives_raw_fallback():
79
+ line = "<30>completely nonstandard payload without structure"
80
+ ev = parse_syslog(line)
81
+ assert ev.source_hint == "unknown"
82
+ assert ev.pri == 30
83
+ assert ev.severity == 6
84
+ assert not ev.msg.startswith("<30>")
85
+
86
+
87
+ def test_classic_rfc3164_without_year_still_parses():
88
+ line = "<166>Jan 1 12:34:56 router1 sshd[123]: Accepted password for user"
89
+ ev = parse_syslog(line)
90
+ assert ev.source_hint == "rfc3164"
91
+ assert ev.host == "router1"
92
+ assert ev.app == "sshd"
93
+ assert ev.pid == "123"
File without changes
File without changes
File without changes
File without changes