syslog2cef 0.1.0__tar.gz → 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {syslog2cef-0.1.0/syslog2cef.egg-info → syslog2cef-0.1.2}/PKG-INFO +46 -4
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/README.md +45 -3
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/pyproject.toml +1 -1
- {syslog2cef-0.1.0 → syslog2cef-0.1.2/syslog2cef.egg-info}/PKG-INFO +46 -4
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslog2cef.egg-info/SOURCES.txt +8 -0
- syslog2cef-0.1.2/syslogcef/adaptive.py +197 -0
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslogcef/api.py +11 -3
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslogcef/cef.py +1 -1
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslogcef/cli.py +1 -1
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslogcef/mappings/__init__.py +6 -0
- syslog2cef-0.1.2/syslogcef/mappings/fortinet.json +19 -0
- syslog2cef-0.1.2/syslogcef/mappings/sophos.json +18 -0
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslogcef/normalizer.py +30 -6
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslogcef/parsers.py +193 -10
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslogcef/utils.py +17 -2
- syslog2cef-0.1.2/tests/test_adaptive.py +137 -0
- syslog2cef-0.1.2/tests/test_real_world.py +93 -0
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/LICENSE +0 -0
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/setup.cfg +0 -0
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslog2cef.egg-info/dependency_links.txt +0 -0
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslog2cef.egg-info/entry_points.txt +0 -0
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslog2cef.egg-info/requires.txt +0 -0
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslog2cef.egg-info/top_level.txt +0 -0
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslogcef/__init__.py +0 -0
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslogcef/__main__.py +0 -0
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslogcef/mappings/cisco_asa.json +0 -0
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslogcef/mappings/cisco_ios.json +0 -0
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslogcef/mappings/f5.json +0 -0
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslogcef/mappings/linux.json +0 -0
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/syslogcef/mappings/vmware.json +0 -0
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/tests/test_cef.py +0 -0
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/tests/test_cli.py +0 -0
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/tests/test_normalizer.py +0 -0
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/tests/test_parsers.py +0 -0
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/tests/test_parsers_extra.py +0 -0
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/tests/test_pipeline.py +0 -0
- {syslog2cef-0.1.0 → syslog2cef-0.1.2}/tests/test_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: syslog2cef
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.2
|
|
4
4
|
Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
|
|
5
5
|
Author-email: Tamir Suliman <allamiro@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/allamiro/syslogcef
|
|
@@ -17,6 +17,10 @@ Provides-Extra: test
|
|
|
17
17
|
Requires-Dist: pytest>=7; extra == "test"
|
|
18
18
|
Dynamic: license-file
|
|
19
19
|
|
|
20
|
+
<p align="center">
|
|
21
|
+
<img src="https://raw.githubusercontent.com/allamiro/syslogcef/main/docs/logo.svg" alt="syslogcef logo" width="540">
|
|
22
|
+
</p>
|
|
23
|
+
|
|
20
24
|
# syslogcef
|
|
21
25
|
|
|
22
26
|
syslogcef converts syslog events into ArcSight Common Event Format (CEF). It
|
|
@@ -58,11 +62,47 @@ cd syslogcef
|
|
|
58
62
|
pip install .
|
|
59
63
|
```
|
|
60
64
|
|
|
61
|
-
|
|
62
|
-
|
|
65
|
+
The GitHub releases page also provides, for every version:
|
|
66
|
+
|
|
67
|
+
- An RPM package (Fedora/RHEL) with a systemd service — see "Running as a
|
|
68
|
+
Service" below.
|
|
69
|
+
- A Debian package (`syslogcef_X.Y.Z-1_all.deb`) with the same systemd
|
|
70
|
+
service: `sudo apt install ./syslogcef_X.Y.Z-1_all.deb`.
|
|
71
|
+
- An Alpine APK with an OpenRC service:
|
|
72
|
+
`apk add --allow-untrusted syslogcef-X.Y.Z-r0.apk` (or install the
|
|
73
|
+
signing key from [packaging/apk/syslogcef.rsa.pub](packaging/apk/syslogcef.rsa.pub)
|
|
74
|
+
into `/etc/apk/keys/` first to verify).
|
|
75
|
+
- A standalone executable (`syslogcef-X.Y.Z.pyz`) that runs on any system
|
|
76
|
+
with Python 3.9+: `chmod +x syslogcef-X.Y.Z.pyz && ./syslogcef-X.Y.Z.pyz`.
|
|
77
|
+
- A source zip and sdist/wheel files.
|
|
78
|
+
|
|
79
|
+
A multi-arch (amd64/arm64) container image is published to GitHub
|
|
80
|
+
Container Registry on each release:
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
docker run -i ghcr.io/allamiro/syslogcef < /var/log/syslog
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Fedora, RHEL/Alma/Rocky 9 and 10, and CentOS Stream users can install
|
|
87
|
+
from the [COPR repository](https://copr.fedorainfracloud.org/coprs/allamiro/syslogcef/),
|
|
88
|
+
which rebuilds automatically from every commit:
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
sudo dnf copr enable allamiro/syslogcef
|
|
92
|
+
sudo dnf install syslogcef
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Every asset ships with a detached GPG signature (`.asc`) and is listed in
|
|
96
|
+
a signed `SHA256SUMS` file; RPMs additionally carry embedded `rpmsign`
|
|
97
|
+
signatures. The public key is committed at
|
|
98
|
+
[packaging/rpm/RPM-GPG-KEY-syslogcef](packaging/rpm/RPM-GPG-KEY-syslogcef).
|
|
63
99
|
|
|
64
100
|
## Command Line Usage
|
|
65
101
|
|
|
102
|
+
<p align="center">
|
|
103
|
+
<img src="https://raw.githubusercontent.com/allamiro/syslogcef/main/docs/demo.svg" alt="Animated demo of syslogcef converting different formats" width="820">
|
|
104
|
+
</p>
|
|
105
|
+
|
|
66
106
|
```bash
|
|
67
107
|
# Read from stdin, write CEF to stdout
|
|
68
108
|
syslogcef < /var/log/syslog
|
|
@@ -159,7 +199,9 @@ for the full CEF extension dictionary.
|
|
|
159
199
|
|
|
160
200
|
## Running as a Service
|
|
161
201
|
|
|
162
|
-
The RPM
|
|
202
|
+
The RPM and Debian packages install a systemd unit and an environment
|
|
203
|
+
file (the Alpine APK installs the equivalent OpenRC service with its
|
|
204
|
+
configuration in `/etc/conf.d/syslogcef`):
|
|
163
205
|
|
|
164
206
|
- `/etc/syslogcef/syslogcef.conf` — input file, output file, and extra
|
|
165
207
|
arguments for the converter.
|
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="https://raw.githubusercontent.com/allamiro/syslogcef/main/docs/logo.svg" alt="syslogcef logo" width="540">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
1
5
|
# syslogcef
|
|
2
6
|
|
|
3
7
|
syslogcef converts syslog events into ArcSight Common Event Format (CEF). It
|
|
@@ -39,11 +43,47 @@ cd syslogcef
|
|
|
39
43
|
pip install .
|
|
40
44
|
```
|
|
41
45
|
|
|
42
|
-
|
|
43
|
-
|
|
46
|
+
The GitHub releases page also provides, for every version:
|
|
47
|
+
|
|
48
|
+
- An RPM package (Fedora/RHEL) with a systemd service — see "Running as a
|
|
49
|
+
Service" below.
|
|
50
|
+
- A Debian package (`syslogcef_X.Y.Z-1_all.deb`) with the same systemd
|
|
51
|
+
service: `sudo apt install ./syslogcef_X.Y.Z-1_all.deb`.
|
|
52
|
+
- An Alpine APK with an OpenRC service:
|
|
53
|
+
`apk add --allow-untrusted syslogcef-X.Y.Z-r0.apk` (or install the
|
|
54
|
+
signing key from [packaging/apk/syslogcef.rsa.pub](packaging/apk/syslogcef.rsa.pub)
|
|
55
|
+
into `/etc/apk/keys/` first to verify).
|
|
56
|
+
- A standalone executable (`syslogcef-X.Y.Z.pyz`) that runs on any system
|
|
57
|
+
with Python 3.9+: `chmod +x syslogcef-X.Y.Z.pyz && ./syslogcef-X.Y.Z.pyz`.
|
|
58
|
+
- A source zip and sdist/wheel files.
|
|
59
|
+
|
|
60
|
+
A multi-arch (amd64/arm64) container image is published to GitHub
|
|
61
|
+
Container Registry on each release:
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
docker run -i ghcr.io/allamiro/syslogcef < /var/log/syslog
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Fedora, RHEL/Alma/Rocky 9 and 10, and CentOS Stream users can install
|
|
68
|
+
from the [COPR repository](https://copr.fedorainfracloud.org/coprs/allamiro/syslogcef/),
|
|
69
|
+
which rebuilds automatically from every commit:
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
sudo dnf copr enable allamiro/syslogcef
|
|
73
|
+
sudo dnf install syslogcef
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Every asset ships with a detached GPG signature (`.asc`) and is listed in
|
|
77
|
+
a signed `SHA256SUMS` file; RPMs additionally carry embedded `rpmsign`
|
|
78
|
+
signatures. The public key is committed at
|
|
79
|
+
[packaging/rpm/RPM-GPG-KEY-syslogcef](packaging/rpm/RPM-GPG-KEY-syslogcef).
|
|
44
80
|
|
|
45
81
|
## Command Line Usage
|
|
46
82
|
|
|
83
|
+
<p align="center">
|
|
84
|
+
<img src="https://raw.githubusercontent.com/allamiro/syslogcef/main/docs/demo.svg" alt="Animated demo of syslogcef converting different formats" width="820">
|
|
85
|
+
</p>
|
|
86
|
+
|
|
47
87
|
```bash
|
|
48
88
|
# Read from stdin, write CEF to stdout
|
|
49
89
|
syslogcef < /var/log/syslog
|
|
@@ -140,7 +180,9 @@ for the full CEF extension dictionary.
|
|
|
140
180
|
|
|
141
181
|
## Running as a Service
|
|
142
182
|
|
|
143
|
-
The RPM
|
|
183
|
+
The RPM and Debian packages install a systemd unit and an environment
|
|
184
|
+
file (the Alpine APK installs the equivalent OpenRC service with its
|
|
185
|
+
configuration in `/etc/conf.d/syslogcef`):
|
|
144
186
|
|
|
145
187
|
- `/etc/syslogcef/syslogcef.conf` — input file, output file, and extra
|
|
146
188
|
arguments for the converter.
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "syslog2cef"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.2"
|
|
8
8
|
description = "Convert RFC3164/RFC5424 syslog lines into ArcSight CEF"
|
|
9
9
|
authors = [{name = "Tamir Suliman", email = "allamiro@gmail.com"}]
|
|
10
10
|
readme = "README.md"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: syslog2cef
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.2
|
|
4
4
|
Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
|
|
5
5
|
Author-email: Tamir Suliman <allamiro@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/allamiro/syslogcef
|
|
@@ -17,6 +17,10 @@ Provides-Extra: test
|
|
|
17
17
|
Requires-Dist: pytest>=7; extra == "test"
|
|
18
18
|
Dynamic: license-file
|
|
19
19
|
|
|
20
|
+
<p align="center">
|
|
21
|
+
<img src="https://raw.githubusercontent.com/allamiro/syslogcef/main/docs/logo.svg" alt="syslogcef logo" width="540">
|
|
22
|
+
</p>
|
|
23
|
+
|
|
20
24
|
# syslogcef
|
|
21
25
|
|
|
22
26
|
syslogcef converts syslog events into ArcSight Common Event Format (CEF). It
|
|
@@ -58,11 +62,47 @@ cd syslogcef
|
|
|
58
62
|
pip install .
|
|
59
63
|
```
|
|
60
64
|
|
|
61
|
-
|
|
62
|
-
|
|
65
|
+
The GitHub releases page also provides, for every version:
|
|
66
|
+
|
|
67
|
+
- An RPM package (Fedora/RHEL) with a systemd service — see "Running as a
|
|
68
|
+
Service" below.
|
|
69
|
+
- A Debian package (`syslogcef_X.Y.Z-1_all.deb`) with the same systemd
|
|
70
|
+
service: `sudo apt install ./syslogcef_X.Y.Z-1_all.deb`.
|
|
71
|
+
- An Alpine APK with an OpenRC service:
|
|
72
|
+
`apk add --allow-untrusted syslogcef-X.Y.Z-r0.apk` (or install the
|
|
73
|
+
signing key from [packaging/apk/syslogcef.rsa.pub](packaging/apk/syslogcef.rsa.pub)
|
|
74
|
+
into `/etc/apk/keys/` first to verify).
|
|
75
|
+
- A standalone executable (`syslogcef-X.Y.Z.pyz`) that runs on any system
|
|
76
|
+
with Python 3.9+: `chmod +x syslogcef-X.Y.Z.pyz && ./syslogcef-X.Y.Z.pyz`.
|
|
77
|
+
- A source zip and sdist/wheel files.
|
|
78
|
+
|
|
79
|
+
A multi-arch (amd64/arm64) container image is published to GitHub
|
|
80
|
+
Container Registry on each release:
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
docker run -i ghcr.io/allamiro/syslogcef < /var/log/syslog
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Fedora, RHEL/Alma/Rocky 9 and 10, and CentOS Stream users can install
|
|
87
|
+
from the [COPR repository](https://copr.fedorainfracloud.org/coprs/allamiro/syslogcef/),
|
|
88
|
+
which rebuilds automatically from every commit:
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
sudo dnf copr enable allamiro/syslogcef
|
|
92
|
+
sudo dnf install syslogcef
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Every asset ships with a detached GPG signature (`.asc`) and is listed in
|
|
96
|
+
a signed `SHA256SUMS` file; RPMs additionally carry embedded `rpmsign`
|
|
97
|
+
signatures. The public key is committed at
|
|
98
|
+
[packaging/rpm/RPM-GPG-KEY-syslogcef](packaging/rpm/RPM-GPG-KEY-syslogcef).
|
|
63
99
|
|
|
64
100
|
## Command Line Usage
|
|
65
101
|
|
|
102
|
+
<p align="center">
|
|
103
|
+
<img src="https://raw.githubusercontent.com/allamiro/syslogcef/main/docs/demo.svg" alt="Animated demo of syslogcef converting different formats" width="820">
|
|
104
|
+
</p>
|
|
105
|
+
|
|
66
106
|
```bash
|
|
67
107
|
# Read from stdin, write CEF to stdout
|
|
68
108
|
syslogcef < /var/log/syslog
|
|
@@ -159,7 +199,9 @@ for the full CEF extension dictionary.
|
|
|
159
199
|
|
|
160
200
|
## Running as a Service
|
|
161
201
|
|
|
162
|
-
The RPM
|
|
202
|
+
The RPM and Debian packages install a systemd unit and an environment
|
|
203
|
+
file (the Alpine APK installs the equivalent OpenRC service with its
|
|
204
|
+
configuration in `/etc/conf.d/syslogcef`):
|
|
163
205
|
|
|
164
206
|
- `/etc/syslogcef/syslogcef.conf` — input file, output file, and extra
|
|
165
207
|
arguments for the converter.
|
|
@@ -3,6 +3,7 @@ README.md
|
|
|
3
3
|
pyproject.toml
|
|
4
4
|
./syslogcef/__init__.py
|
|
5
5
|
./syslogcef/__main__.py
|
|
6
|
+
./syslogcef/adaptive.py
|
|
6
7
|
./syslogcef/api.py
|
|
7
8
|
./syslogcef/cef.py
|
|
8
9
|
./syslogcef/cli.py
|
|
@@ -13,7 +14,9 @@ pyproject.toml
|
|
|
13
14
|
./syslogcef/mappings/cisco_asa.json
|
|
14
15
|
./syslogcef/mappings/cisco_ios.json
|
|
15
16
|
./syslogcef/mappings/f5.json
|
|
17
|
+
./syslogcef/mappings/fortinet.json
|
|
16
18
|
./syslogcef/mappings/linux.json
|
|
19
|
+
./syslogcef/mappings/sophos.json
|
|
17
20
|
./syslogcef/mappings/vmware.json
|
|
18
21
|
syslog2cef.egg-info/PKG-INFO
|
|
19
22
|
syslog2cef.egg-info/SOURCES.txt
|
|
@@ -23,6 +26,7 @@ syslog2cef.egg-info/requires.txt
|
|
|
23
26
|
syslog2cef.egg-info/top_level.txt
|
|
24
27
|
syslogcef/__init__.py
|
|
25
28
|
syslogcef/__main__.py
|
|
29
|
+
syslogcef/adaptive.py
|
|
26
30
|
syslogcef/api.py
|
|
27
31
|
syslogcef/cef.py
|
|
28
32
|
syslogcef/cli.py
|
|
@@ -33,12 +37,16 @@ syslogcef/mappings/__init__.py
|
|
|
33
37
|
syslogcef/mappings/cisco_asa.json
|
|
34
38
|
syslogcef/mappings/cisco_ios.json
|
|
35
39
|
syslogcef/mappings/f5.json
|
|
40
|
+
syslogcef/mappings/fortinet.json
|
|
36
41
|
syslogcef/mappings/linux.json
|
|
42
|
+
syslogcef/mappings/sophos.json
|
|
37
43
|
syslogcef/mappings/vmware.json
|
|
44
|
+
tests/test_adaptive.py
|
|
38
45
|
tests/test_cef.py
|
|
39
46
|
tests/test_cli.py
|
|
40
47
|
tests/test_normalizer.py
|
|
41
48
|
tests/test_parsers.py
|
|
42
49
|
tests/test_parsers_extra.py
|
|
43
50
|
tests/test_pipeline.py
|
|
51
|
+
tests/test_real_world.py
|
|
44
52
|
tests/test_utils.py
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
"""Adaptive pattern detection for lines no known parser matches.
|
|
2
|
+
|
|
3
|
+
When every dedicated parser fails, this module analyzes the line: it
|
|
4
|
+
locates a timestamp using a library of datetime shapes, identifies a
|
|
5
|
+
plausible host token, and synthesizes a compiled regex for that line
|
|
6
|
+
layout. The synthesized pattern is cached by the line's shape signature,
|
|
7
|
+
so subsequent lines with the same layout parse with a single regex match
|
|
8
|
+
instead of a fresh analysis. Events parsed this way report
|
|
9
|
+
``source_hint="adaptive"``.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import re
|
|
15
|
+
from datetime import datetime, timezone
|
|
16
|
+
from typing import Callable, Dict, Optional, Pattern, Tuple
|
|
17
|
+
|
|
18
|
+
from .utils import convert_pri, month_abbr_to_int, parse_iso8601, sanitize_message
|
|
19
|
+
|
|
20
|
+
PRI_RE = re.compile(r"^<(\d{1,3})>")
|
|
21
|
+
|
|
22
|
+
# Each entry: (name, regex source, converter(match, now) -> datetime)
|
|
23
|
+
def _iso(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
|
|
24
|
+
return parse_iso8601(match.group(0).replace(" ", "T", 1) if "T" not in match.group(0) else match.group(0))
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _epoch(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
|
|
28
|
+
return parse_iso8601(match.group(0))
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _mon_day(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
|
|
32
|
+
month = month_abbr_to_int(match.group("month"))
|
|
33
|
+
day = int(match.group("day"))
|
|
34
|
+
hour, minute, second = map(int, match.group("hms").split(":"))
|
|
35
|
+
year = int(match.group("year")) if match.groupdict().get("year") else (now or datetime.now(timezone.utc)).year
|
|
36
|
+
return datetime(year, month, day, hour, minute, second, tzinfo=timezone.utc)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _ymd_slash(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
|
|
40
|
+
y, m, d = int(match.group("y")), int(match.group("m")), int(match.group("d"))
|
|
41
|
+
hour, minute, second = map(int, match.group("hms").split(":"))
|
|
42
|
+
return datetime(y, m, d, hour, minute, second, tzinfo=timezone.utc)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _dmy_dash(match: "re.Match[str]", now: Optional[datetime]) -> datetime:
|
|
46
|
+
d = int(match.group("d"))
|
|
47
|
+
month = month_abbr_to_int(match.group("month").title())
|
|
48
|
+
y = int(match.group("y"))
|
|
49
|
+
hour, minute, second = map(int, match.group("hms").split(":"))
|
|
50
|
+
return datetime(y, month, d, hour, minute, second, tzinfo=timezone.utc)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
TIMESTAMP_LIB: Tuple[Tuple[str, str, Callable], ...] = (
|
|
54
|
+
("iso", r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?", _iso),
|
|
55
|
+
("mon_day_year", r"(?P<month>[A-Z][a-z]{2})\s+(?P<day>\d{1,2})\s+(?P<year>\d{4})\s+(?P<hms>\d{2}:\d{2}:\d{2})(?:\.\d+)?", _mon_day),
|
|
56
|
+
("mon_day", r"(?P<month>[A-Z][a-z]{2})\s+(?P<day>\d{1,2})\s+(?P<hms>\d{2}:\d{2}:\d{2})(?:\.\d+)?", _mon_day),
|
|
57
|
+
("ymd_slash", r"(?P<y>\d{4})/(?P<m>\d{2})/(?P<d>\d{2})[ T](?P<hms>\d{2}:\d{2}:\d{2})", _ymd_slash),
|
|
58
|
+
("dmy_dash", r"(?P<d>\d{1,2})-(?P<month>[A-Za-z]{3})-(?P<y>\d{4})\s+(?P<hms>\d{2}:\d{2}:\d{2})", _dmy_dash),
|
|
59
|
+
("epoch", r"\b1\d{9}(?:\d{6})?\b", _epoch),
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
HOST_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
|
|
63
|
+
_SKIP_HOST_TOKENS = {"error", "warning", "info", "debug", "notice", "kernel"}
|
|
64
|
+
|
|
65
|
+
_CACHE: Dict[str, Tuple[Pattern[str], str, Callable]] = {}
|
|
66
|
+
_CACHE_MAX = 256
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _signature(line: str) -> str:
|
|
70
|
+
"""Reduce a line to a layout signature: token classes of the prefix.
|
|
71
|
+
|
|
72
|
+
Only the first few tokens participate — the layout (timestamp + host)
|
|
73
|
+
lives there, while later tokens are free-form message words that must
|
|
74
|
+
not fragment the cache.
|
|
75
|
+
"""
|
|
76
|
+
tokens = line[:96].split()[:4]
|
|
77
|
+
classes = []
|
|
78
|
+
for token in tokens:
|
|
79
|
+
cls = re.sub(r"[A-Za-z]+", "A", token)
|
|
80
|
+
cls = re.sub(r"\d+", "9", cls)
|
|
81
|
+
classes.append(cls)
|
|
82
|
+
return " ".join(classes)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def clear_cache() -> None:
|
|
86
|
+
_CACHE.clear()
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def cache_size() -> int:
|
|
90
|
+
return len(_CACHE)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _build_event(line, pri, ts, host, msg):
|
|
94
|
+
from .parsers import ParsedEvent
|
|
95
|
+
|
|
96
|
+
facility, severity = convert_pri(pri)
|
|
97
|
+
return ParsedEvent(
|
|
98
|
+
pri=pri,
|
|
99
|
+
facility=facility,
|
|
100
|
+
severity=severity,
|
|
101
|
+
ts=ts,
|
|
102
|
+
ts_orig="",
|
|
103
|
+
host=host,
|
|
104
|
+
app=None,
|
|
105
|
+
pid=None,
|
|
106
|
+
msgid=None,
|
|
107
|
+
sd={},
|
|
108
|
+
msg=sanitize_message(msg),
|
|
109
|
+
raw=line,
|
|
110
|
+
source_hint="adaptive",
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def adaptive_parse(line: str, *, now: Optional[datetime] = None):
|
|
115
|
+
"""Parse an unknown-format line, learning its layout on first sight."""
|
|
116
|
+
|
|
117
|
+
raw = line
|
|
118
|
+
pri = None
|
|
119
|
+
pri_match = PRI_RE.match(line)
|
|
120
|
+
if pri_match:
|
|
121
|
+
pri = int(pri_match.group(1))
|
|
122
|
+
line = line[pri_match.end():]
|
|
123
|
+
|
|
124
|
+
sig = _signature(line)
|
|
125
|
+
cached = _CACHE.get(sig)
|
|
126
|
+
if cached is not None:
|
|
127
|
+
pattern, _name, conv = cached
|
|
128
|
+
match = pattern.match(line)
|
|
129
|
+
if match is not None:
|
|
130
|
+
try:
|
|
131
|
+
ts = conv(pattern_ts_match(pattern, match), now)
|
|
132
|
+
except (ValueError, KeyError):
|
|
133
|
+
ts = None
|
|
134
|
+
host = match.groupdict().get("adaptive_host")
|
|
135
|
+
msg = match.groupdict().get("adaptive_msg", "") or line
|
|
136
|
+
return _build_event(raw, pri, ts, host, msg)
|
|
137
|
+
|
|
138
|
+
# Analysis pass: find the earliest timestamp in the line head.
|
|
139
|
+
best = None
|
|
140
|
+
for name, src, conv in TIMESTAMP_LIB:
|
|
141
|
+
match = re.compile(src).search(line[:96])
|
|
142
|
+
if match and (best is None or match.start() < best[0].start()):
|
|
143
|
+
best = (match, name, src, conv)
|
|
144
|
+
if best is None:
|
|
145
|
+
return None
|
|
146
|
+
ts_match, name, src, conv = best
|
|
147
|
+
try:
|
|
148
|
+
ts = conv(ts_match, now)
|
|
149
|
+
except (ValueError, KeyError):
|
|
150
|
+
return None
|
|
151
|
+
|
|
152
|
+
# Host candidate: the first plausible token after the timestamp that is
|
|
153
|
+
# not a key=value pair or severity word.
|
|
154
|
+
rest = line[ts_match.end():].lstrip(" :,\t")
|
|
155
|
+
host = None
|
|
156
|
+
msg = rest
|
|
157
|
+
tokens = rest.split(None, 1)
|
|
158
|
+
if tokens:
|
|
159
|
+
candidate = tokens[0].rstrip(":,")
|
|
160
|
+
if (
|
|
161
|
+
"=" not in candidate
|
|
162
|
+
and HOST_RE.match(candidate)
|
|
163
|
+
and candidate.lower() not in _SKIP_HOST_TOKENS
|
|
164
|
+
and not candidate.isdigit()
|
|
165
|
+
):
|
|
166
|
+
host = candidate
|
|
167
|
+
msg = tokens[1] if len(tokens) > 1 else ""
|
|
168
|
+
|
|
169
|
+
# Synthesize and cache a pattern for this layout.
|
|
170
|
+
prefix = line[: ts_match.start()]
|
|
171
|
+
parts = [re.escape(prefix), f"(?P<adaptive_ts>{src})", r"[\s:,]*"]
|
|
172
|
+
if host is not None:
|
|
173
|
+
parts.append(r"(?P<adaptive_host>\S+?):?(?:\s+|$)")
|
|
174
|
+
parts.append(r"(?P<adaptive_msg>.*)$")
|
|
175
|
+
try:
|
|
176
|
+
pattern = re.compile("".join(parts))
|
|
177
|
+
except re.error:
|
|
178
|
+
pattern = None
|
|
179
|
+
if pattern is not None and pattern.match(line):
|
|
180
|
+
if len(_CACHE) >= _CACHE_MAX:
|
|
181
|
+
_CACHE.pop(next(iter(_CACHE)))
|
|
182
|
+
_CACHE[sig] = (pattern, name, conv)
|
|
183
|
+
|
|
184
|
+
return _build_event(raw, pri, ts, host, msg)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def pattern_ts_match(pattern: Pattern[str], match: "re.Match[str]"):
|
|
188
|
+
"""Re-match the timestamp portion so converters see their own groups."""
|
|
189
|
+
ts_text = match.group("adaptive_ts")
|
|
190
|
+
for _name, src, _conv in TIMESTAMP_LIB:
|
|
191
|
+
m = re.compile(src).fullmatch(ts_text)
|
|
192
|
+
if m:
|
|
193
|
+
return m
|
|
194
|
+
raise ValueError(f"cached timestamp no longer matches: {ts_text!r}")
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
__all__ = ["adaptive_parse", "cache_size", "clear_cache"]
|
|
@@ -9,13 +9,13 @@ from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional
|
|
|
9
9
|
|
|
10
10
|
from .parsers import ParsedEvent, autodetect_and_parse
|
|
11
11
|
from .normalizer import NormalizedEvent, normalize
|
|
12
|
-
from .mappings import CISCO_ASA, CISCO_IOS, F5, LINUX, VMWARE
|
|
12
|
+
from .mappings import CISCO_ASA, CISCO_IOS, F5, FORTINET, LINUX, SOPHOS, VMWARE
|
|
13
13
|
from .cef import CEFEvent, build_cef
|
|
14
14
|
|
|
15
15
|
logger = logging.getLogger(__name__)
|
|
16
16
|
|
|
17
17
|
|
|
18
|
-
@dataclass
|
|
18
|
+
@dataclass
|
|
19
19
|
class ParseResult:
|
|
20
20
|
"""Container returned by :func:`parse_syslog`.
|
|
21
21
|
|
|
@@ -103,15 +103,23 @@ def convert_line(
|
|
|
103
103
|
def _guess_mapping(event: NormalizedEvent) -> Mapping[str, Any]:
|
|
104
104
|
msg_upper = event.msg.upper()
|
|
105
105
|
app_upper = (event.app or "").upper()
|
|
106
|
+
event_code = event.kv.get("event_code", "")
|
|
106
107
|
|
|
107
|
-
if "%ASA-" in msg_upper or "ASA" in app_upper:
|
|
108
|
+
if "%ASA-" in msg_upper or "ASA" in app_upper or event_code.startswith(("ASA-", "FTD-")):
|
|
108
109
|
return CISCO_ASA
|
|
109
110
|
if "IOS" in app_upper or "%IOS-" in msg_upper:
|
|
110
111
|
return CISCO_IOS
|
|
111
112
|
if "BIG-IP" in msg_upper or "F5" in app_upper:
|
|
112
113
|
return F5
|
|
114
|
+
if "devname" in event.kv or "logid" in event.kv:
|
|
115
|
+
return FORTINET
|
|
116
|
+
if "log_id" in event.kv or event.kv.get("device") == "SFW":
|
|
117
|
+
return SOPHOS
|
|
113
118
|
if "VMWARE" in msg_upper or "ESXI" in msg_upper or "VMWARE" in app_upper:
|
|
114
119
|
return VMWARE
|
|
120
|
+
if event_code:
|
|
121
|
+
# Other %FAC-SEV-MNEMONIC codes are Cisco IOS style.
|
|
122
|
+
return CISCO_IOS
|
|
115
123
|
return LINUX
|
|
116
124
|
|
|
117
125
|
|
|
@@ -102,7 +102,7 @@ def main(argv: Optional[list[str]] = None) -> int:
|
|
|
102
102
|
parser.add_argument("paths", nargs="*", type=Path, help="Input files (defaults to stdin)")
|
|
103
103
|
parser.add_argument("-o", "--output", type=Path, help="Output file")
|
|
104
104
|
parser.add_argument("-a", "--append", action="store_true", help="Append to --output instead of truncating it")
|
|
105
|
-
parser.add_argument("--mode", choices=["rfc3164", "rfc5424", "rsyslog_json", "rsyslog_file", "journald_json", "journald_short", "journald_iso"], help="Parser mode override")
|
|
105
|
+
parser.add_argument("--mode", choices=["rfc3164", "rfc5424", "rsyslog_json", "rsyslog_file", "journald_json", "journald_short", "journald_iso", "cisco_seq", "iso_syslog", "kv"], help="Parser mode override")
|
|
106
106
|
parser.add_argument("--mapping", type=str, help="Mapping JSON file")
|
|
107
107
|
parser.add_argument("--tail", action="store_true", help="Follow file like tail -f")
|
|
108
108
|
parser.add_argument("--multiprocess", action="store_true", help="Process lines using a process pool")
|
|
@@ -9,7 +9,9 @@ __all__ = [
|
|
|
9
9
|
"CISCO_ASA",
|
|
10
10
|
"CISCO_IOS",
|
|
11
11
|
"F5",
|
|
12
|
+
"FORTINET",
|
|
12
13
|
"LINUX",
|
|
14
|
+
"SOPHOS",
|
|
13
15
|
"VMWARE",
|
|
14
16
|
]
|
|
15
17
|
|
|
@@ -22,7 +24,9 @@ def _load(name: str) -> Mapping[str, Any]:
|
|
|
22
24
|
CISCO_ASA = _load("cisco_asa")
|
|
23
25
|
CISCO_IOS = _load("cisco_ios")
|
|
24
26
|
F5 = _load("f5")
|
|
27
|
+
FORTINET = _load("fortinet")
|
|
25
28
|
LINUX = _load("linux")
|
|
29
|
+
SOPHOS = _load("sophos")
|
|
26
30
|
VMWARE = _load("vmware")
|
|
27
31
|
|
|
28
32
|
|
|
@@ -31,7 +35,9 @@ def load_mapping(name: str) -> Mapping[str, Any]:
|
|
|
31
35
|
"cisco_asa": CISCO_ASA,
|
|
32
36
|
"cisco_ios": CISCO_IOS,
|
|
33
37
|
"f5": F5,
|
|
38
|
+
"fortinet": FORTINET,
|
|
34
39
|
"linux": LINUX,
|
|
40
|
+
"sophos": SOPHOS,
|
|
35
41
|
"vmware": VMWARE,
|
|
36
42
|
}
|
|
37
43
|
try:
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
{
|
|
2
|
+
"deviceVendor": "Fortinet",
|
|
3
|
+
"deviceProduct": "FortiGate",
|
|
4
|
+
"deviceVersion": "auto",
|
|
5
|
+
"eventClassId": "fortinet.%(logid)s",
|
|
6
|
+
"name": "%(message_short)s",
|
|
7
|
+
"extensions": {
|
|
8
|
+
"dvchost": "%(devname)s",
|
|
9
|
+
"src": "%(srcip)s",
|
|
10
|
+
"dst": "%(dstip)s",
|
|
11
|
+
"spt": "%(srcport)s",
|
|
12
|
+
"dpt": "%(dstport)s",
|
|
13
|
+
"act": "%(action)s",
|
|
14
|
+
"app": "%(service)s",
|
|
15
|
+
"cat": "%(type)s",
|
|
16
|
+
"cs1Label": "rawEvent",
|
|
17
|
+
"cs1": "%(raw)s"
|
|
18
|
+
}
|
|
19
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
{
|
|
2
|
+
"deviceVendor": "Sophos",
|
|
3
|
+
"deviceProduct": "XG Firewall",
|
|
4
|
+
"deviceVersion": "auto",
|
|
5
|
+
"eventClassId": "sophos.%(log_id)s",
|
|
6
|
+
"name": "%(message_short)s",
|
|
7
|
+
"extensions": {
|
|
8
|
+
"dvchost": "%(device_name)s",
|
|
9
|
+
"src": "%(src_ip)s",
|
|
10
|
+
"dst": "%(dst_ip)s",
|
|
11
|
+
"spt": "%(src_port)s",
|
|
12
|
+
"dpt": "%(dst_port)s",
|
|
13
|
+
"suser": "%(user_name)s",
|
|
14
|
+
"cat": "%(log_type)s",
|
|
15
|
+
"cs1Label": "rawEvent",
|
|
16
|
+
"cs1": "%(raw)s"
|
|
17
|
+
}
|
|
18
|
+
}
|
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
|
+
import re
|
|
4
5
|
from dataclasses import dataclass, replace
|
|
5
6
|
from datetime import datetime, timezone
|
|
6
7
|
from typing import Any, Dict, Mapping, Optional
|
|
7
8
|
|
|
8
9
|
from .parsers import ParsedEvent
|
|
9
|
-
from .utils import ensure_tzaware, parse_key_value_pairs, sanitize_message
|
|
10
|
+
from .utils import ensure_tzaware, parse_key_value_pairs, sanitize_message, severity_from_word
|
|
10
11
|
|
|
11
12
|
logger = logging.getLogger(__name__)
|
|
12
13
|
|
|
13
14
|
|
|
14
|
-
@dataclass
|
|
15
|
+
@dataclass
|
|
15
16
|
class NormalizedEvent(ParsedEvent):
|
|
16
17
|
kv: Dict[str, str] = None
|
|
17
18
|
extras: Dict[str, Any] = None
|
|
@@ -91,12 +92,35 @@ def _derive_common_fields(event: NormalizedEvent) -> None:
|
|
|
91
92
|
if code:
|
|
92
93
|
event.kv["event_code"] = code
|
|
93
94
|
|
|
95
|
+
# Cisco codes embed the severity digit (%SEC-6-IPACCESSLOGRP -> 6);
|
|
96
|
+
# use it when the line carried no PRI.
|
|
97
|
+
if event.severity is None:
|
|
98
|
+
code = event.kv.get("event_code", "")
|
|
99
|
+
sev_match = re.search(r"-(\d)-", code)
|
|
100
|
+
if sev_match:
|
|
101
|
+
event.severity = int(sev_match.group(1))
|
|
102
|
+
|
|
103
|
+
# Key=value formats often carry a textual level (level="notice",
|
|
104
|
+
# priority=Information); use it when no PRI was present.
|
|
105
|
+
if event.severity is None:
|
|
106
|
+
for key in ("level", "priority", "severity"):
|
|
107
|
+
word = event.kv.get(key)
|
|
108
|
+
if word:
|
|
109
|
+
sev = severity_from_word(word)
|
|
110
|
+
if sev is not None:
|
|
111
|
+
event.severity = sev
|
|
112
|
+
break
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
CISCO_CODE_RE = re.compile(r"%([A-Z][A-Z0-9_]*(?:-[A-Z0-9_]+)*-\d-[A-Z0-9_]+)")
|
|
116
|
+
|
|
94
117
|
|
|
95
118
|
def _extract_event_code(msg: str) -> Optional[str]:
|
|
96
|
-
# Cisco
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
119
|
+
# Cisco style codes (%ASA-6-106100, %SEC-6-IPACCESSLOGRP, %LINK-3-UPDOWN)
|
|
120
|
+
# may appear at the start of the message or after an inline timestamp.
|
|
121
|
+
match = CISCO_CODE_RE.search(msg)
|
|
122
|
+
if match:
|
|
123
|
+
return match.group(1)
|
|
100
124
|
return None
|
|
101
125
|
|
|
102
126
|
|
|
@@ -8,18 +8,20 @@ from datetime import datetime, timezone, timedelta
|
|
|
8
8
|
from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional, Tuple
|
|
9
9
|
|
|
10
10
|
from .utils import (
|
|
11
|
+
KEY_VALUE_RE,
|
|
11
12
|
JournalJSONMapping,
|
|
12
13
|
convert_pri,
|
|
13
14
|
guess_hostname,
|
|
14
15
|
month_abbr_to_int,
|
|
15
16
|
parse_iso8601,
|
|
17
|
+
parse_key_value_pairs,
|
|
16
18
|
sanitize_message,
|
|
17
19
|
)
|
|
18
20
|
|
|
19
21
|
logger = logging.getLogger(__name__)
|
|
20
22
|
|
|
21
23
|
|
|
22
|
-
@dataclass
|
|
24
|
+
@dataclass
|
|
23
25
|
class ParsedEvent:
|
|
24
26
|
pri: Optional[int]
|
|
25
27
|
facility: Optional[int]
|
|
@@ -39,9 +41,11 @@ class ParsedEvent:
|
|
|
39
41
|
RFC3164_RE = re.compile(
|
|
40
42
|
r"^(?:<(?P<pri>\d+)>)?"
|
|
41
43
|
r"(?P<month>[A-Z][a-z]{2})\s+"
|
|
42
|
-
r"(?P<day>\d{1,2})\s"
|
|
44
|
+
r"(?P<day>\d{1,2})\s+"
|
|
45
|
+
r"(?:(?P<year>\d{4})\s+)?"
|
|
43
46
|
r"(?P<time>\d{2}:\d{2}:\d{2})\s"
|
|
44
|
-
r"(?P<host>[^\s]
|
|
47
|
+
r"(?P<host>[^\s]+?):?\s+"
|
|
48
|
+
r"(?::\s+)?"
|
|
45
49
|
r"(?:(?P<tag>[\w\-/\.]+)(?:\[(?P<pid>[^\]]+)\])?:\s?)?"
|
|
46
50
|
r"(?P<msg>.*)$"
|
|
47
51
|
)
|
|
@@ -82,6 +86,44 @@ JOURNALCTL_ISO_RE = re.compile(
|
|
|
82
86
|
|
|
83
87
|
KV_SPLIT_RE = re.compile(r"\s+|,|")
|
|
84
88
|
|
|
89
|
+
# Native Cisco console/buffer format: optional sequence number, optional
|
|
90
|
+
# clock-state marker (* = unsynced, . = drifting), timestamp with optional
|
|
91
|
+
# year/milliseconds/timezone, then the %CODE message.
|
|
92
|
+
CISCO_SEQ_RE = re.compile(
|
|
93
|
+
r"^(?:<(?P<pri>\d+)>)?"
|
|
94
|
+
r"(?:(?P<seq>\d+):\s+)?"
|
|
95
|
+
r"[*.]?"
|
|
96
|
+
r"(?P<month>[A-Z][a-z]{2})\s+"
|
|
97
|
+
r"(?P<day>\d{1,2})\s+"
|
|
98
|
+
r"(?:(?P<year>\d{4})\s+)?"
|
|
99
|
+
r"(?P<time>\d{2}:\d{2}:\d{2})(?:\.\d+)?"
|
|
100
|
+
r"(?:\s+[A-Z]{3,4})?:\s+"
|
|
101
|
+
r"(?P<msg>%[A-Z].*)$"
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
# ISO timestamp syslog without RFC5424 framing: <PRI>ISO host [tag[pid]:] msg
|
|
105
|
+
ISO_SYSLOG_RE = re.compile(
|
|
106
|
+
r"^(?:<(?P<pri>\d+)>)?"
|
|
107
|
+
r"(?P<timestamp>\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?)\s+"
|
|
108
|
+
r"(?P<host>[^\s]+?):?\s+"
|
|
109
|
+
r"(?:(?P<tag>[\w\-/\.]+)(?:\[(?P<pid>[^\]]+)\])?:\s?)?"
|
|
110
|
+
r"(?P<msg>.*)$"
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
# Key=value stream format (Fortinet FortiGate, Sophos XG, ...): the whole
|
|
114
|
+
# line is key=value pairs, optionally preceded by a PRI. Detection is done
|
|
115
|
+
# in _looks_like_kv_line (linear scan) rather than a repeated-group regex,
|
|
116
|
+
# which backtracks catastrophically on long non-matching lines.
|
|
117
|
+
PRI_PREFIX_RE = re.compile(r"^<(?P<pri>\d{1,3})>")
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _looks_like_kv_line(body: str) -> bool:
|
|
121
|
+
matches = list(KEY_VALUE_RE.finditer(body))
|
|
122
|
+
if len(matches) < 3:
|
|
123
|
+
return False
|
|
124
|
+
covered = sum(m.end() - m.start() for m in matches)
|
|
125
|
+
return covered >= 0.7 * len(body.strip())
|
|
126
|
+
|
|
85
127
|
|
|
86
128
|
def _infer_timestamp(month: int, day: int, time_str: str, *, now: Optional[datetime]) -> datetime:
|
|
87
129
|
if now is None:
|
|
@@ -129,7 +171,13 @@ def parse_rfc3164(line: str, *, now: Optional[datetime]) -> ParsedEvent | None:
|
|
|
129
171
|
pri = int(gd["pri"]) if gd.get("pri") else None
|
|
130
172
|
facility, severity = convert_pri(pri)
|
|
131
173
|
month = month_abbr_to_int(gd["month"])
|
|
132
|
-
|
|
174
|
+
hour, minute, second = map(int, gd["time"].split(":"))
|
|
175
|
+
if gd.get("year"):
|
|
176
|
+
ts = datetime(int(gd["year"]), month, int(gd["day"]), hour, minute, second, tzinfo=timezone.utc)
|
|
177
|
+
ts_orig = f"{gd['month']} {gd['day']} {gd['year']} {gd['time']}"
|
|
178
|
+
else:
|
|
179
|
+
ts = _infer_timestamp(month, int(gd["day"]), gd["time"], now=now)
|
|
180
|
+
ts_orig = f"{gd['month']} {gd['day']} {gd['time']}"
|
|
133
181
|
app = gd.get("tag")
|
|
134
182
|
msg = gd.get("msg", "")
|
|
135
183
|
return ParsedEvent(
|
|
@@ -137,7 +185,7 @@ def parse_rfc3164(line: str, *, now: Optional[datetime]) -> ParsedEvent | None:
|
|
|
137
185
|
facility=facility,
|
|
138
186
|
severity=severity,
|
|
139
187
|
ts=ts,
|
|
140
|
-
ts_orig=
|
|
188
|
+
ts_orig=ts_orig,
|
|
141
189
|
host=gd.get("host"),
|
|
142
190
|
app=app,
|
|
143
191
|
pid=gd.get("pid"),
|
|
@@ -312,6 +360,115 @@ def parse_journalctl_iso(line: str) -> ParsedEvent | None:
|
|
|
312
360
|
)
|
|
313
361
|
|
|
314
362
|
|
|
363
|
+
def parse_cisco_seq(line: str, *, now: Optional[datetime]) -> ParsedEvent | None:
|
|
364
|
+
match = CISCO_SEQ_RE.match(line)
|
|
365
|
+
if not match:
|
|
366
|
+
return None
|
|
367
|
+
gd = match.groupdict()
|
|
368
|
+
pri = int(gd["pri"]) if gd.get("pri") else None
|
|
369
|
+
facility, severity = convert_pri(pri)
|
|
370
|
+
month = month_abbr_to_int(gd["month"])
|
|
371
|
+
hour, minute, second = map(int, gd["time"].split(":"))
|
|
372
|
+
if gd.get("year"):
|
|
373
|
+
ts = datetime(int(gd["year"]), month, int(gd["day"]), hour, minute, second, tzinfo=timezone.utc)
|
|
374
|
+
else:
|
|
375
|
+
ts = _infer_timestamp(month, int(gd["day"]), gd["time"], now=now)
|
|
376
|
+
sd = {"cisco.sequence": gd["seq"]} if gd.get("seq") else {}
|
|
377
|
+
return ParsedEvent(
|
|
378
|
+
pri=pri,
|
|
379
|
+
facility=facility,
|
|
380
|
+
severity=severity,
|
|
381
|
+
ts=ts,
|
|
382
|
+
ts_orig=f"{gd['month']} {gd['day']} {gd['time']}",
|
|
383
|
+
host=None,
|
|
384
|
+
app=None,
|
|
385
|
+
pid=None,
|
|
386
|
+
msgid=None,
|
|
387
|
+
sd=sd,
|
|
388
|
+
msg=gd.get("msg", ""),
|
|
389
|
+
raw=line,
|
|
390
|
+
source_hint="cisco",
|
|
391
|
+
)
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def parse_iso_syslog(line: str) -> ParsedEvent | None:
|
|
395
|
+
match = ISO_SYSLOG_RE.match(line)
|
|
396
|
+
if not match:
|
|
397
|
+
return None
|
|
398
|
+
gd = match.groupdict()
|
|
399
|
+
pri = int(gd["pri"]) if gd.get("pri") else None
|
|
400
|
+
facility, severity = convert_pri(pri)
|
|
401
|
+
try:
|
|
402
|
+
ts = parse_iso8601(gd["timestamp"])
|
|
403
|
+
except ValueError:
|
|
404
|
+
return None
|
|
405
|
+
return ParsedEvent(
|
|
406
|
+
pri=pri,
|
|
407
|
+
facility=facility,
|
|
408
|
+
severity=severity,
|
|
409
|
+
ts=ts,
|
|
410
|
+
ts_orig=gd["timestamp"],
|
|
411
|
+
host=gd.get("host"),
|
|
412
|
+
app=gd.get("tag"),
|
|
413
|
+
pid=gd.get("pid"),
|
|
414
|
+
msgid=None,
|
|
415
|
+
sd={},
|
|
416
|
+
msg=gd.get("msg", ""),
|
|
417
|
+
raw=line,
|
|
418
|
+
source_hint="iso_syslog",
|
|
419
|
+
)
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def parse_kv_stream(line: str) -> ParsedEvent | None:
|
|
423
|
+
pri = None
|
|
424
|
+
body = line
|
|
425
|
+
pri_match = PRI_PREFIX_RE.match(line)
|
|
426
|
+
if pri_match:
|
|
427
|
+
pri = int(pri_match.group("pri"))
|
|
428
|
+
body = line[pri_match.end():]
|
|
429
|
+
if not _looks_like_kv_line(body):
|
|
430
|
+
return None
|
|
431
|
+
facility, severity = convert_pri(pri)
|
|
432
|
+
pairs = parse_key_value_pairs(body)
|
|
433
|
+
if len(pairs) < 3:
|
|
434
|
+
return None
|
|
435
|
+
|
|
436
|
+
ts = None
|
|
437
|
+
if pairs.get("date") and pairs.get("time"):
|
|
438
|
+
try:
|
|
439
|
+
ts = parse_iso8601(f"{pairs['date']}T{pairs['time']}")
|
|
440
|
+
except ValueError:
|
|
441
|
+
ts = None
|
|
442
|
+
elif pairs.get("eventtime", "").isdigit():
|
|
443
|
+
try:
|
|
444
|
+
ts = parse_iso8601(pairs["eventtime"][:16])
|
|
445
|
+
except ValueError:
|
|
446
|
+
ts = None
|
|
447
|
+
|
|
448
|
+
host = (
|
|
449
|
+
pairs.get("devname")
|
|
450
|
+
or pairs.get("device_name")
|
|
451
|
+
or pairs.get("dvchost")
|
|
452
|
+
or pairs.get("devid")
|
|
453
|
+
or pairs.get("device_id")
|
|
454
|
+
)
|
|
455
|
+
return ParsedEvent(
|
|
456
|
+
pri=pri,
|
|
457
|
+
facility=facility,
|
|
458
|
+
severity=severity,
|
|
459
|
+
ts=ts,
|
|
460
|
+
ts_orig=f"{pairs.get('date', '')} {pairs.get('time', '')}".strip(),
|
|
461
|
+
host=host,
|
|
462
|
+
app=pairs.get("type") or pairs.get("log_type"),
|
|
463
|
+
pid=None,
|
|
464
|
+
msgid=pairs.get("logid") or pairs.get("log_id"),
|
|
465
|
+
sd={},
|
|
466
|
+
msg=body.strip(),
|
|
467
|
+
raw=line,
|
|
468
|
+
source_hint="kv",
|
|
469
|
+
)
|
|
470
|
+
|
|
471
|
+
|
|
315
472
|
PARSERS = {
|
|
316
473
|
"rfc3164": parse_rfc3164,
|
|
317
474
|
"rfc5424": parse_rfc5424,
|
|
@@ -320,6 +477,9 @@ PARSERS = {
|
|
|
320
477
|
"journald_json": parse_journalctl_json,
|
|
321
478
|
"journald_short": parse_journalctl_short,
|
|
322
479
|
"journald_iso": parse_journalctl_iso,
|
|
480
|
+
"cisco_seq": parse_cisco_seq,
|
|
481
|
+
"iso_syslog": parse_iso_syslog,
|
|
482
|
+
"kv": parse_kv_stream,
|
|
323
483
|
}
|
|
324
484
|
|
|
325
485
|
|
|
@@ -330,6 +490,9 @@ def autodetect_and_parse(line: str, *, mode: Optional[str] = None, now: Optional
|
|
|
330
490
|
("rfc5424", lambda s: s.startswith("<") and ">" in s[:10]),
|
|
331
491
|
("rsyslog_file", lambda s: bool(RSYSLOG_FILE_RE.match(s))),
|
|
332
492
|
("journald_iso", lambda s: bool(JOURNALCTL_ISO_RE.match(s))),
|
|
493
|
+
("iso_syslog", lambda s: bool(ISO_SYSLOG_RE.match(s))),
|
|
494
|
+
("kv", lambda s: "=" in s[:80]),
|
|
495
|
+
("cisco_seq", lambda s: bool(CISCO_SEQ_RE.match(s))),
|
|
333
496
|
("journald_short", lambda s: bool(JOURNALCTL_SHORT_RE.match(s))),
|
|
334
497
|
("rfc3164", lambda s: bool(RFC3164_RE.match(s))),
|
|
335
498
|
]
|
|
@@ -354,12 +517,32 @@ def autodetect_and_parse(line: str, *, mode: Optional[str] = None, now: Optional
|
|
|
354
517
|
logger.exception("Parser %s failed for line", key)
|
|
355
518
|
continue
|
|
356
519
|
|
|
357
|
-
#
|
|
520
|
+
# Nothing known matched: let the adaptive detector analyze the line and
|
|
521
|
+
# synthesize a reusable pattern before giving up.
|
|
522
|
+
from .adaptive import adaptive_parse
|
|
523
|
+
|
|
524
|
+
try:
|
|
525
|
+
adaptive_event = adaptive_parse(line, now=now)
|
|
526
|
+
except Exception:
|
|
527
|
+
logger.exception("Adaptive parser failed for line")
|
|
528
|
+
adaptive_event = None
|
|
529
|
+
if adaptive_event is not None:
|
|
530
|
+
return adaptive_event
|
|
531
|
+
|
|
532
|
+
# Fallback to raw message if nothing matches. A leading <PRI> is still
|
|
533
|
+
# honored so facility/severity survive for non-standard formats.
|
|
358
534
|
logger.debug("Falling back to raw parser for line: %s", line)
|
|
535
|
+
pri = None
|
|
536
|
+
body = line
|
|
537
|
+
pri_match = re.match(r"^<(\d{1,3})>", line)
|
|
538
|
+
if pri_match:
|
|
539
|
+
pri = int(pri_match.group(1))
|
|
540
|
+
body = line[pri_match.end():]
|
|
541
|
+
facility, severity = convert_pri(pri)
|
|
359
542
|
return ParsedEvent(
|
|
360
|
-
pri=
|
|
361
|
-
facility=
|
|
362
|
-
severity=
|
|
543
|
+
pri=pri,
|
|
544
|
+
facility=facility,
|
|
545
|
+
severity=severity,
|
|
363
546
|
ts=None,
|
|
364
547
|
ts_orig="",
|
|
365
548
|
host=guess_hostname(),
|
|
@@ -367,7 +550,7 @@ def autodetect_and_parse(line: str, *, mode: Optional[str] = None, now: Optional
|
|
|
367
550
|
pid=None,
|
|
368
551
|
msgid=None,
|
|
369
552
|
sd={},
|
|
370
|
-
msg=sanitize_message(
|
|
553
|
+
msg=sanitize_message(body),
|
|
371
554
|
raw=line,
|
|
372
555
|
source_hint="unknown",
|
|
373
556
|
)
|
|
@@ -26,7 +26,18 @@ MONTHS = {
|
|
|
26
26
|
"Dec": 12,
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
-
KEY_VALUE_RE = re.compile(r"(?P<key>[A-Za-z0-9_.-]+)=(?P<value>\S+)")
|
|
29
|
+
KEY_VALUE_RE = re.compile(r"(?P<key>[A-Za-z0-9_.-]+)=(?P<value>\"[^\"]*\"|'[^']*'|\S+)")
|
|
30
|
+
|
|
31
|
+
SEVERITY_WORDS = {
|
|
32
|
+
"emergency": 0, "emerg": 0, "panic": 0,
|
|
33
|
+
"alert": 1,
|
|
34
|
+
"critical": 2, "crit": 2,
|
|
35
|
+
"error": 3, "err": 3,
|
|
36
|
+
"warning": 4, "warn": 4,
|
|
37
|
+
"notice": 5,
|
|
38
|
+
"information": 6, "informational": 6, "info": 6,
|
|
39
|
+
"debug": 7,
|
|
40
|
+
}
|
|
30
41
|
UTF8_REPLACEMENT_CHAR = "\uFFFD"
|
|
31
42
|
|
|
32
43
|
|
|
@@ -80,12 +91,16 @@ def parse_key_value_pairs(msg: str) -> Dict[str, str]:
|
|
|
80
91
|
for match in KEY_VALUE_RE.finditer(msg):
|
|
81
92
|
key = match.group("key")
|
|
82
93
|
value = match.group("value")
|
|
83
|
-
if value
|
|
94
|
+
if len(value) >= 2 and value[0] in "\"'" and value[-1] == value[0]:
|
|
84
95
|
value = value[1:-1]
|
|
85
96
|
pairs[key] = value
|
|
86
97
|
return pairs
|
|
87
98
|
|
|
88
99
|
|
|
100
|
+
def severity_from_word(word: str) -> Optional[int]:
|
|
101
|
+
return SEVERITY_WORDS.get(word.strip().lower())
|
|
102
|
+
|
|
103
|
+
|
|
89
104
|
def guess_hostname() -> str:
|
|
90
105
|
try:
|
|
91
106
|
return socket.gethostname()
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
"""Tests for the new format parsers and the adaptive pattern detector."""
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
|
|
7
|
+
from syslogcef import convert_line, normalize_event, parse_syslog
|
|
8
|
+
from syslogcef.adaptive import cache_size, clear_cache
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def fields(cef: str) -> list[str]:
|
|
12
|
+
return cef.split("|")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
# --- key=value stream formats -------------------------------------------------
|
|
16
|
+
|
|
17
|
+
FORTINET_LINE = '<189>date=2020-04-23 time=12:32:48 devname="testswitch3" devid="someotherrouteridagain" logid="0102043014" type="event" subtype="user" level="notice" vd="root" srcip=10.1.1.1 dstip=10.2.2.2 action="login"'
|
|
18
|
+
SOPHOS_LINE = '<30>device="SFW" date=2020-05-18 time=14:38:48 timezone="CEST" device_name="XG230" device_id=1234567890123456 log_id=041101618035 log_type="Anti-Spam" log_component="SMTP" log_subtype="Allowed" priority=Information src_ip="10.9.9.9" user_name="alice"'
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def test_fortinet_kv_line():
|
|
22
|
+
ev = parse_syslog(FORTINET_LINE)
|
|
23
|
+
assert ev.source_hint == "kv"
|
|
24
|
+
assert ev.host == "testswitch3"
|
|
25
|
+
assert ev.pri == 189
|
|
26
|
+
assert ev.ts.year == 2020
|
|
27
|
+
cef = convert_line(FORTINET_LINE)
|
|
28
|
+
assert fields(cef)[1] == "Fortinet"
|
|
29
|
+
assert fields(cef)[4] == "fortinet.0102043014"
|
|
30
|
+
assert "src=10.1.1.1" in cef
|
|
31
|
+
assert "dvchost=testswitch3" in cef
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def test_sophos_kv_line():
|
|
35
|
+
ev = parse_syslog(SOPHOS_LINE)
|
|
36
|
+
assert ev.source_hint == "kv"
|
|
37
|
+
assert ev.host == "XG230"
|
|
38
|
+
cef = convert_line(SOPHOS_LINE)
|
|
39
|
+
assert fields(cef)[1] == "Sophos"
|
|
40
|
+
assert fields(cef)[4] == "sophos.041101618035"
|
|
41
|
+
assert "suser=alice" in cef
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_kv_severity_from_level_word_without_pri():
|
|
45
|
+
line = 'date=2020-04-23 time=12:32:48 devname="fw1" logid="0100000001" level="warning" msgtext="x"'
|
|
46
|
+
ev = normalize_event(parse_syslog(line))
|
|
47
|
+
assert ev.severity == 4
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def test_quoted_kv_values_with_spaces():
|
|
51
|
+
line = '<30>device="SFW" date=2020-05-18 time=14:38:48 device_name="XG230" log_id=1 reason="Email has been accepted" x=1'
|
|
52
|
+
ev = normalize_event(parse_syslog(line))
|
|
53
|
+
assert ev.kv["reason"] == "Email has been accepted"
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
# --- native Cisco console formats ---------------------------------------------
|
|
57
|
+
|
|
58
|
+
def test_cisco_native_sequence_line():
|
|
59
|
+
line = "000123: Feb 8 04:00:47.272: %SEC-6-IPACCESSLOGP: list 100 denied tcp 10.1.1.1(1024) -> 10.2.2.2(80), 1 packet"
|
|
60
|
+
ev = parse_syslog(line)
|
|
61
|
+
assert ev.source_hint == "cisco"
|
|
62
|
+
assert ev.sd["cisco.sequence"] == "000123"
|
|
63
|
+
norm = normalize_event(ev)
|
|
64
|
+
assert norm.kv["event_code"] == "SEC-6-IPACCESSLOGP"
|
|
65
|
+
assert norm.severity == 6
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def test_cisco_star_clock_line():
|
|
69
|
+
line = "*Mar 1 18:46:11.012: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36)"
|
|
70
|
+
ev = parse_syslog(line)
|
|
71
|
+
assert ev.source_hint == "cisco"
|
|
72
|
+
norm = normalize_event(ev)
|
|
73
|
+
assert norm.kv["event_code"] == "SYS-5-CONFIG_I"
|
|
74
|
+
assert norm.severity == 5
|
|
75
|
+
assert fields(convert_line(line))[2] == "IOS"
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def test_cisco_native_with_year_and_timezone():
|
|
79
|
+
line = "Feb 8 2024 04:00:47 UTC: %LINK-3-UPDOWN: Interface GigabitEthernet0/1, changed state to down"
|
|
80
|
+
ev = parse_syslog(line)
|
|
81
|
+
assert ev.source_hint == "cisco"
|
|
82
|
+
assert ev.ts.year == 2024
|
|
83
|
+
assert normalize_event(ev).severity == 3
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
# --- ISO syslog without RFC5424 framing ---------------------------------------
|
|
87
|
+
|
|
88
|
+
def test_iso_timestamp_with_pri_and_tag():
|
|
89
|
+
line = "<13>2023-07-01T12:00:00Z web01 nginx[42]: GET /index.html"
|
|
90
|
+
ev = parse_syslog(line)
|
|
91
|
+
assert ev.source_hint == "iso_syslog"
|
|
92
|
+
assert ev.pri == 13
|
|
93
|
+
assert ev.host == "web01"
|
|
94
|
+
assert ev.app == "nginx"
|
|
95
|
+
assert ev.pid == "42"
|
|
96
|
+
assert ev.msg == "GET /index.html"
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
# --- adaptive detection -------------------------------------------------------
|
|
100
|
+
|
|
101
|
+
def test_adaptive_learns_unknown_layout_and_caches():
|
|
102
|
+
clear_cache()
|
|
103
|
+
line1 = "2020/04/23 12:32:48 fw-edge-01 session established peer=10.0.0.1"
|
|
104
|
+
line2 = "2020/04/24 09:10:11 fw-edge-02 session torn down peer=10.0.0.2"
|
|
105
|
+
ev1 = parse_syslog(line1)
|
|
106
|
+
assert ev1.source_hint == "adaptive"
|
|
107
|
+
assert ev1.host == "fw-edge-01"
|
|
108
|
+
assert ev1.ts.year == 2020 and ev1.ts.month == 4 and ev1.ts.day == 23
|
|
109
|
+
assert cache_size() == 1
|
|
110
|
+
ev2 = parse_syslog(line2)
|
|
111
|
+
assert ev2.source_hint == "adaptive"
|
|
112
|
+
assert ev2.host == "fw-edge-02"
|
|
113
|
+
assert cache_size() == 1 # reused the learned pattern
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def test_adaptive_dmy_dash_timestamp():
|
|
117
|
+
line = "17-Apr-2020 14:08:08 core01 link flap detected"
|
|
118
|
+
ev = parse_syslog(line)
|
|
119
|
+
assert ev.source_hint == "adaptive"
|
|
120
|
+
assert ev.ts.year == 2020 and ev.ts.month == 4
|
|
121
|
+
assert ev.host == "core01"
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def test_adaptive_preserves_pri():
|
|
125
|
+
line = "<86>2020/04/23 12:32:48 bastion sudo session opened"
|
|
126
|
+
ev = parse_syslog(line)
|
|
127
|
+
assert ev.source_hint == "adaptive"
|
|
128
|
+
assert ev.pri == 86
|
|
129
|
+
assert ev.severity == 6
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def test_garbage_still_falls_back_to_unknown():
|
|
133
|
+
ev = parse_syslog("complete garbage with no timestamp at all ###")
|
|
134
|
+
assert ev.source_hint == "unknown"
|
|
135
|
+
ev2 = parse_syslog("<30>still garbage but with a pri")
|
|
136
|
+
assert ev2.source_hint == "unknown"
|
|
137
|
+
assert ev2.pri == 30
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
"""Regression tests built from real sample lines in the Elastic Beats
|
|
4
|
+
filebeat module test corpus (x-pack/filebeat/module/*/test)."""
|
|
5
|
+
|
|
6
|
+
from syslogcef import convert_line, normalize_event, parse_syslog
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def fields(cef: str) -> list[str]:
|
|
10
|
+
return cef.split("|")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def test_asa_with_year_and_tag():
|
|
14
|
+
line = "Oct 10 2018 12:34:56 localhost CiscoASA[999]: %ASA-6-305011: Built dynamic TCP translation from inside:172.31.98.44/1772 to outside:100.66.98.44/8256"
|
|
15
|
+
ev = parse_syslog(line)
|
|
16
|
+
assert ev.source_hint == "rfc3164"
|
|
17
|
+
assert ev.host == "localhost"
|
|
18
|
+
assert ev.app == "CiscoASA"
|
|
19
|
+
assert ev.pid == "999"
|
|
20
|
+
assert ev.ts.year == 2018
|
|
21
|
+
cef = convert_line(line)
|
|
22
|
+
assert fields(cef)[1] == "Cisco"
|
|
23
|
+
assert fields(cef)[4] == "asa.ASA-6-305011"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_asa_with_year_and_standalone_colon_host():
|
|
27
|
+
line = "Apr 17 2020 14:08:08 SNL-ASA-VPN-A01 : %ASA-6-302016: Teardown UDP connection 110577675 for Outside:10.123.123.123/53723(LOCAL\\Elastic) to Inside:10.233.123.123/53 duration 0:00:00 bytes 148"
|
|
28
|
+
ev = parse_syslog(line)
|
|
29
|
+
assert ev.source_hint == "rfc3164"
|
|
30
|
+
assert ev.host == "SNL-ASA-VPN-A01"
|
|
31
|
+
assert ev.ts.year == 2020
|
|
32
|
+
cef = convert_line(line)
|
|
33
|
+
assert fields(cef)[4] == "asa.ASA-6-302016"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def test_asa_host_with_trailing_colon():
|
|
37
|
+
line = "Oct 10 2019 10:21:36 localhost: %ASA-6-302021: Teardown ICMP connection for faddr 192.0.2.15/0 gaddr 192.0.2.134/57808 laddr 192.0.2.134/57808"
|
|
38
|
+
ev = parse_syslog(line)
|
|
39
|
+
assert ev.source_hint == "rfc3164"
|
|
40
|
+
assert ev.host == "localhost"
|
|
41
|
+
assert normalize_event(ev).kv["event_code"] == "ASA-6-302021"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_ios_sequence_number_line_gets_code_and_severity():
|
|
45
|
+
line = "Feb 8 04:00:48 198.51.100.2 585917: Feb 8 04:00:47.272: %SEC-6-IPACCESSLOGRP: list 177 denied igmp 198.51.100.197 -> 224.0.0.22, 1 packet"
|
|
46
|
+
ev = normalize_event(parse_syslog(line))
|
|
47
|
+
assert ev.kv["event_code"] == "SEC-6-IPACCESSLOGRP"
|
|
48
|
+
assert ev.severity == 6
|
|
49
|
+
cef = convert_line(line)
|
|
50
|
+
assert fields(cef)[1] == "Cisco"
|
|
51
|
+
assert fields(cef)[2] == "IOS"
|
|
52
|
+
assert fields(cef)[4] == "ios.SEC-6-IPACCESSLOGRP"
|
|
53
|
+
# syslog severity 6 maps to CEF severity 2 via the default map
|
|
54
|
+
assert fields(cef)[6] == "2"
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def test_ftd_event_code_routes_to_asa_mapping():
|
|
58
|
+
line = "Apr 17 2020 14:00:00 firepower : %FTD-6-305012: Teardown dynamic TCP translation from Inside:10.1.1.1/54000 to Outside:198.51.100.1/54000 duration 0:00:30"
|
|
59
|
+
cef = convert_line(line)
|
|
60
|
+
assert fields(cef)[1] == "Cisco"
|
|
61
|
+
assert fields(cef)[4] == "asa.FTD-6-305012"
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def test_pri_survives_sophos_kv_line():
|
|
65
|
+
# Sophos XG style: PRI prefix followed by key=value pairs, no syslog
|
|
66
|
+
# header. Parsed by the dedicated kv parser; PRI is preserved.
|
|
67
|
+
line = '<30>device="SFW" date=2020-05-18 time=14:38:48 timezone="CEST" device_name="XG230" log_type="Anti-Spam" src_ip="" dst_ip=""'
|
|
68
|
+
ev = parse_syslog(line)
|
|
69
|
+
assert ev.source_hint == "kv"
|
|
70
|
+
assert ev.pri == 30
|
|
71
|
+
assert ev.facility == 3
|
|
72
|
+
assert ev.severity == 6
|
|
73
|
+
assert not ev.msg.startswith("<30>")
|
|
74
|
+
cef = convert_line(line)
|
|
75
|
+
assert fields(cef)[6] == "2"
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def test_pri_survives_raw_fallback():
|
|
79
|
+
line = "<30>completely nonstandard payload without structure"
|
|
80
|
+
ev = parse_syslog(line)
|
|
81
|
+
assert ev.source_hint == "unknown"
|
|
82
|
+
assert ev.pri == 30
|
|
83
|
+
assert ev.severity == 6
|
|
84
|
+
assert not ev.msg.startswith("<30>")
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def test_classic_rfc3164_without_year_still_parses():
|
|
88
|
+
line = "<166>Jan 1 12:34:56 router1 sshd[123]: Accepted password for user"
|
|
89
|
+
ev = parse_syslog(line)
|
|
90
|
+
assert ev.source_hint == "rfc3164"
|
|
91
|
+
assert ev.host == "router1"
|
|
92
|
+
assert ev.app == "sshd"
|
|
93
|
+
assert ev.pid == "123"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|