syslog2cef 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- syslog2cef-0.1.0.dist-info/METADATA +202 -0
- syslog2cef-0.1.0.dist-info/RECORD +20 -0
- syslog2cef-0.1.0.dist-info/WHEEL +5 -0
- syslog2cef-0.1.0.dist-info/entry_points.txt +2 -0
- syslog2cef-0.1.0.dist-info/licenses/LICENSE +21 -0
- syslog2cef-0.1.0.dist-info/top_level.txt +1 -0
- syslogcef/__init__.py +19 -0
- syslogcef/__main__.py +7 -0
- syslogcef/api.py +127 -0
- syslogcef/cef.py +103 -0
- syslogcef/cli.py +136 -0
- syslogcef/mappings/__init__.py +40 -0
- syslogcef/mappings/cisco_asa.json +26 -0
- syslogcef/mappings/cisco_ios.json +13 -0
- syslogcef/mappings/f5.json +13 -0
- syslogcef/mappings/linux.json +13 -0
- syslogcef/mappings/vmware.json +12 -0
- syslogcef/normalizer.py +106 -0
- syslogcef/parsers.py +380 -0
- syslogcef/utils.py +144 -0
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: syslog2cef
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
|
|
5
|
+
Author-email: Tamir Suliman <allamiro@gmail.com>
|
|
6
|
+
Project-URL: Homepage, https://github.com/allamiro/syslogcef
|
|
7
|
+
Project-URL: Issues, https://github.com/allamiro/syslogcef/issues
|
|
8
|
+
Keywords: syslog,cef,logging,security
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Topic :: System :: Logging
|
|
13
|
+
Requires-Python: >=3.9
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENSE
|
|
16
|
+
Provides-Extra: test
|
|
17
|
+
Requires-Dist: pytest>=7; extra == "test"
|
|
18
|
+
Dynamic: license-file
|
|
19
|
+
|
|
20
|
+
# syslogcef
|
|
21
|
+
|
|
22
|
+
syslogcef converts syslog events into ArcSight Common Event Format (CEF). It
|
|
23
|
+
is a small, dependency-free Python package with a command line interface,
|
|
24
|
+
built for feeding syslog data from network devices and Linux hosts into SIEM
|
|
25
|
+
platforms that consume CEF.
|
|
26
|
+
|
|
27
|
+
## Features
|
|
28
|
+
|
|
29
|
+
- Automatic detection and parsing of multiple syslog dialects:
|
|
30
|
+
RFC3164 (BSD), RFC5424 (including structured data), rsyslog JSON and
|
|
31
|
+
file formats, and systemd journal exports (JSON, short, and ISO formats).
|
|
32
|
+
- Deterministic vendor/product mappings bundled for Cisco ASA, Cisco IOS,
|
|
33
|
+
F5 BIG-IP, generic Linux, and VMware ESXi, with automatic mapping
|
|
34
|
+
selection based on message content.
|
|
35
|
+
- Custom mappings supplied as JSON files or Python dictionaries.
|
|
36
|
+
- Escaping of untrusted log content in both CEF header and extension
|
|
37
|
+
fields, so crafted messages cannot forge header fields or split records.
|
|
38
|
+
- Streaming CLI with stdin/file input, tail (follow) mode across multiple
|
|
39
|
+
files, optional multiprocessing, and no runtime dependencies.
|
|
40
|
+
- Malformed lines never abort a stream: unparseable input falls back to a
|
|
41
|
+
raw-message event.
|
|
42
|
+
|
|
43
|
+
## Installation
|
|
44
|
+
|
|
45
|
+
From PyPI (the distribution is named `syslog2cef` because the `syslogcef`
|
|
46
|
+
name on PyPI belongs to an unrelated project; the import package and CLI
|
|
47
|
+
are still `syslogcef`):
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
pip install syslog2cef
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
From source:
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
git clone https://github.com/allamiro/syslogcef.git
|
|
57
|
+
cd syslogcef
|
|
58
|
+
pip install .
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
An RPM package with a systemd service is also available from the GitHub
|
|
62
|
+
releases page; see "Running as a service" below.
|
|
63
|
+
|
|
64
|
+
## Command Line Usage
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
# Read from stdin, write CEF to stdout
|
|
68
|
+
syslogcef < /var/log/syslog
|
|
69
|
+
|
|
70
|
+
# Convert one or more files and write to an output file
|
|
71
|
+
syslogcef /var/log/messages /var/log/secure --output events.cef
|
|
72
|
+
|
|
73
|
+
# Follow files in real time (all files are tailed concurrently)
|
|
74
|
+
syslogcef /var/log/asa.log /var/log/messages --tail
|
|
75
|
+
|
|
76
|
+
# Force a parser and mapping instead of auto-detection
|
|
77
|
+
syslogcef asa.log --mode rfc3164 --mapping syslogcef/mappings/cisco_asa.json
|
|
78
|
+
|
|
79
|
+
# Use multiprocessing for high-volume batch conversion
|
|
80
|
+
syslogcef big.log --multiprocess --pool-size 4
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Options:
|
|
84
|
+
|
|
85
|
+
| Option | Description |
|
|
86
|
+
| ------ | ----------- |
|
|
87
|
+
| `paths` | Input files; stdin is used when omitted. |
|
|
88
|
+
| `-o, --output FILE` | Write CEF lines to a file instead of stdout. |
|
|
89
|
+
| `--mode MODE` | Parser override: `rfc3164`, `rfc5424`, `rsyslog_json`, `rsyslog_file`, `journald_json`, `journald_short`, `journald_iso`. Auto-detected when omitted. |
|
|
90
|
+
| `--mapping FILE` | Mapping JSON file. Auto-selected from message content when omitted. |
|
|
91
|
+
| `--tail` | Follow input files like `tail -f`. |
|
|
92
|
+
| `--multiprocess` | Convert lines using a process pool. |
|
|
93
|
+
| `--pool-size N` | Worker count for `--multiprocess` (default: CPU count minus one). |
|
|
94
|
+
| `--log-level LEVEL` | Python logging level (default `WARNING`). |
|
|
95
|
+
|
|
96
|
+
`python -m syslogcef` is equivalent to the `syslogcef` entry point.
|
|
97
|
+
|
|
98
|
+
## Python API
|
|
99
|
+
|
|
100
|
+
High-level, one call per line:
|
|
101
|
+
|
|
102
|
+
```python
|
|
103
|
+
from syslogcef import convert_line
|
|
104
|
+
|
|
105
|
+
line = "<166>Jan 1 12:34:56 fw01 %ASA-6-302013: Built inbound TCP connection src=10.0.0.1 dst=10.0.0.2"
|
|
106
|
+
print(convert_line(line))
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
Lower-level pipeline when granular control is required:
|
|
110
|
+
|
|
111
|
+
```python
|
|
112
|
+
from syslogcef import parse_syslog, normalize_event, to_cef
|
|
113
|
+
|
|
114
|
+
parsed = parse_syslog(line) # ParsedEvent: pri, timestamp, host, app, msg, ...
|
|
115
|
+
normalized = normalize_event(parsed) # adds key/value pairs, event codes, derived fields
|
|
116
|
+
cef = to_cef(normalized, mapping="my_mapping.json")
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Bundled mappings are importable from `syslogcef.mappings` (`CISCO_ASA`,
|
|
120
|
+
`CISCO_IOS`, `F5`, `LINUX`, `VMWARE`, or `load_mapping(name)`).
|
|
121
|
+
|
|
122
|
+
## Mapping Files
|
|
123
|
+
|
|
124
|
+
A mapping is a JSON object that controls the CEF header and extension
|
|
125
|
+
fields. Values are Python %-format templates resolved against the
|
|
126
|
+
normalized event's fields:
|
|
127
|
+
|
|
128
|
+
```json
|
|
129
|
+
{
|
|
130
|
+
"deviceVendor": "Cisco",
|
|
131
|
+
"deviceProduct": "ASA",
|
|
132
|
+
"deviceVersion": "auto",
|
|
133
|
+
"eventClassId": "asa.%(event_code)s",
|
|
134
|
+
"name": "%(message_short)s",
|
|
135
|
+
"severity_map": { "6": "2", "3": "6" },
|
|
136
|
+
"extensions": {
|
|
137
|
+
"src": "%(src)s",
|
|
138
|
+
"dst": "%(dst)s",
|
|
139
|
+
"cs1Label": "rawEvent",
|
|
140
|
+
"cs1": "%(raw_kv)s"
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
- Header keys: `deviceVendor`, `deviceProduct`, `deviceVersion`,
|
|
146
|
+
`eventClassId`, `name`.
|
|
147
|
+
- `severity_map` translates syslog severity (0-7) to CEF severity (0-10);
|
|
148
|
+
unmapped values pass through.
|
|
149
|
+
- `extensions` maps CEF extension keys to templates. Extensions that
|
|
150
|
+
resolve to an empty value are omitted.
|
|
151
|
+
- Available template fields include `host`, `app`, `pid`, `msgid`, `msg`,
|
|
152
|
+
`message_short` (first 120 characters), `raw`, `raw_kv`, `event_code`,
|
|
153
|
+
`facility`, `severity`, `ts`, plus every key=value pair extracted from
|
|
154
|
+
the message and any RFC5424 structured-data or journald fields.
|
|
155
|
+
|
|
156
|
+
Field templates that reference missing keys resolve to an empty string
|
|
157
|
+
rather than failing the event. See [docs/cef_fields.md](docs/cef_fields.md)
|
|
158
|
+
for the full CEF extension dictionary.
|
|
159
|
+
|
|
160
|
+
## Running as a Service
|
|
161
|
+
|
|
162
|
+
The RPM package installs a systemd unit and an environment file:
|
|
163
|
+
|
|
164
|
+
- `/etc/syslogcef/syslogcef.conf` — input file, output file, and extra
|
|
165
|
+
arguments for the converter.
|
|
166
|
+
- `syslogcef.service` — runs `syslogcef --tail` against the configured
|
|
167
|
+
input and appends CEF to the configured output.
|
|
168
|
+
|
|
169
|
+
```bash
|
|
170
|
+
sudo dnf install syslogcef-*.rpm
|
|
171
|
+
sudo vi /etc/syslogcef/syslogcef.conf
|
|
172
|
+
sudo systemctl enable --now syslogcef
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
See [packaging/rpm/](packaging/rpm/) for the spec file and build
|
|
176
|
+
instructions, including GPG signing of the RPM.
|
|
177
|
+
|
|
178
|
+
## Security
|
|
179
|
+
|
|
180
|
+
Log content is treated as untrusted input. Header and extension values are
|
|
181
|
+
escaped per the CEF specification before rendering, and CR/LF are removed
|
|
182
|
+
from header fields so records cannot be split or spoofed. To report a
|
|
183
|
+
vulnerability, see [SECURITY.md](SECURITY.md) — please do not open public
|
|
184
|
+
issues for security reports.
|
|
185
|
+
|
|
186
|
+
## Development
|
|
187
|
+
|
|
188
|
+
```bash
|
|
189
|
+
git clone https://github.com/allamiro/syslogcef.git
|
|
190
|
+
cd syslogcef
|
|
191
|
+
python -m venv .venv
|
|
192
|
+
source .venv/bin/activate
|
|
193
|
+
pip install -e .[test]
|
|
194
|
+
pytest
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
Contributions are welcome — see [CONTRIBUTING.md](CONTRIBUTING.md). Notable
|
|
198
|
+
changes are tracked in [CHANGELOG.md](CHANGELOG.md).
|
|
199
|
+
|
|
200
|
+
## License
|
|
201
|
+
|
|
202
|
+
MIT — see [LICENSE](LICENSE). Copyright (c) Tamir Suliman.
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
syslog2cef-0.1.0.dist-info/licenses/LICENSE,sha256=l2nGswRGvDbaAf9OMG61hhS9jYY8GDgRtOYICQtvaE0,1070
|
|
2
|
+
syslogcef/__init__.py,sha256=Q_SFz4laNK4ipLeAS-PxHPgOGLWbSI2aeWuX0tPmC2E,496
|
|
3
|
+
syslogcef/__main__.py,sha256=nbdP6zFlOqcCirgxoIQ6HiWVhdh7URm5GOWr5_EwmBk,116
|
|
4
|
+
syslogcef/api.py,sha256=RdVZSo1J39agJoCzA7qANJENq9a7SwN8G-eBQU_Z34I,3304
|
|
5
|
+
syslogcef/cef.py,sha256=mThPrk6Fzcv2rBG96WQEhv_jnLUZr4_GUosvcs0ae9k,3352
|
|
6
|
+
syslogcef/cli.py,sha256=rFKkx9ffq-QANlDPfWbzuH1GWu2vMRkXTCACrfPCbUg,4794
|
|
7
|
+
syslogcef/normalizer.py,sha256=qmqMufqyldce5kJd0Dg-iCaR-mURx6YhHpRfp0FcsHU,2990
|
|
8
|
+
syslogcef/parsers.py,sha256=gK2zN66WUG5FJ6UZThb4I2fOMlRf5duFWjFlgGE8C8s,11088
|
|
9
|
+
syslogcef/utils.py,sha256=8UIif4b9Iloar7rTlebELXC80Wc1PihFufpfp9Foj9M,3699
|
|
10
|
+
syslogcef/mappings/__init__.py,sha256=X9Y7Suon2RXwMRdtK8Fpsm2ueBy-wDsbY67QG89H61U,852
|
|
11
|
+
syslogcef/mappings/cisco_asa.json,sha256=QbCeQWD999C08DjpMiPitB89Z_wLtNof-pMe8kVmVOU,483
|
|
12
|
+
syslogcef/mappings/cisco_ios.json,sha256=uHz5Y1bzfA1zoyByJnm6yVqEwbdmIUNrqU2kPRw6JgI,270
|
|
13
|
+
syslogcef/mappings/f5.json,sha256=h-19rqQ8c703jJwo2RN5fXx8cFpmZUdGF2i1x-Ra-t4,278
|
|
14
|
+
syslogcef/mappings/linux.json,sha256=fA5ZuxJXVzt5vlQV8kF4p5TJfyrLvX11c_G7owLR7Sc,272
|
|
15
|
+
syslogcef/mappings/vmware.json,sha256=kruNvpSb0wglZalAXXHLQurmw9bFlxxgI_hGrcpSWiQ,254
|
|
16
|
+
syslog2cef-0.1.0.dist-info/METADATA,sha256=W_DXPiJ62VFf2D1uFPMF8jJxMbVGDkWFshKuDttbWUc,6913
|
|
17
|
+
syslog2cef-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
18
|
+
syslog2cef-0.1.0.dist-info/entry_points.txt,sha256=-_AClsJ3zP6yWMf1MiRZLtdEF2olVWQKJhI6nWL-vMA,49
|
|
19
|
+
syslog2cef-0.1.0.dist-info/top_level.txt,sha256=7Xt2pSdcLZ9y3ufEbHHz47Bi3rXBVwZZ6nDFUNpvPIM,10
|
|
20
|
+
syslog2cef-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2023 Tamir Suliman
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
syslogcef
|
syslogcef/__init__.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Core API for converting syslog lines into ArcSight CEF.
|
|
2
|
+
|
|
3
|
+
This package exposes a high level pipeline composed of four functions:
|
|
4
|
+
|
|
5
|
+
``parse_syslog`` -> ``normalize_event`` -> ``to_cef`` -> ``convert_line``
|
|
6
|
+
|
|
7
|
+
Each function can be used independently or as part of the complete
|
|
8
|
+
conversion workflow.
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from .api import convert_line, normalize_event, parse_syslog, to_cef
|
|
13
|
+
|
|
14
|
+
__all__ = [
|
|
15
|
+
"convert_line",
|
|
16
|
+
"normalize_event",
|
|
17
|
+
"parse_syslog",
|
|
18
|
+
"to_cef",
|
|
19
|
+
]
|
syslogcef/__main__.py
ADDED
syslogcef/api.py
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import logging
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from datetime import datetime, timezone
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional
|
|
9
|
+
|
|
10
|
+
from .parsers import ParsedEvent, autodetect_and_parse
|
|
11
|
+
from .normalizer import NormalizedEvent, normalize
|
|
12
|
+
from .mappings import CISCO_ASA, CISCO_IOS, F5, LINUX, VMWARE
|
|
13
|
+
from .cef import CEFEvent, build_cef
|
|
14
|
+
|
|
15
|
+
logger = logging.getLogger(__name__)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(slots=True)
|
|
19
|
+
class ParseResult:
|
|
20
|
+
"""Container returned by :func:`parse_syslog`.
|
|
21
|
+
|
|
22
|
+
The ``raw`` attribute always contains the original syslog line.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
event: NormalizedEvent
|
|
26
|
+
raw: str
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def parse_syslog(
|
|
30
|
+
line: str,
|
|
31
|
+
*,
|
|
32
|
+
mode: Optional[str] = None,
|
|
33
|
+
now: Optional[datetime] = None,
|
|
34
|
+
) -> ParsedEvent:
|
|
35
|
+
"""Parse a raw syslog line.
|
|
36
|
+
|
|
37
|
+
Parameters
|
|
38
|
+
----------
|
|
39
|
+
line:
|
|
40
|
+
Raw syslog string.
|
|
41
|
+
mode:
|
|
42
|
+
Optional parser mode. When ``None`` the parser auto-detects RFC3164,
|
|
43
|
+
RFC5424, rsyslog formats and journalctl exports.
|
|
44
|
+
now:
|
|
45
|
+
Optional datetime used when inferring missing year information.
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
parsed = autodetect_and_parse(line, mode=mode, now=now)
|
|
49
|
+
return parsed
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def normalize_event(event: ParsedEvent | NormalizedEvent) -> NormalizedEvent:
|
|
53
|
+
"""Normalize parsed syslog data.
|
|
54
|
+
|
|
55
|
+
The normalizer enriches the event with structured fields, sanitised
|
|
56
|
+
messages and derived metadata that the CEF renderer relies on.
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
if isinstance(event, NormalizedEvent):
|
|
60
|
+
return event
|
|
61
|
+
return normalize(event)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _load_mapping(mapping: Mapping[str, Any] | Path | str | None) -> Mapping[str, Any]:
|
|
65
|
+
if mapping is None:
|
|
66
|
+
return {}
|
|
67
|
+
if isinstance(mapping, Mapping):
|
|
68
|
+
return mapping
|
|
69
|
+
path = Path(mapping)
|
|
70
|
+
with path.open("r", encoding="utf-8") as fp:
|
|
71
|
+
return json.load(fp)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def to_cef(
|
|
75
|
+
event: NormalizedEvent,
|
|
76
|
+
mapping: Mapping[str, Any] | Path | str | None = None,
|
|
77
|
+
) -> str:
|
|
78
|
+
"""Convert a normalized event into a CEF string."""
|
|
79
|
+
|
|
80
|
+
if mapping is None:
|
|
81
|
+
mapping_data = _guess_mapping(event)
|
|
82
|
+
else:
|
|
83
|
+
mapping_data = _load_mapping(mapping)
|
|
84
|
+
cef_event = build_cef(event, mapping_data)
|
|
85
|
+
return cef_event.render()
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def convert_line(
|
|
89
|
+
line: str,
|
|
90
|
+
*,
|
|
91
|
+
mode: Optional[str] = None,
|
|
92
|
+
mapping: Mapping[str, Any] | Path | str | None = None,
|
|
93
|
+
now: Optional[datetime] = None,
|
|
94
|
+
) -> str:
|
|
95
|
+
"""Full pipeline that parses, normalizes and converts a syslog line."""
|
|
96
|
+
|
|
97
|
+
parsed = parse_syslog(line, mode=mode, now=now)
|
|
98
|
+
normalized = normalize_event(parsed)
|
|
99
|
+
|
|
100
|
+
return to_cef(normalized, mapping)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _guess_mapping(event: NormalizedEvent) -> Mapping[str, Any]:
|
|
104
|
+
msg_upper = event.msg.upper()
|
|
105
|
+
app_upper = (event.app or "").upper()
|
|
106
|
+
|
|
107
|
+
if "%ASA-" in msg_upper or "ASA" in app_upper:
|
|
108
|
+
return CISCO_ASA
|
|
109
|
+
if "IOS" in app_upper or "%IOS-" in msg_upper:
|
|
110
|
+
return CISCO_IOS
|
|
111
|
+
if "BIG-IP" in msg_upper or "F5" in app_upper:
|
|
112
|
+
return F5
|
|
113
|
+
if "VMWARE" in msg_upper or "ESXI" in msg_upper or "VMWARE" in app_upper:
|
|
114
|
+
return VMWARE
|
|
115
|
+
return LINUX
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
__all__ = [
|
|
119
|
+
"CEFEvent",
|
|
120
|
+
"ParsedEvent",
|
|
121
|
+
"NormalizedEvent",
|
|
122
|
+
"ParseResult",
|
|
123
|
+
"convert_line",
|
|
124
|
+
"normalize_event",
|
|
125
|
+
"parse_syslog",
|
|
126
|
+
"to_cef",
|
|
127
|
+
]
|
syslogcef/cef.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from datetime import datetime
|
|
6
|
+
from typing import Any, Dict, Mapping, Optional
|
|
7
|
+
|
|
8
|
+
from .normalizer import NormalizedEvent
|
|
9
|
+
from .utils import cef_escape, cef_escape_header
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
DEFAULT_MAPPING = {
|
|
15
|
+
"deviceVendor": "Generic",
|
|
16
|
+
"deviceProduct": "Syslog",
|
|
17
|
+
"deviceVersion": "1.0",
|
|
18
|
+
"eventClassId": "syslog",
|
|
19
|
+
"name": "%(message_short)s",
|
|
20
|
+
"severity_map": {
|
|
21
|
+
"0": "10",
|
|
22
|
+
"1": "8",
|
|
23
|
+
"2": "7",
|
|
24
|
+
"3": "6",
|
|
25
|
+
"4": "5",
|
|
26
|
+
"5": "3",
|
|
27
|
+
"6": "2",
|
|
28
|
+
"7": "1",
|
|
29
|
+
},
|
|
30
|
+
"extensions": {
|
|
31
|
+
"msg": "%(msg)s",
|
|
32
|
+
"rawEvent": "%(raw)s",
|
|
33
|
+
},
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass(slots=True)
|
|
38
|
+
class CEFEvent:
|
|
39
|
+
header: Dict[str, str]
|
|
40
|
+
extensions: Dict[str, str]
|
|
41
|
+
|
|
42
|
+
def render(self) -> str:
|
|
43
|
+
extension_str = " ".join(f"{k}={v}" for k, v in self.extensions.items())
|
|
44
|
+
return "CEF:0|{deviceVendor}|{deviceProduct}|{deviceVersion}|{eventClassId}|{name}|{severity}|{extension}".format(
|
|
45
|
+
extension=extension_str,
|
|
46
|
+
**self.header,
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class MappingResolver:
|
|
51
|
+
def __init__(self, mapping: Mapping[str, Any]) -> None:
|
|
52
|
+
merged = DEFAULT_MAPPING | mapping
|
|
53
|
+
self.mapping = merged
|
|
54
|
+
|
|
55
|
+
def resolve_header(self, fields: Mapping[str, Any]) -> Dict[str, str]:
|
|
56
|
+
header = {}
|
|
57
|
+
for key in ["deviceVendor", "deviceProduct", "deviceVersion", "eventClassId", "name"]:
|
|
58
|
+
template = self.mapping.get(key, DEFAULT_MAPPING[key])
|
|
59
|
+
header[key] = cef_escape_header(self._format(template, fields))
|
|
60
|
+
severity = self._resolve_severity(fields)
|
|
61
|
+
header["severity"] = severity
|
|
62
|
+
return header
|
|
63
|
+
|
|
64
|
+
def _resolve_severity(self, fields: Mapping[str, Any]) -> str:
|
|
65
|
+
severity_map = {**DEFAULT_MAPPING["severity_map"], **self.mapping.get("severity_map", {})}
|
|
66
|
+
value = fields.get("severity")
|
|
67
|
+
if value is None and "pri" in fields:
|
|
68
|
+
_, severity_value = divmod(int(fields.get("pri", 0)), 8)
|
|
69
|
+
value = severity_value
|
|
70
|
+
if value is None:
|
|
71
|
+
return "3"
|
|
72
|
+
return severity_map.get(str(value), str(value))
|
|
73
|
+
|
|
74
|
+
def resolve_extensions(self, fields: Mapping[str, Any]) -> Dict[str, str]:
|
|
75
|
+
extensions = {}
|
|
76
|
+
merged = DEFAULT_MAPPING["extensions"].copy()
|
|
77
|
+
merged.update(self.mapping.get("extensions", {}))
|
|
78
|
+
for key, template in merged.items():
|
|
79
|
+
value = self._format(template, fields)
|
|
80
|
+
if value:
|
|
81
|
+
extensions[key] = cef_escape(str(value))
|
|
82
|
+
return extensions
|
|
83
|
+
|
|
84
|
+
def _format(self, template: str, fields: Mapping[str, Any]) -> str:
|
|
85
|
+
try:
|
|
86
|
+
return template % fields
|
|
87
|
+
except KeyError as exc:
|
|
88
|
+
logger.debug("Missing key %s for template %s", exc, template)
|
|
89
|
+
return ""
|
|
90
|
+
except (ValueError, TypeError) as exc:
|
|
91
|
+
logger.warning("Invalid template %r: %s", template, exc)
|
|
92
|
+
return ""
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def build_cef(event: NormalizedEvent, mapping: Mapping[str, Any]) -> CEFEvent:
|
|
96
|
+
resolver = MappingResolver(mapping)
|
|
97
|
+
fields = event.as_field_dict()
|
|
98
|
+
header = resolver.resolve_header(fields)
|
|
99
|
+
extensions = resolver.resolve_extensions(fields)
|
|
100
|
+
return CEFEvent(header=header, extensions=extensions)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
__all__ = ["CEFEvent", "build_cef"]
|
syslogcef/cli.py
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import logging
|
|
5
|
+
import sys
|
|
6
|
+
import time
|
|
7
|
+
from functools import partial
|
|
8
|
+
from multiprocessing import Pool, cpu_count
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Iterable, Iterator, Optional
|
|
11
|
+
|
|
12
|
+
from .api import convert_line
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def iter_lines_from_sources(paths: Iterable[Path] | None) -> Iterator[str]:
|
|
18
|
+
if not paths:
|
|
19
|
+
for line in sys.stdin:
|
|
20
|
+
yield line.rstrip("\n")
|
|
21
|
+
return
|
|
22
|
+
|
|
23
|
+
for path in paths:
|
|
24
|
+
with path.open("r", encoding="utf-8", errors="replace") as handle:
|
|
25
|
+
for line in handle:
|
|
26
|
+
yield line.rstrip("\n")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def tail_file(path: Path, *, follow: bool = False) -> Iterator[str]:
|
|
30
|
+
with path.open("r", encoding="utf-8", errors="replace") as handle:
|
|
31
|
+
while True:
|
|
32
|
+
line = handle.readline()
|
|
33
|
+
if not line:
|
|
34
|
+
if not follow:
|
|
35
|
+
break
|
|
36
|
+
time.sleep(0.5)
|
|
37
|
+
continue
|
|
38
|
+
yield line.rstrip("\n")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def follow_files(
|
|
42
|
+
paths: Iterable[Path],
|
|
43
|
+
*,
|
|
44
|
+
poll_interval: float = 0.5,
|
|
45
|
+
max_idle_polls: Optional[int] = None,
|
|
46
|
+
) -> Iterator[str]:
|
|
47
|
+
"""Tail several files at once, round-robin, yielding lines as they appear.
|
|
48
|
+
|
|
49
|
+
``max_idle_polls`` bounds how many consecutive empty polls are allowed
|
|
50
|
+
before the generator stops; ``None`` follows forever.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
handles = [path.open("r", encoding="utf-8", errors="replace") for path in paths]
|
|
54
|
+
idle_polls = 0
|
|
55
|
+
try:
|
|
56
|
+
while True:
|
|
57
|
+
got_line = False
|
|
58
|
+
for handle in handles:
|
|
59
|
+
while True:
|
|
60
|
+
line = handle.readline()
|
|
61
|
+
if not line:
|
|
62
|
+
break
|
|
63
|
+
got_line = True
|
|
64
|
+
yield line.rstrip("\n")
|
|
65
|
+
if got_line:
|
|
66
|
+
idle_polls = 0
|
|
67
|
+
continue
|
|
68
|
+
idle_polls += 1
|
|
69
|
+
if max_idle_polls is not None and idle_polls >= max_idle_polls:
|
|
70
|
+
return
|
|
71
|
+
time.sleep(poll_interval)
|
|
72
|
+
finally:
|
|
73
|
+
for handle in handles:
|
|
74
|
+
handle.close()
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _convert_line_mp(line: str, mode: Optional[str], mapping: Optional[str]) -> str:
|
|
78
|
+
return convert_line(line, mode=mode, mapping=mapping)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def process_lines(
|
|
82
|
+
lines: Iterable[str],
|
|
83
|
+
*,
|
|
84
|
+
mode: Optional[str],
|
|
85
|
+
mapping: Optional[str],
|
|
86
|
+
use_multiprocessing: bool,
|
|
87
|
+
pool_size: Optional[int],
|
|
88
|
+
) -> Iterator[str]:
|
|
89
|
+
if use_multiprocessing:
|
|
90
|
+
size = pool_size or max(1, cpu_count() - 1)
|
|
91
|
+
with Pool(size) as pool:
|
|
92
|
+
worker = partial(_convert_line_mp, mode=mode, mapping=mapping)
|
|
93
|
+
for cef in pool.imap(worker, lines):
|
|
94
|
+
yield cef
|
|
95
|
+
else:
|
|
96
|
+
for line in lines:
|
|
97
|
+
yield convert_line(line, mode=mode, mapping=mapping)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def main(argv: Optional[list[str]] = None) -> int:
|
|
101
|
+
parser = argparse.ArgumentParser(description="Convert syslog lines to ArcSight CEF")
|
|
102
|
+
parser.add_argument("paths", nargs="*", type=Path, help="Input files (defaults to stdin)")
|
|
103
|
+
parser.add_argument("-o", "--output", type=Path, help="Output file")
|
|
104
|
+
parser.add_argument("-a", "--append", action="store_true", help="Append to --output instead of truncating it")
|
|
105
|
+
parser.add_argument("--mode", choices=["rfc3164", "rfc5424", "rsyslog_json", "rsyslog_file", "journald_json", "journald_short", "journald_iso"], help="Parser mode override")
|
|
106
|
+
parser.add_argument("--mapping", type=str, help="Mapping JSON file")
|
|
107
|
+
parser.add_argument("--tail", action="store_true", help="Follow file like tail -f")
|
|
108
|
+
parser.add_argument("--multiprocess", action="store_true", help="Process lines using a process pool")
|
|
109
|
+
parser.add_argument("--pool-size", type=int, help="Number of worker processes for --multiprocess")
|
|
110
|
+
parser.add_argument("--log-level", default="WARNING", help="Logging level")
|
|
111
|
+
|
|
112
|
+
args = parser.parse_args(argv)
|
|
113
|
+
logging.basicConfig(level=getattr(logging, args.log_level.upper(), logging.WARNING))
|
|
114
|
+
|
|
115
|
+
mapping = args.mapping
|
|
116
|
+
|
|
117
|
+
outputs: Iterator[str]
|
|
118
|
+
if args.tail and args.paths:
|
|
119
|
+
outputs = process_lines(follow_files(args.paths), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size)
|
|
120
|
+
else:
|
|
121
|
+
outputs = process_lines(iter_lines_from_sources(args.paths), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size)
|
|
122
|
+
|
|
123
|
+
if args.output:
|
|
124
|
+
with args.output.open("a" if args.append else "w", encoding="utf-8") as fp:
|
|
125
|
+
for cef in outputs:
|
|
126
|
+
fp.write(cef + "\n")
|
|
127
|
+
if args.tail:
|
|
128
|
+
fp.flush()
|
|
129
|
+
else:
|
|
130
|
+
for cef in outputs:
|
|
131
|
+
print(cef)
|
|
132
|
+
return 0
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
if __name__ == "__main__":
|
|
136
|
+
raise SystemExit(main())
|