syslog2cef 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,40 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ from importlib import resources
5
+ from typing import Any, Mapping
6
+
7
+ __all__ = [
8
+ "load_mapping",
9
+ "CISCO_ASA",
10
+ "CISCO_IOS",
11
+ "F5",
12
+ "LINUX",
13
+ "VMWARE",
14
+ ]
15
+
16
+
17
+ def _load(name: str) -> Mapping[str, Any]:
18
+ with resources.files(__package__).joinpath(f"{name}.json").open("r", encoding="utf-8") as fp:
19
+ return json.load(fp)
20
+
21
+
22
+ CISCO_ASA = _load("cisco_asa")
23
+ CISCO_IOS = _load("cisco_ios")
24
+ F5 = _load("f5")
25
+ LINUX = _load("linux")
26
+ VMWARE = _load("vmware")
27
+
28
+
29
+ def load_mapping(name: str) -> Mapping[str, Any]:
30
+ mapping = {
31
+ "cisco_asa": CISCO_ASA,
32
+ "cisco_ios": CISCO_IOS,
33
+ "f5": F5,
34
+ "linux": LINUX,
35
+ "vmware": VMWARE,
36
+ }
37
+ try:
38
+ return mapping[name]
39
+ except KeyError as exc:
40
+ raise KeyError(f"Unknown mapping {name!r}") from exc
@@ -0,0 +1,26 @@
1
+ {
2
+ "deviceVendor": "Cisco",
3
+ "deviceProduct": "ASA",
4
+ "deviceVersion": "auto",
5
+ "eventClassId": "asa.%(event_code)s",
6
+ "name": "%(message_short)s",
7
+ "severity_map": {
8
+ "0": "10",
9
+ "1": "8",
10
+ "2": "7",
11
+ "3": "6",
12
+ "4": "5",
13
+ "5": "3",
14
+ "6": "2",
15
+ "7": "1"
16
+ },
17
+ "extensions": {
18
+ "src": "%(src)s",
19
+ "dst": "%(dst)s",
20
+ "spt": "%(spt)s",
21
+ "dpt": "%(dpt)s",
22
+ "proto": "%(protocol)s",
23
+ "cs1Label": "rawEvent",
24
+ "cs1": "%(raw_kv)s"
25
+ }
26
+ }
@@ -0,0 +1,13 @@
1
+ {
2
+ "deviceVendor": "Cisco",
3
+ "deviceProduct": "IOS",
4
+ "deviceVersion": "auto",
5
+ "eventClassId": "ios.%(event_code)s",
6
+ "name": "%(message_short)s",
7
+ "extensions": {
8
+ "src": "%(src)s",
9
+ "dst": "%(dst)s",
10
+ "cs1Label": "rawEvent",
11
+ "cs1": "%(raw)s"
12
+ }
13
+ }
@@ -0,0 +1,13 @@
1
+ {
2
+ "deviceVendor": "F5 Networks",
3
+ "deviceProduct": "BIG-IP",
4
+ "deviceVersion": "auto",
5
+ "eventClassId": "f5.%(event_code)s",
6
+ "name": "%(message_short)s",
7
+ "extensions": {
8
+ "src": "%(src)s",
9
+ "dst": "%(dst)s",
10
+ "cs1Label": "rawEvent",
11
+ "cs1": "%(raw)s"
12
+ }
13
+ }
@@ -0,0 +1,13 @@
1
+ {
2
+ "deviceVendor": "Linux",
3
+ "deviceProduct": "Syslog",
4
+ "deviceVersion": "auto",
5
+ "eventClassId": "linux.syslog",
6
+ "name": "%(message_short)s",
7
+ "extensions": {
8
+ "dhost": "%(host)s",
9
+ "sproc": "%(app)s",
10
+ "cs1Label": "rawEvent",
11
+ "cs1": "%(raw)s"
12
+ }
13
+ }
@@ -0,0 +1,12 @@
1
+ {
2
+ "deviceVendor": "VMware",
3
+ "deviceProduct": "ESXi",
4
+ "deviceVersion": "auto",
5
+ "eventClassId": "vmware.%(event_code)s",
6
+ "name": "%(message_short)s",
7
+ "extensions": {
8
+ "dvc": "%(host)s",
9
+ "cs1Label": "rawEvent",
10
+ "cs1": "%(raw)s"
11
+ }
12
+ }
@@ -0,0 +1,106 @@
1
+ from __future__ import annotations
2
+
3
+ import logging
4
+ from dataclasses import dataclass, replace
5
+ from datetime import datetime, timezone
6
+ from typing import Any, Dict, Mapping, Optional
7
+
8
+ from .parsers import ParsedEvent
9
+ from .utils import ensure_tzaware, parse_key_value_pairs, sanitize_message
10
+
11
+ logger = logging.getLogger(__name__)
12
+
13
+
14
+ @dataclass(slots=True)
15
+ class NormalizedEvent(ParsedEvent):
16
+ kv: Dict[str, str] = None
17
+ extras: Dict[str, Any] = None
18
+
19
+ def __post_init__(self) -> None:
20
+ if self.kv is None:
21
+ self.kv = {}
22
+ if self.extras is None:
23
+ self.extras = {}
24
+
25
+ def as_field_dict(self) -> Dict[str, Any]:
26
+ base = {
27
+ "pri": self.pri,
28
+ "facility": self.facility,
29
+ "severity": self.severity,
30
+ "timestamp": self.ts.isoformat() if self.ts else None,
31
+ "ts": self.ts.isoformat() if self.ts else None,
32
+ "ts_orig": self.ts_orig,
33
+ "host": self.host,
34
+ "app": self.app,
35
+ "pid": self.pid,
36
+ "msgid": self.msgid,
37
+ "source_hint": self.source_hint,
38
+ "msg": self.msg,
39
+ }
40
+ base.update(self.sd)
41
+ base.update(self.kv)
42
+ base.update(self.extras)
43
+ return {k: v for k, v in base.items() if v is not None}
44
+
45
+
46
+ def normalize(event: ParsedEvent) -> NormalizedEvent:
47
+ ts = ensure_tzaware(event.ts)
48
+ msg = sanitize_message(event.msg)
49
+ kv = parse_key_value_pairs(msg)
50
+
51
+ extras = {
52
+ "message_length": len(msg),
53
+ "message_short": msg[:120],
54
+ "raw": event.raw,
55
+ "raw_kv": " ".join(f"{k}={v}" for k, v in kv.items()),
56
+ }
57
+
58
+ if event.source_hint in {"rfc3164", "rsyslog", "journald"} and event.host is None:
59
+ extras["host"] = "localhost"
60
+
61
+ normalized = NormalizedEvent(
62
+ pri=event.pri,
63
+ facility=event.facility,
64
+ severity=event.severity,
65
+ ts=ts or datetime.now(timezone.utc),
66
+ ts_orig=event.ts_orig,
67
+ host=event.host,
68
+ app=event.app,
69
+ pid=event.pid,
70
+ msgid=event.msgid,
71
+ sd=dict(event.sd),
72
+ msg=msg,
73
+ raw=event.raw,
74
+ source_hint=event.source_hint,
75
+ kv=kv,
76
+ extras=extras,
77
+ )
78
+
79
+ _derive_common_fields(normalized)
80
+ return normalized
81
+
82
+
83
+ def _derive_common_fields(event: NormalizedEvent) -> None:
84
+ if "src" not in event.kv and "src_ip" in event.kv:
85
+ event.kv.setdefault("src", event.kv["src_ip"])
86
+ if "dst" not in event.kv and "dst_ip" in event.kv:
87
+ event.kv.setdefault("dst", event.kv["dst_ip"])
88
+
89
+ if "event_code" not in event.kv:
90
+ code = _extract_event_code(event.msg)
91
+ if code:
92
+ event.kv["event_code"] = code
93
+
94
+
95
+ def _extract_event_code(msg: str) -> Optional[str]:
96
+ # Cisco ASA style: %ASA-6-106100: message
97
+ if msg.startswith("%") and ":" in msg:
98
+ head = msg.split(":", 1)[0]
99
+ return head.strip("%")
100
+ return None
101
+
102
+
103
+ __all__ = [
104
+ "NormalizedEvent",
105
+ "normalize",
106
+ ]
syslogcef/parsers.py ADDED
@@ -0,0 +1,380 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ import logging
5
+ import re
6
+ from dataclasses import dataclass, field
7
+ from datetime import datetime, timezone, timedelta
8
+ from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional, Tuple
9
+
10
+ from .utils import (
11
+ JournalJSONMapping,
12
+ convert_pri,
13
+ guess_hostname,
14
+ month_abbr_to_int,
15
+ parse_iso8601,
16
+ sanitize_message,
17
+ )
18
+
19
+ logger = logging.getLogger(__name__)
20
+
21
+
22
+ @dataclass(slots=True)
23
+ class ParsedEvent:
24
+ pri: Optional[int]
25
+ facility: Optional[int]
26
+ severity: Optional[int]
27
+ ts: Optional[datetime]
28
+ ts_orig: str
29
+ host: Optional[str]
30
+ app: Optional[str]
31
+ pid: Optional[str]
32
+ msgid: Optional[str]
33
+ sd: Dict[str, Any] = field(default_factory=dict)
34
+ msg: str = ""
35
+ raw: str = ""
36
+ source_hint: str = "unknown"
37
+
38
+
39
+ RFC3164_RE = re.compile(
40
+ r"^(?:<(?P<pri>\d+)>)?"
41
+ r"(?P<month>[A-Z][a-z]{2})\s+"
42
+ r"(?P<day>\d{1,2})\s"
43
+ r"(?P<time>\d{2}:\d{2}:\d{2})\s"
44
+ r"(?P<host>[^\s]+)\s"
45
+ r"(?:(?P<tag>[\w\-/\.]+)(?:\[(?P<pid>[^\]]+)\])?:\s?)?"
46
+ r"(?P<msg>.*)$"
47
+ )
48
+
49
+ RFC5424_RE = re.compile(
50
+ r"^<(?P<pri>\d+)>(?P<version>\d)\s"
51
+ r"(?P<timestamp>[^\s]+)\s"
52
+ r"(?P<host>[^\s]+)\s"
53
+ r"(?P<app>[^\s]+)\s"
54
+ r"(?P<procid>[^\s]+)\s"
55
+ r"(?P<msgid>[^\s]+)\s"
56
+ r"(?P<sd>(?:-|\[.*\]))\s?"
57
+ r"(?P<msg>.*)$"
58
+ )
59
+
60
+ RSYSLOG_FILE_RE = re.compile(
61
+ r"^(?P<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2}))\s"
62
+ r"(?P<host>[^\s]+)\s"
63
+ r"(?P<tag>[\w\-/\.]+)(?:\[(?P<pid>[^\]]+)\])?:\s?"
64
+ r"(?P<msg>.*)$"
65
+ )
66
+
67
+ JOURNALCTL_SHORT_RE = re.compile(
68
+ r"^(?P<month>[A-Z][a-z]{2})\s+"
69
+ r"(?P<day>\d{1,2})\s"
70
+ r"(?P<time>\d{2}:\d{2}:\d{2})\s"
71
+ r"(?P<host>[^\s]+)\s"
72
+ r"(?P<tag>[^:]+):\s?"
73
+ r"(?P<msg>.*)$"
74
+ )
75
+
76
+ JOURNALCTL_ISO_RE = re.compile(
77
+ r"^(?P<timestamp>\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?)(?:Z|[+-]\d{2}:?\d{2})?)\s"
78
+ r"(?P<host>[^\s]+)\s"
79
+ r"(?P<tag>[^:]+):\s?"
80
+ r"(?P<msg>.*)$"
81
+ )
82
+
83
+ KV_SPLIT_RE = re.compile(r"\s+|,|")
84
+
85
+
86
+ def _infer_timestamp(month: int, day: int, time_str: str, *, now: Optional[datetime]) -> datetime:
87
+ if now is None:
88
+ now = datetime.now(timezone.utc)
89
+ hour, minute, second = map(int, time_str.split(":"))
90
+ candidate = now.replace(month=month, day=day, hour=hour, minute=minute, second=second, microsecond=0)
91
+ # Handle rollover into previous year.
92
+ if candidate - now > timedelta(days=180):
93
+ candidate = candidate.replace(year=now.year - 1)
94
+ elif now - candidate > timedelta(days=180):
95
+ candidate = candidate.replace(year=now.year + 1)
96
+ return candidate
97
+
98
+
99
+ STRUCTURED_DATA_RE = re.compile(
100
+ r"\[(?P<id>[^\s\]]+)" # identifier
101
+ r"(?P<body>(?:\s+[^=]+=\"[^\"]*\"|\s+[^=]+=[^\s\]]+)*)" # key="value" or key=value
102
+ r"\]"
103
+ )
104
+ STRUCTURED_DATA_VALUE_RE = re.compile(r"\s+([^=]+)=(?:\"([^\"]*)\"|([^\s\]]+))")
105
+
106
+
107
+ class ParserError(RuntimeError):
108
+ pass
109
+
110
+
111
+ def parse_structured_data(sd_raw: str) -> Dict[str, Any]:
112
+ if sd_raw == "-":
113
+ return {}
114
+ data: Dict[str, Any] = {}
115
+ for match in STRUCTURED_DATA_RE.finditer(sd_raw):
116
+ sd_id = match.group("id")
117
+ body = match.group("body")
118
+ for key, quoted, bare in STRUCTURED_DATA_VALUE_RE.findall(body):
119
+ value = quoted or bare
120
+ data[f"{sd_id}.{key}"] = value
121
+ return data
122
+
123
+
124
+ def parse_rfc3164(line: str, *, now: Optional[datetime]) -> ParsedEvent | None:
125
+ match = RFC3164_RE.match(line)
126
+ if not match:
127
+ return None
128
+ gd = match.groupdict()
129
+ pri = int(gd["pri"]) if gd.get("pri") else None
130
+ facility, severity = convert_pri(pri)
131
+ month = month_abbr_to_int(gd["month"])
132
+ ts = _infer_timestamp(month, int(gd["day"]), gd["time"], now=now)
133
+ app = gd.get("tag")
134
+ msg = gd.get("msg", "")
135
+ return ParsedEvent(
136
+ pri=pri,
137
+ facility=facility,
138
+ severity=severity,
139
+ ts=ts,
140
+ ts_orig=f"{gd['month']} {gd['day']} {gd['time']}",
141
+ host=gd.get("host"),
142
+ app=app,
143
+ pid=gd.get("pid"),
144
+ msgid=None,
145
+ sd={},
146
+ msg=msg,
147
+ raw=line,
148
+ source_hint="rfc3164",
149
+ )
150
+
151
+
152
+ def parse_rfc5424(line: str) -> ParsedEvent | None:
153
+ match = RFC5424_RE.match(line)
154
+ if not match:
155
+ return None
156
+ gd = match.groupdict()
157
+ pri = int(gd["pri"]) if gd.get("pri") else None
158
+ facility, severity = convert_pri(pri)
159
+ ts = parse_iso8601(gd["timestamp"])
160
+ sd = parse_structured_data(gd.get("sd") or "-")
161
+ msg = gd.get("msg", "")
162
+ return ParsedEvent(
163
+ pri=pri,
164
+ facility=facility,
165
+ severity=severity,
166
+ ts=ts,
167
+ ts_orig=gd["timestamp"],
168
+ host=None if gd["host"] == "-" else gd["host"],
169
+ app=None if gd["app"] == "-" else gd["app"],
170
+ pid=None if gd["procid"] == "-" else gd["procid"],
171
+ msgid=None if gd["msgid"] == "-" else gd["msgid"],
172
+ sd=sd,
173
+ msg=msg,
174
+ raw=line,
175
+ source_hint="rfc5424",
176
+ )
177
+
178
+
179
+ def parse_rsyslog_json(line: str) -> ParsedEvent | None:
180
+ try:
181
+ data = json.loads(line)
182
+ except json.JSONDecodeError:
183
+ return None
184
+ pri = data.get("pri") or data.get("syslogpriority")
185
+ pri = int(pri) if pri is not None else None
186
+ facility, severity = convert_pri(pri)
187
+ timestamp = data.get("timestamp") or data.get("timegenerated")
188
+ ts = parse_iso8601(timestamp) if timestamp else None
189
+ host = data.get("hostname") or data.get("host")
190
+ app = data.get("app-name") or data.get("programname")
191
+ pid = data.get("procid") or data.get("pid")
192
+ msg = data.get("msg") or data.get("message") or ""
193
+ sd = {k: v for k, v in data.items() if isinstance(k, str) and k.startswith("structured-data")}
194
+ return ParsedEvent(
195
+ pri=pri,
196
+ facility=facility,
197
+ severity=severity,
198
+ ts=ts,
199
+ ts_orig=timestamp or "",
200
+ host=host,
201
+ app=app,
202
+ pid=str(pid) if pid is not None else None,
203
+ msgid=data.get("msgid"),
204
+ sd=sd,
205
+ msg=msg,
206
+ raw=line,
207
+ source_hint="rsyslog",
208
+ )
209
+
210
+
211
+ def parse_rsyslog_file(line: str) -> ParsedEvent | None:
212
+ match = RSYSLOG_FILE_RE.match(line)
213
+ if not match:
214
+ return None
215
+ gd = match.groupdict()
216
+ ts = parse_iso8601(gd["timestamp"])
217
+ pri = None
218
+ facility, severity = convert_pri(pri)
219
+ return ParsedEvent(
220
+ pri=pri,
221
+ facility=facility,
222
+ severity=severity,
223
+ ts=ts,
224
+ ts_orig=gd["timestamp"],
225
+ host=gd.get("host"),
226
+ app=gd.get("tag"),
227
+ pid=gd.get("pid"),
228
+ msgid=None,
229
+ sd={},
230
+ msg=gd.get("msg", ""),
231
+ raw=line,
232
+ source_hint="rsyslog",
233
+ )
234
+
235
+
236
+ def parse_journalctl_json(line: str) -> ParsedEvent | None:
237
+ try:
238
+ data = json.loads(line)
239
+ except json.JSONDecodeError:
240
+ return None
241
+ pri_raw = data.get("PRIORITY") or data.get("priority")
242
+ pri = int(pri_raw) if pri_raw is not None else None
243
+ facility, severity = convert_pri(pri)
244
+ timestamp = data.get("__REALTIME_TIMESTAMP") or data.get("timestamp")
245
+ ts = parse_iso8601(timestamp) if timestamp else None
246
+ host = data.get("_HOSTNAME") or data.get("hostname")
247
+ app = data.get("SYSLOG_IDENTIFIER") or data.get("app-name")
248
+ pid = data.get("_PID") or data.get("pid")
249
+ msg = data.get("MESSAGE") or data.get("msg") or ""
250
+ sd = JournalJSONMapping(data).to_structured_data()
251
+ return ParsedEvent(
252
+ pri=pri,
253
+ facility=facility,
254
+ severity=severity,
255
+ ts=ts,
256
+ ts_orig=str(timestamp or ""),
257
+ host=host,
258
+ app=app,
259
+ pid=str(pid) if pid is not None else None,
260
+ msgid=None,
261
+ sd=sd,
262
+ msg=msg,
263
+ raw=line,
264
+ source_hint="journald",
265
+ )
266
+
267
+
268
+ def parse_journalctl_short(line: str, *, now: Optional[datetime]) -> ParsedEvent | None:
269
+ match = JOURNALCTL_SHORT_RE.match(line)
270
+ if not match:
271
+ return None
272
+ gd = match.groupdict()
273
+ month = month_abbr_to_int(gd["month"])
274
+ ts = _infer_timestamp(month, int(gd["day"]), gd["time"], now=now)
275
+ return ParsedEvent(
276
+ pri=None,
277
+ facility=None,
278
+ severity=None,
279
+ ts=ts,
280
+ ts_orig=f"{gd['month']} {gd['day']} {gd['time']}",
281
+ host=gd.get("host"),
282
+ app=gd.get("tag"),
283
+ pid=None,
284
+ msgid=None,
285
+ sd={},
286
+ msg=gd.get("msg", ""),
287
+ raw=line,
288
+ source_hint="journald",
289
+ )
290
+
291
+
292
+ def parse_journalctl_iso(line: str) -> ParsedEvent | None:
293
+ match = JOURNALCTL_ISO_RE.match(line)
294
+ if not match:
295
+ return None
296
+ gd = match.groupdict()
297
+ ts = parse_iso8601(gd["timestamp"])
298
+ return ParsedEvent(
299
+ pri=None,
300
+ facility=None,
301
+ severity=None,
302
+ ts=ts,
303
+ ts_orig=gd["timestamp"],
304
+ host=gd.get("host"),
305
+ app=gd.get("tag"),
306
+ pid=None,
307
+ msgid=None,
308
+ sd={},
309
+ msg=gd.get("msg", ""),
310
+ raw=line,
311
+ source_hint="journald",
312
+ )
313
+
314
+
315
+ PARSERS = {
316
+ "rfc3164": parse_rfc3164,
317
+ "rfc5424": parse_rfc5424,
318
+ "rsyslog_json": parse_rsyslog_json,
319
+ "rsyslog_file": parse_rsyslog_file,
320
+ "journald_json": parse_journalctl_json,
321
+ "journald_short": parse_journalctl_short,
322
+ "journald_iso": parse_journalctl_iso,
323
+ }
324
+
325
+
326
+ def autodetect_and_parse(line: str, *, mode: Optional[str] = None, now: Optional[datetime] = None) -> ParsedEvent:
327
+ detectors = [
328
+ ("journald_json", lambda s: s.lstrip().startswith("{")),
329
+ ("rsyslog_json", lambda s: s.lstrip().startswith("{")),
330
+ ("rfc5424", lambda s: s.startswith("<") and ">" in s[:10]),
331
+ ("rsyslog_file", lambda s: bool(RSYSLOG_FILE_RE.match(s))),
332
+ ("journald_iso", lambda s: bool(JOURNALCTL_ISO_RE.match(s))),
333
+ ("journald_short", lambda s: bool(JOURNALCTL_SHORT_RE.match(s))),
334
+ ("rfc3164", lambda s: bool(RFC3164_RE.match(s))),
335
+ ]
336
+
337
+ if mode:
338
+ parser = PARSERS.get(mode)
339
+ if parser is None:
340
+ raise ParserError(f"Unknown parser mode: {mode}")
341
+ result = parser(line, now=now) if "now" in parser.__code__.co_varnames else parser(line)
342
+ if result is None:
343
+ raise ParserError(f"Parser '{mode}' could not parse line")
344
+ return result
345
+
346
+ for key, detector in detectors:
347
+ try:
348
+ if detector(line):
349
+ parser = PARSERS[key]
350
+ result = parser(line, now=now) if "now" in parser.__code__.co_varnames else parser(line)
351
+ if result is not None:
352
+ return result
353
+ except Exception:
354
+ logger.exception("Parser %s failed for line", key)
355
+ continue
356
+
357
+ # Fallback to raw message if nothing matches.
358
+ logger.debug("Falling back to raw parser for line: %s", line)
359
+ return ParsedEvent(
360
+ pri=None,
361
+ facility=None,
362
+ severity=None,
363
+ ts=None,
364
+ ts_orig="",
365
+ host=guess_hostname(),
366
+ app=None,
367
+ pid=None,
368
+ msgid=None,
369
+ sd={},
370
+ msg=sanitize_message(line),
371
+ raw=line,
372
+ source_hint="unknown",
373
+ )
374
+
375
+
376
+ __all__ = [
377
+ "ParsedEvent",
378
+ "ParserError",
379
+ "autodetect_and_parse",
380
+ ]