syslog2cef 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- syslog2cef-0.1.0.dist-info/METADATA +202 -0
- syslog2cef-0.1.0.dist-info/RECORD +20 -0
- syslog2cef-0.1.0.dist-info/WHEEL +5 -0
- syslog2cef-0.1.0.dist-info/entry_points.txt +2 -0
- syslog2cef-0.1.0.dist-info/licenses/LICENSE +21 -0
- syslog2cef-0.1.0.dist-info/top_level.txt +1 -0
- syslogcef/__init__.py +19 -0
- syslogcef/__main__.py +7 -0
- syslogcef/api.py +127 -0
- syslogcef/cef.py +103 -0
- syslogcef/cli.py +136 -0
- syslogcef/mappings/__init__.py +40 -0
- syslogcef/mappings/cisco_asa.json +26 -0
- syslogcef/mappings/cisco_ios.json +13 -0
- syslogcef/mappings/f5.json +13 -0
- syslogcef/mappings/linux.json +13 -0
- syslogcef/mappings/vmware.json +12 -0
- syslogcef/normalizer.py +106 -0
- syslogcef/parsers.py +380 -0
- syslogcef/utils.py +144 -0
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from importlib import resources
|
|
5
|
+
from typing import Any, Mapping
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"load_mapping",
|
|
9
|
+
"CISCO_ASA",
|
|
10
|
+
"CISCO_IOS",
|
|
11
|
+
"F5",
|
|
12
|
+
"LINUX",
|
|
13
|
+
"VMWARE",
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _load(name: str) -> Mapping[str, Any]:
|
|
18
|
+
with resources.files(__package__).joinpath(f"{name}.json").open("r", encoding="utf-8") as fp:
|
|
19
|
+
return json.load(fp)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
CISCO_ASA = _load("cisco_asa")
|
|
23
|
+
CISCO_IOS = _load("cisco_ios")
|
|
24
|
+
F5 = _load("f5")
|
|
25
|
+
LINUX = _load("linux")
|
|
26
|
+
VMWARE = _load("vmware")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def load_mapping(name: str) -> Mapping[str, Any]:
|
|
30
|
+
mapping = {
|
|
31
|
+
"cisco_asa": CISCO_ASA,
|
|
32
|
+
"cisco_ios": CISCO_IOS,
|
|
33
|
+
"f5": F5,
|
|
34
|
+
"linux": LINUX,
|
|
35
|
+
"vmware": VMWARE,
|
|
36
|
+
}
|
|
37
|
+
try:
|
|
38
|
+
return mapping[name]
|
|
39
|
+
except KeyError as exc:
|
|
40
|
+
raise KeyError(f"Unknown mapping {name!r}") from exc
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
{
|
|
2
|
+
"deviceVendor": "Cisco",
|
|
3
|
+
"deviceProduct": "ASA",
|
|
4
|
+
"deviceVersion": "auto",
|
|
5
|
+
"eventClassId": "asa.%(event_code)s",
|
|
6
|
+
"name": "%(message_short)s",
|
|
7
|
+
"severity_map": {
|
|
8
|
+
"0": "10",
|
|
9
|
+
"1": "8",
|
|
10
|
+
"2": "7",
|
|
11
|
+
"3": "6",
|
|
12
|
+
"4": "5",
|
|
13
|
+
"5": "3",
|
|
14
|
+
"6": "2",
|
|
15
|
+
"7": "1"
|
|
16
|
+
},
|
|
17
|
+
"extensions": {
|
|
18
|
+
"src": "%(src)s",
|
|
19
|
+
"dst": "%(dst)s",
|
|
20
|
+
"spt": "%(spt)s",
|
|
21
|
+
"dpt": "%(dpt)s",
|
|
22
|
+
"proto": "%(protocol)s",
|
|
23
|
+
"cs1Label": "rawEvent",
|
|
24
|
+
"cs1": "%(raw_kv)s"
|
|
25
|
+
}
|
|
26
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
{
|
|
2
|
+
"deviceVendor": "Cisco",
|
|
3
|
+
"deviceProduct": "IOS",
|
|
4
|
+
"deviceVersion": "auto",
|
|
5
|
+
"eventClassId": "ios.%(event_code)s",
|
|
6
|
+
"name": "%(message_short)s",
|
|
7
|
+
"extensions": {
|
|
8
|
+
"src": "%(src)s",
|
|
9
|
+
"dst": "%(dst)s",
|
|
10
|
+
"cs1Label": "rawEvent",
|
|
11
|
+
"cs1": "%(raw)s"
|
|
12
|
+
}
|
|
13
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
{
|
|
2
|
+
"deviceVendor": "F5 Networks",
|
|
3
|
+
"deviceProduct": "BIG-IP",
|
|
4
|
+
"deviceVersion": "auto",
|
|
5
|
+
"eventClassId": "f5.%(event_code)s",
|
|
6
|
+
"name": "%(message_short)s",
|
|
7
|
+
"extensions": {
|
|
8
|
+
"src": "%(src)s",
|
|
9
|
+
"dst": "%(dst)s",
|
|
10
|
+
"cs1Label": "rawEvent",
|
|
11
|
+
"cs1": "%(raw)s"
|
|
12
|
+
}
|
|
13
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
{
|
|
2
|
+
"deviceVendor": "Linux",
|
|
3
|
+
"deviceProduct": "Syslog",
|
|
4
|
+
"deviceVersion": "auto",
|
|
5
|
+
"eventClassId": "linux.syslog",
|
|
6
|
+
"name": "%(message_short)s",
|
|
7
|
+
"extensions": {
|
|
8
|
+
"dhost": "%(host)s",
|
|
9
|
+
"sproc": "%(app)s",
|
|
10
|
+
"cs1Label": "rawEvent",
|
|
11
|
+
"cs1": "%(raw)s"
|
|
12
|
+
}
|
|
13
|
+
}
|
syslogcef/normalizer.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from dataclasses import dataclass, replace
|
|
5
|
+
from datetime import datetime, timezone
|
|
6
|
+
from typing import Any, Dict, Mapping, Optional
|
|
7
|
+
|
|
8
|
+
from .parsers import ParsedEvent
|
|
9
|
+
from .utils import ensure_tzaware, parse_key_value_pairs, sanitize_message
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(slots=True)
|
|
15
|
+
class NormalizedEvent(ParsedEvent):
|
|
16
|
+
kv: Dict[str, str] = None
|
|
17
|
+
extras: Dict[str, Any] = None
|
|
18
|
+
|
|
19
|
+
def __post_init__(self) -> None:
|
|
20
|
+
if self.kv is None:
|
|
21
|
+
self.kv = {}
|
|
22
|
+
if self.extras is None:
|
|
23
|
+
self.extras = {}
|
|
24
|
+
|
|
25
|
+
def as_field_dict(self) -> Dict[str, Any]:
|
|
26
|
+
base = {
|
|
27
|
+
"pri": self.pri,
|
|
28
|
+
"facility": self.facility,
|
|
29
|
+
"severity": self.severity,
|
|
30
|
+
"timestamp": self.ts.isoformat() if self.ts else None,
|
|
31
|
+
"ts": self.ts.isoformat() if self.ts else None,
|
|
32
|
+
"ts_orig": self.ts_orig,
|
|
33
|
+
"host": self.host,
|
|
34
|
+
"app": self.app,
|
|
35
|
+
"pid": self.pid,
|
|
36
|
+
"msgid": self.msgid,
|
|
37
|
+
"source_hint": self.source_hint,
|
|
38
|
+
"msg": self.msg,
|
|
39
|
+
}
|
|
40
|
+
base.update(self.sd)
|
|
41
|
+
base.update(self.kv)
|
|
42
|
+
base.update(self.extras)
|
|
43
|
+
return {k: v for k, v in base.items() if v is not None}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def normalize(event: ParsedEvent) -> NormalizedEvent:
|
|
47
|
+
ts = ensure_tzaware(event.ts)
|
|
48
|
+
msg = sanitize_message(event.msg)
|
|
49
|
+
kv = parse_key_value_pairs(msg)
|
|
50
|
+
|
|
51
|
+
extras = {
|
|
52
|
+
"message_length": len(msg),
|
|
53
|
+
"message_short": msg[:120],
|
|
54
|
+
"raw": event.raw,
|
|
55
|
+
"raw_kv": " ".join(f"{k}={v}" for k, v in kv.items()),
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
if event.source_hint in {"rfc3164", "rsyslog", "journald"} and event.host is None:
|
|
59
|
+
extras["host"] = "localhost"
|
|
60
|
+
|
|
61
|
+
normalized = NormalizedEvent(
|
|
62
|
+
pri=event.pri,
|
|
63
|
+
facility=event.facility,
|
|
64
|
+
severity=event.severity,
|
|
65
|
+
ts=ts or datetime.now(timezone.utc),
|
|
66
|
+
ts_orig=event.ts_orig,
|
|
67
|
+
host=event.host,
|
|
68
|
+
app=event.app,
|
|
69
|
+
pid=event.pid,
|
|
70
|
+
msgid=event.msgid,
|
|
71
|
+
sd=dict(event.sd),
|
|
72
|
+
msg=msg,
|
|
73
|
+
raw=event.raw,
|
|
74
|
+
source_hint=event.source_hint,
|
|
75
|
+
kv=kv,
|
|
76
|
+
extras=extras,
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
_derive_common_fields(normalized)
|
|
80
|
+
return normalized
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _derive_common_fields(event: NormalizedEvent) -> None:
|
|
84
|
+
if "src" not in event.kv and "src_ip" in event.kv:
|
|
85
|
+
event.kv.setdefault("src", event.kv["src_ip"])
|
|
86
|
+
if "dst" not in event.kv and "dst_ip" in event.kv:
|
|
87
|
+
event.kv.setdefault("dst", event.kv["dst_ip"])
|
|
88
|
+
|
|
89
|
+
if "event_code" not in event.kv:
|
|
90
|
+
code = _extract_event_code(event.msg)
|
|
91
|
+
if code:
|
|
92
|
+
event.kv["event_code"] = code
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _extract_event_code(msg: str) -> Optional[str]:
|
|
96
|
+
# Cisco ASA style: %ASA-6-106100: message
|
|
97
|
+
if msg.startswith("%") and ":" in msg:
|
|
98
|
+
head = msg.split(":", 1)[0]
|
|
99
|
+
return head.strip("%")
|
|
100
|
+
return None
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
__all__ = [
|
|
104
|
+
"NormalizedEvent",
|
|
105
|
+
"normalize",
|
|
106
|
+
]
|
syslogcef/parsers.py
ADDED
|
@@ -0,0 +1,380 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import logging
|
|
5
|
+
import re
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from datetime import datetime, timezone, timedelta
|
|
8
|
+
from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional, Tuple
|
|
9
|
+
|
|
10
|
+
from .utils import (
|
|
11
|
+
JournalJSONMapping,
|
|
12
|
+
convert_pri,
|
|
13
|
+
guess_hostname,
|
|
14
|
+
month_abbr_to_int,
|
|
15
|
+
parse_iso8601,
|
|
16
|
+
sanitize_message,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
logger = logging.getLogger(__name__)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(slots=True)
|
|
23
|
+
class ParsedEvent:
|
|
24
|
+
pri: Optional[int]
|
|
25
|
+
facility: Optional[int]
|
|
26
|
+
severity: Optional[int]
|
|
27
|
+
ts: Optional[datetime]
|
|
28
|
+
ts_orig: str
|
|
29
|
+
host: Optional[str]
|
|
30
|
+
app: Optional[str]
|
|
31
|
+
pid: Optional[str]
|
|
32
|
+
msgid: Optional[str]
|
|
33
|
+
sd: Dict[str, Any] = field(default_factory=dict)
|
|
34
|
+
msg: str = ""
|
|
35
|
+
raw: str = ""
|
|
36
|
+
source_hint: str = "unknown"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
RFC3164_RE = re.compile(
|
|
40
|
+
r"^(?:<(?P<pri>\d+)>)?"
|
|
41
|
+
r"(?P<month>[A-Z][a-z]{2})\s+"
|
|
42
|
+
r"(?P<day>\d{1,2})\s"
|
|
43
|
+
r"(?P<time>\d{2}:\d{2}:\d{2})\s"
|
|
44
|
+
r"(?P<host>[^\s]+)\s"
|
|
45
|
+
r"(?:(?P<tag>[\w\-/\.]+)(?:\[(?P<pid>[^\]]+)\])?:\s?)?"
|
|
46
|
+
r"(?P<msg>.*)$"
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
RFC5424_RE = re.compile(
|
|
50
|
+
r"^<(?P<pri>\d+)>(?P<version>\d)\s"
|
|
51
|
+
r"(?P<timestamp>[^\s]+)\s"
|
|
52
|
+
r"(?P<host>[^\s]+)\s"
|
|
53
|
+
r"(?P<app>[^\s]+)\s"
|
|
54
|
+
r"(?P<procid>[^\s]+)\s"
|
|
55
|
+
r"(?P<msgid>[^\s]+)\s"
|
|
56
|
+
r"(?P<sd>(?:-|\[.*\]))\s?"
|
|
57
|
+
r"(?P<msg>.*)$"
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
RSYSLOG_FILE_RE = re.compile(
|
|
61
|
+
r"^(?P<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2}))\s"
|
|
62
|
+
r"(?P<host>[^\s]+)\s"
|
|
63
|
+
r"(?P<tag>[\w\-/\.]+)(?:\[(?P<pid>[^\]]+)\])?:\s?"
|
|
64
|
+
r"(?P<msg>.*)$"
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
JOURNALCTL_SHORT_RE = re.compile(
|
|
68
|
+
r"^(?P<month>[A-Z][a-z]{2})\s+"
|
|
69
|
+
r"(?P<day>\d{1,2})\s"
|
|
70
|
+
r"(?P<time>\d{2}:\d{2}:\d{2})\s"
|
|
71
|
+
r"(?P<host>[^\s]+)\s"
|
|
72
|
+
r"(?P<tag>[^:]+):\s?"
|
|
73
|
+
r"(?P<msg>.*)$"
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
JOURNALCTL_ISO_RE = re.compile(
|
|
77
|
+
r"^(?P<timestamp>\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?)(?:Z|[+-]\d{2}:?\d{2})?)\s"
|
|
78
|
+
r"(?P<host>[^\s]+)\s"
|
|
79
|
+
r"(?P<tag>[^:]+):\s?"
|
|
80
|
+
r"(?P<msg>.*)$"
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
KV_SPLIT_RE = re.compile(r"\s+|,|")
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _infer_timestamp(month: int, day: int, time_str: str, *, now: Optional[datetime]) -> datetime:
|
|
87
|
+
if now is None:
|
|
88
|
+
now = datetime.now(timezone.utc)
|
|
89
|
+
hour, minute, second = map(int, time_str.split(":"))
|
|
90
|
+
candidate = now.replace(month=month, day=day, hour=hour, minute=minute, second=second, microsecond=0)
|
|
91
|
+
# Handle rollover into previous year.
|
|
92
|
+
if candidate - now > timedelta(days=180):
|
|
93
|
+
candidate = candidate.replace(year=now.year - 1)
|
|
94
|
+
elif now - candidate > timedelta(days=180):
|
|
95
|
+
candidate = candidate.replace(year=now.year + 1)
|
|
96
|
+
return candidate
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
STRUCTURED_DATA_RE = re.compile(
|
|
100
|
+
r"\[(?P<id>[^\s\]]+)" # identifier
|
|
101
|
+
r"(?P<body>(?:\s+[^=]+=\"[^\"]*\"|\s+[^=]+=[^\s\]]+)*)" # key="value" or key=value
|
|
102
|
+
r"\]"
|
|
103
|
+
)
|
|
104
|
+
STRUCTURED_DATA_VALUE_RE = re.compile(r"\s+([^=]+)=(?:\"([^\"]*)\"|([^\s\]]+))")
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class ParserError(RuntimeError):
|
|
108
|
+
pass
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def parse_structured_data(sd_raw: str) -> Dict[str, Any]:
|
|
112
|
+
if sd_raw == "-":
|
|
113
|
+
return {}
|
|
114
|
+
data: Dict[str, Any] = {}
|
|
115
|
+
for match in STRUCTURED_DATA_RE.finditer(sd_raw):
|
|
116
|
+
sd_id = match.group("id")
|
|
117
|
+
body = match.group("body")
|
|
118
|
+
for key, quoted, bare in STRUCTURED_DATA_VALUE_RE.findall(body):
|
|
119
|
+
value = quoted or bare
|
|
120
|
+
data[f"{sd_id}.{key}"] = value
|
|
121
|
+
return data
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def parse_rfc3164(line: str, *, now: Optional[datetime]) -> ParsedEvent | None:
|
|
125
|
+
match = RFC3164_RE.match(line)
|
|
126
|
+
if not match:
|
|
127
|
+
return None
|
|
128
|
+
gd = match.groupdict()
|
|
129
|
+
pri = int(gd["pri"]) if gd.get("pri") else None
|
|
130
|
+
facility, severity = convert_pri(pri)
|
|
131
|
+
month = month_abbr_to_int(gd["month"])
|
|
132
|
+
ts = _infer_timestamp(month, int(gd["day"]), gd["time"], now=now)
|
|
133
|
+
app = gd.get("tag")
|
|
134
|
+
msg = gd.get("msg", "")
|
|
135
|
+
return ParsedEvent(
|
|
136
|
+
pri=pri,
|
|
137
|
+
facility=facility,
|
|
138
|
+
severity=severity,
|
|
139
|
+
ts=ts,
|
|
140
|
+
ts_orig=f"{gd['month']} {gd['day']} {gd['time']}",
|
|
141
|
+
host=gd.get("host"),
|
|
142
|
+
app=app,
|
|
143
|
+
pid=gd.get("pid"),
|
|
144
|
+
msgid=None,
|
|
145
|
+
sd={},
|
|
146
|
+
msg=msg,
|
|
147
|
+
raw=line,
|
|
148
|
+
source_hint="rfc3164",
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def parse_rfc5424(line: str) -> ParsedEvent | None:
|
|
153
|
+
match = RFC5424_RE.match(line)
|
|
154
|
+
if not match:
|
|
155
|
+
return None
|
|
156
|
+
gd = match.groupdict()
|
|
157
|
+
pri = int(gd["pri"]) if gd.get("pri") else None
|
|
158
|
+
facility, severity = convert_pri(pri)
|
|
159
|
+
ts = parse_iso8601(gd["timestamp"])
|
|
160
|
+
sd = parse_structured_data(gd.get("sd") or "-")
|
|
161
|
+
msg = gd.get("msg", "")
|
|
162
|
+
return ParsedEvent(
|
|
163
|
+
pri=pri,
|
|
164
|
+
facility=facility,
|
|
165
|
+
severity=severity,
|
|
166
|
+
ts=ts,
|
|
167
|
+
ts_orig=gd["timestamp"],
|
|
168
|
+
host=None if gd["host"] == "-" else gd["host"],
|
|
169
|
+
app=None if gd["app"] == "-" else gd["app"],
|
|
170
|
+
pid=None if gd["procid"] == "-" else gd["procid"],
|
|
171
|
+
msgid=None if gd["msgid"] == "-" else gd["msgid"],
|
|
172
|
+
sd=sd,
|
|
173
|
+
msg=msg,
|
|
174
|
+
raw=line,
|
|
175
|
+
source_hint="rfc5424",
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def parse_rsyslog_json(line: str) -> ParsedEvent | None:
|
|
180
|
+
try:
|
|
181
|
+
data = json.loads(line)
|
|
182
|
+
except json.JSONDecodeError:
|
|
183
|
+
return None
|
|
184
|
+
pri = data.get("pri") or data.get("syslogpriority")
|
|
185
|
+
pri = int(pri) if pri is not None else None
|
|
186
|
+
facility, severity = convert_pri(pri)
|
|
187
|
+
timestamp = data.get("timestamp") or data.get("timegenerated")
|
|
188
|
+
ts = parse_iso8601(timestamp) if timestamp else None
|
|
189
|
+
host = data.get("hostname") or data.get("host")
|
|
190
|
+
app = data.get("app-name") or data.get("programname")
|
|
191
|
+
pid = data.get("procid") or data.get("pid")
|
|
192
|
+
msg = data.get("msg") or data.get("message") or ""
|
|
193
|
+
sd = {k: v for k, v in data.items() if isinstance(k, str) and k.startswith("structured-data")}
|
|
194
|
+
return ParsedEvent(
|
|
195
|
+
pri=pri,
|
|
196
|
+
facility=facility,
|
|
197
|
+
severity=severity,
|
|
198
|
+
ts=ts,
|
|
199
|
+
ts_orig=timestamp or "",
|
|
200
|
+
host=host,
|
|
201
|
+
app=app,
|
|
202
|
+
pid=str(pid) if pid is not None else None,
|
|
203
|
+
msgid=data.get("msgid"),
|
|
204
|
+
sd=sd,
|
|
205
|
+
msg=msg,
|
|
206
|
+
raw=line,
|
|
207
|
+
source_hint="rsyslog",
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def parse_rsyslog_file(line: str) -> ParsedEvent | None:
|
|
212
|
+
match = RSYSLOG_FILE_RE.match(line)
|
|
213
|
+
if not match:
|
|
214
|
+
return None
|
|
215
|
+
gd = match.groupdict()
|
|
216
|
+
ts = parse_iso8601(gd["timestamp"])
|
|
217
|
+
pri = None
|
|
218
|
+
facility, severity = convert_pri(pri)
|
|
219
|
+
return ParsedEvent(
|
|
220
|
+
pri=pri,
|
|
221
|
+
facility=facility,
|
|
222
|
+
severity=severity,
|
|
223
|
+
ts=ts,
|
|
224
|
+
ts_orig=gd["timestamp"],
|
|
225
|
+
host=gd.get("host"),
|
|
226
|
+
app=gd.get("tag"),
|
|
227
|
+
pid=gd.get("pid"),
|
|
228
|
+
msgid=None,
|
|
229
|
+
sd={},
|
|
230
|
+
msg=gd.get("msg", ""),
|
|
231
|
+
raw=line,
|
|
232
|
+
source_hint="rsyslog",
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def parse_journalctl_json(line: str) -> ParsedEvent | None:
|
|
237
|
+
try:
|
|
238
|
+
data = json.loads(line)
|
|
239
|
+
except json.JSONDecodeError:
|
|
240
|
+
return None
|
|
241
|
+
pri_raw = data.get("PRIORITY") or data.get("priority")
|
|
242
|
+
pri = int(pri_raw) if pri_raw is not None else None
|
|
243
|
+
facility, severity = convert_pri(pri)
|
|
244
|
+
timestamp = data.get("__REALTIME_TIMESTAMP") or data.get("timestamp")
|
|
245
|
+
ts = parse_iso8601(timestamp) if timestamp else None
|
|
246
|
+
host = data.get("_HOSTNAME") or data.get("hostname")
|
|
247
|
+
app = data.get("SYSLOG_IDENTIFIER") or data.get("app-name")
|
|
248
|
+
pid = data.get("_PID") or data.get("pid")
|
|
249
|
+
msg = data.get("MESSAGE") or data.get("msg") or ""
|
|
250
|
+
sd = JournalJSONMapping(data).to_structured_data()
|
|
251
|
+
return ParsedEvent(
|
|
252
|
+
pri=pri,
|
|
253
|
+
facility=facility,
|
|
254
|
+
severity=severity,
|
|
255
|
+
ts=ts,
|
|
256
|
+
ts_orig=str(timestamp or ""),
|
|
257
|
+
host=host,
|
|
258
|
+
app=app,
|
|
259
|
+
pid=str(pid) if pid is not None else None,
|
|
260
|
+
msgid=None,
|
|
261
|
+
sd=sd,
|
|
262
|
+
msg=msg,
|
|
263
|
+
raw=line,
|
|
264
|
+
source_hint="journald",
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def parse_journalctl_short(line: str, *, now: Optional[datetime]) -> ParsedEvent | None:
|
|
269
|
+
match = JOURNALCTL_SHORT_RE.match(line)
|
|
270
|
+
if not match:
|
|
271
|
+
return None
|
|
272
|
+
gd = match.groupdict()
|
|
273
|
+
month = month_abbr_to_int(gd["month"])
|
|
274
|
+
ts = _infer_timestamp(month, int(gd["day"]), gd["time"], now=now)
|
|
275
|
+
return ParsedEvent(
|
|
276
|
+
pri=None,
|
|
277
|
+
facility=None,
|
|
278
|
+
severity=None,
|
|
279
|
+
ts=ts,
|
|
280
|
+
ts_orig=f"{gd['month']} {gd['day']} {gd['time']}",
|
|
281
|
+
host=gd.get("host"),
|
|
282
|
+
app=gd.get("tag"),
|
|
283
|
+
pid=None,
|
|
284
|
+
msgid=None,
|
|
285
|
+
sd={},
|
|
286
|
+
msg=gd.get("msg", ""),
|
|
287
|
+
raw=line,
|
|
288
|
+
source_hint="journald",
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def parse_journalctl_iso(line: str) -> ParsedEvent | None:
|
|
293
|
+
match = JOURNALCTL_ISO_RE.match(line)
|
|
294
|
+
if not match:
|
|
295
|
+
return None
|
|
296
|
+
gd = match.groupdict()
|
|
297
|
+
ts = parse_iso8601(gd["timestamp"])
|
|
298
|
+
return ParsedEvent(
|
|
299
|
+
pri=None,
|
|
300
|
+
facility=None,
|
|
301
|
+
severity=None,
|
|
302
|
+
ts=ts,
|
|
303
|
+
ts_orig=gd["timestamp"],
|
|
304
|
+
host=gd.get("host"),
|
|
305
|
+
app=gd.get("tag"),
|
|
306
|
+
pid=None,
|
|
307
|
+
msgid=None,
|
|
308
|
+
sd={},
|
|
309
|
+
msg=gd.get("msg", ""),
|
|
310
|
+
raw=line,
|
|
311
|
+
source_hint="journald",
|
|
312
|
+
)
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
PARSERS = {
|
|
316
|
+
"rfc3164": parse_rfc3164,
|
|
317
|
+
"rfc5424": parse_rfc5424,
|
|
318
|
+
"rsyslog_json": parse_rsyslog_json,
|
|
319
|
+
"rsyslog_file": parse_rsyslog_file,
|
|
320
|
+
"journald_json": parse_journalctl_json,
|
|
321
|
+
"journald_short": parse_journalctl_short,
|
|
322
|
+
"journald_iso": parse_journalctl_iso,
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def autodetect_and_parse(line: str, *, mode: Optional[str] = None, now: Optional[datetime] = None) -> ParsedEvent:
|
|
327
|
+
detectors = [
|
|
328
|
+
("journald_json", lambda s: s.lstrip().startswith("{")),
|
|
329
|
+
("rsyslog_json", lambda s: s.lstrip().startswith("{")),
|
|
330
|
+
("rfc5424", lambda s: s.startswith("<") and ">" in s[:10]),
|
|
331
|
+
("rsyslog_file", lambda s: bool(RSYSLOG_FILE_RE.match(s))),
|
|
332
|
+
("journald_iso", lambda s: bool(JOURNALCTL_ISO_RE.match(s))),
|
|
333
|
+
("journald_short", lambda s: bool(JOURNALCTL_SHORT_RE.match(s))),
|
|
334
|
+
("rfc3164", lambda s: bool(RFC3164_RE.match(s))),
|
|
335
|
+
]
|
|
336
|
+
|
|
337
|
+
if mode:
|
|
338
|
+
parser = PARSERS.get(mode)
|
|
339
|
+
if parser is None:
|
|
340
|
+
raise ParserError(f"Unknown parser mode: {mode}")
|
|
341
|
+
result = parser(line, now=now) if "now" in parser.__code__.co_varnames else parser(line)
|
|
342
|
+
if result is None:
|
|
343
|
+
raise ParserError(f"Parser '{mode}' could not parse line")
|
|
344
|
+
return result
|
|
345
|
+
|
|
346
|
+
for key, detector in detectors:
|
|
347
|
+
try:
|
|
348
|
+
if detector(line):
|
|
349
|
+
parser = PARSERS[key]
|
|
350
|
+
result = parser(line, now=now) if "now" in parser.__code__.co_varnames else parser(line)
|
|
351
|
+
if result is not None:
|
|
352
|
+
return result
|
|
353
|
+
except Exception:
|
|
354
|
+
logger.exception("Parser %s failed for line", key)
|
|
355
|
+
continue
|
|
356
|
+
|
|
357
|
+
# Fallback to raw message if nothing matches.
|
|
358
|
+
logger.debug("Falling back to raw parser for line: %s", line)
|
|
359
|
+
return ParsedEvent(
|
|
360
|
+
pri=None,
|
|
361
|
+
facility=None,
|
|
362
|
+
severity=None,
|
|
363
|
+
ts=None,
|
|
364
|
+
ts_orig="",
|
|
365
|
+
host=guess_hostname(),
|
|
366
|
+
app=None,
|
|
367
|
+
pid=None,
|
|
368
|
+
msgid=None,
|
|
369
|
+
sd={},
|
|
370
|
+
msg=sanitize_message(line),
|
|
371
|
+
raw=line,
|
|
372
|
+
source_hint="unknown",
|
|
373
|
+
)
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
__all__ = [
|
|
377
|
+
"ParsedEvent",
|
|
378
|
+
"ParserError",
|
|
379
|
+
"autodetect_and_parse",
|
|
380
|
+
]
|