syslog2cef 0.1.5__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {syslog2cef-0.1.5/syslog2cef.egg-info → syslog2cef-0.2.0}/PKG-INFO +24 -1
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/README.md +23 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/pyproject.toml +2 -2
- {syslog2cef-0.1.5 → syslog2cef-0.2.0/syslog2cef.egg-info}/PKG-INFO +24 -1
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslog2cef.egg-info/SOURCES.txt +6 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/cef.py +32 -5
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/custom.py +6 -5
- syslog2cef-0.2.0/syslogcef/dictionary.json +943 -0
- syslog2cef-0.2.0/syslogcef/dictionary.py +61 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/normalizer.py +10 -4
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/parsers.py +124 -12
- syslog2cef-0.2.0/syslogcef/validation.py +138 -0
- syslog2cef-0.2.0/tests/test_dictionary.py +142 -0
- syslog2cef-0.2.0/tests/test_fuzz_grammar.py +293 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/tests/test_fuzz_properties.py +30 -31
- syslog2cef-0.1.5/syslogcef/validation.py +0 -177
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/LICENSE +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/setup.cfg +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslog2cef.egg-info/dependency_links.txt +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslog2cef.egg-info/entry_points.txt +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslog2cef.egg-info/requires.txt +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslog2cef.egg-info/top_level.txt +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/__init__.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/__main__.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/adaptive.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/api.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/cli.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/mappings/__init__.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/mappings/cisco_asa.json +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/mappings/cisco_ios.json +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/mappings/f5.json +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/mappings/fortinet.json +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/mappings/linux.json +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/mappings/sophos.json +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/mappings/vmware.json +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/net.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/utils.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/tests/test_adaptive.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/tests/test_cef.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/tests/test_cli.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/tests/test_custom_parsers.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/tests/test_net.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/tests/test_normalizer.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/tests/test_parsers.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/tests/test_parsers_extra.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/tests/test_pipeline.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/tests/test_real_world.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/tests/test_utils.py +0 -0
- {syslog2cef-0.1.5 → syslog2cef-0.2.0}/tests/test_validation.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: syslog2cef
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
|
|
5
5
|
Author-email: Tamir Suliman <allamiro@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/allamiro/syslogcef
|
|
@@ -201,6 +201,29 @@ Field templates that reference missing keys resolve to an empty string
|
|
|
201
201
|
rather than failing the event. See [docs/cef_fields.md](docs/cef_fields.md)
|
|
202
202
|
for the full CEF extension dictionary.
|
|
203
203
|
|
|
204
|
+
## Field Dictionary
|
|
205
|
+
|
|
206
|
+
`syslogcef/dictionary.json` is the single source of truth for CEF field
|
|
207
|
+
knowledge, derived from the ArcSight Extension Dictionary (see
|
|
208
|
+
[docs/cef_fields.md](docs/cef_fields.md)):
|
|
209
|
+
|
|
210
|
+
- **Key metadata** — data type, maximum length, and producer/consumer
|
|
211
|
+
scope for every CEF key. `--validate`/`--strict` check types and
|
|
212
|
+
lengths from it, and warn (without failing) when producer output sets
|
|
213
|
+
a consumer-side key such as `rawEvent`.
|
|
214
|
+
- **Field aliases** — common source-log names mapped to canonical CEF
|
|
215
|
+
keys: `srcip`/`source_ip` → `src`, `dstport` → `dpt`, `user` →
|
|
216
|
+
`suser`, `rcvdbyte` → `in`, `action` → `act`, and ~50 more. Aliases
|
|
217
|
+
are applied during normalization for **every** event — including
|
|
218
|
+
key=value pairs extracted from adaptively-parsed unknown formats — so
|
|
219
|
+
mappings and validation always see canonical names. Original keys are
|
|
220
|
+
preserved and an explicit canonical key is never overwritten.
|
|
221
|
+
|
|
222
|
+
This means a Fortinet-style `srcip=10.1.1.1 dstport=443` and an unknown
|
|
223
|
+
device emitting the same pairs behind an unrecognized prefix both end up
|
|
224
|
+
with `src` and `dpt` available to mapping templates, with no per-device
|
|
225
|
+
configuration.
|
|
226
|
+
|
|
204
227
|
## Custom Parsers
|
|
205
228
|
|
|
206
229
|
When a device emits a format no built-in parser handles, add your own
|
|
@@ -178,6 +178,29 @@ Field templates that reference missing keys resolve to an empty string
|
|
|
178
178
|
rather than failing the event. See [docs/cef_fields.md](docs/cef_fields.md)
|
|
179
179
|
for the full CEF extension dictionary.
|
|
180
180
|
|
|
181
|
+
## Field Dictionary
|
|
182
|
+
|
|
183
|
+
`syslogcef/dictionary.json` is the single source of truth for CEF field
|
|
184
|
+
knowledge, derived from the ArcSight Extension Dictionary (see
|
|
185
|
+
[docs/cef_fields.md](docs/cef_fields.md)):
|
|
186
|
+
|
|
187
|
+
- **Key metadata** — data type, maximum length, and producer/consumer
|
|
188
|
+
scope for every CEF key. `--validate`/`--strict` check types and
|
|
189
|
+
lengths from it, and warn (without failing) when producer output sets
|
|
190
|
+
a consumer-side key such as `rawEvent`.
|
|
191
|
+
- **Field aliases** — common source-log names mapped to canonical CEF
|
|
192
|
+
keys: `srcip`/`source_ip` → `src`, `dstport` → `dpt`, `user` →
|
|
193
|
+
`suser`, `rcvdbyte` → `in`, `action` → `act`, and ~50 more. Aliases
|
|
194
|
+
are applied during normalization for **every** event — including
|
|
195
|
+
key=value pairs extracted from adaptively-parsed unknown formats — so
|
|
196
|
+
mappings and validation always see canonical names. Original keys are
|
|
197
|
+
preserved and an explicit canonical key is never overwritten.
|
|
198
|
+
|
|
199
|
+
This means a Fortinet-style `srcip=10.1.1.1 dstport=443` and an unknown
|
|
200
|
+
device emitting the same pairs behind an unrecognized prefix both end up
|
|
201
|
+
with `src` and `dpt` available to mapping templates, with no per-device
|
|
202
|
+
configuration.
|
|
203
|
+
|
|
181
204
|
## Custom Parsers
|
|
182
205
|
|
|
183
206
|
When a device emits a format no built-in parser handles, add your own
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "syslog2cef"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.2.0"
|
|
8
8
|
description = "Convert RFC3164/RFC5424 syslog lines into ArcSight CEF"
|
|
9
9
|
authors = [{name = "Tamir Suliman", email = "allamiro@gmail.com"}]
|
|
10
10
|
readme = "README.md"
|
|
@@ -41,7 +41,7 @@ package-dir = {"" = "."}
|
|
|
41
41
|
packages = ["syslogcef", "syslogcef.mappings"]
|
|
42
42
|
|
|
43
43
|
[tool.setuptools.package-data]
|
|
44
|
-
syslogcef = ["mappings/*.json"]
|
|
44
|
+
syslogcef = ["mappings/*.json", "dictionary.json"]
|
|
45
45
|
|
|
46
46
|
[tool.pytest.ini_options]
|
|
47
47
|
addopts = "-ra"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: syslog2cef
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
|
|
5
5
|
Author-email: Tamir Suliman <allamiro@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/allamiro/syslogcef
|
|
@@ -201,6 +201,29 @@ Field templates that reference missing keys resolve to an empty string
|
|
|
201
201
|
rather than failing the event. See [docs/cef_fields.md](docs/cef_fields.md)
|
|
202
202
|
for the full CEF extension dictionary.
|
|
203
203
|
|
|
204
|
+
## Field Dictionary
|
|
205
|
+
|
|
206
|
+
`syslogcef/dictionary.json` is the single source of truth for CEF field
|
|
207
|
+
knowledge, derived from the ArcSight Extension Dictionary (see
|
|
208
|
+
[docs/cef_fields.md](docs/cef_fields.md)):
|
|
209
|
+
|
|
210
|
+
- **Key metadata** — data type, maximum length, and producer/consumer
|
|
211
|
+
scope for every CEF key. `--validate`/`--strict` check types and
|
|
212
|
+
lengths from it, and warn (without failing) when producer output sets
|
|
213
|
+
a consumer-side key such as `rawEvent`.
|
|
214
|
+
- **Field aliases** — common source-log names mapped to canonical CEF
|
|
215
|
+
keys: `srcip`/`source_ip` → `src`, `dstport` → `dpt`, `user` →
|
|
216
|
+
`suser`, `rcvdbyte` → `in`, `action` → `act`, and ~50 more. Aliases
|
|
217
|
+
are applied during normalization for **every** event — including
|
|
218
|
+
key=value pairs extracted from adaptively-parsed unknown formats — so
|
|
219
|
+
mappings and validation always see canonical names. Original keys are
|
|
220
|
+
preserved and an explicit canonical key is never overwritten.
|
|
221
|
+
|
|
222
|
+
This means a Fortinet-style `srcip=10.1.1.1 dstport=443` and an unknown
|
|
223
|
+
device emitting the same pairs behind an unrecognized prefix both end up
|
|
224
|
+
with `src` and `dpt` available to mapping templates, with no per-device
|
|
225
|
+
configuration.
|
|
226
|
+
|
|
204
227
|
## Custom Parsers
|
|
205
228
|
|
|
206
229
|
When a device emits a format no built-in parser handles, add your own
|
|
@@ -8,6 +8,8 @@ pyproject.toml
|
|
|
8
8
|
./syslogcef/cef.py
|
|
9
9
|
./syslogcef/cli.py
|
|
10
10
|
./syslogcef/custom.py
|
|
11
|
+
./syslogcef/dictionary.json
|
|
12
|
+
./syslogcef/dictionary.py
|
|
11
13
|
./syslogcef/net.py
|
|
12
14
|
./syslogcef/normalizer.py
|
|
13
15
|
./syslogcef/parsers.py
|
|
@@ -34,6 +36,8 @@ syslogcef/api.py
|
|
|
34
36
|
syslogcef/cef.py
|
|
35
37
|
syslogcef/cli.py
|
|
36
38
|
syslogcef/custom.py
|
|
39
|
+
syslogcef/dictionary.json
|
|
40
|
+
syslogcef/dictionary.py
|
|
37
41
|
syslogcef/net.py
|
|
38
42
|
syslogcef/normalizer.py
|
|
39
43
|
syslogcef/parsers.py
|
|
@@ -51,6 +55,8 @@ tests/test_adaptive.py
|
|
|
51
55
|
tests/test_cef.py
|
|
52
56
|
tests/test_cli.py
|
|
53
57
|
tests/test_custom_parsers.py
|
|
58
|
+
tests/test_dictionary.py
|
|
59
|
+
tests/test_fuzz_grammar.py
|
|
54
60
|
tests/test_fuzz_properties.py
|
|
55
61
|
tests/test_net.py
|
|
56
62
|
tests/test_normalizer.py
|
|
@@ -28,7 +28,11 @@ DEFAULT_MAPPING = {
|
|
|
28
28
|
},
|
|
29
29
|
"extensions": {
|
|
30
30
|
"msg": "%(msg)s",
|
|
31
|
-
|
|
31
|
+
# The raw line travels in a labeled custom string: rawEvent is a
|
|
32
|
+
# consumer-side key producers must not set (see docs/cef_fields.md),
|
|
33
|
+
# and every bundled mapping already uses this shape.
|
|
34
|
+
"cs1Label": "rawEvent",
|
|
35
|
+
"cs1": "%(raw)s",
|
|
32
36
|
},
|
|
33
37
|
}
|
|
34
38
|
|
|
@@ -69,16 +73,39 @@ class MappingResolver:
|
|
|
69
73
|
severity_map = {**DEFAULT_MAPPING["severity_map"], **self.mapping.get("severity_map", {})}
|
|
70
74
|
value = fields.get("severity")
|
|
71
75
|
if value is None and "pri" in fields:
|
|
72
|
-
|
|
73
|
-
|
|
76
|
+
# "pri" may come from an untrusted kv pair (pri=zzz), not just
|
|
77
|
+
# a parsed <PRI> header; garbage must not abort the pipeline.
|
|
78
|
+
try:
|
|
79
|
+
_, value = divmod(int(fields.get("pri", 0)), 8)
|
|
80
|
+
except (TypeError, ValueError):
|
|
81
|
+
value = None
|
|
74
82
|
if value is None:
|
|
75
83
|
return "3"
|
|
76
|
-
|
|
84
|
+
mapped = str(severity_map.get(str(value), value))
|
|
85
|
+
# The CEF:0 header severity slot must be numeric 0-10; anything
|
|
86
|
+
# else (non-numeric kv severity, bad severity_map value) falls
|
|
87
|
+
# back to the default rather than corrupting the header. ASCII
|
|
88
|
+
# check required: str.isdigit() accepts Unicode digits like "²"
|
|
89
|
+
# that int() rejects.
|
|
90
|
+
# Length guard first: int() on a multi-thousand-digit string hits
|
|
91
|
+
# Python 3.11+'s integer-conversion limit and raises. Valid CEF
|
|
92
|
+
# severities are at most two characters ("10").
|
|
93
|
+
if not (mapped.isascii() and mapped.isdigit() and len(mapped) <= 2) or int(mapped) > 10:
|
|
94
|
+
return "3"
|
|
95
|
+
return mapped
|
|
77
96
|
|
|
78
97
|
def resolve_raw_extensions(self, fields: Mapping[str, Any]) -> Dict[str, str]:
|
|
79
98
|
raw = {}
|
|
80
99
|
merged = DEFAULT_MAPPING["extensions"].copy()
|
|
81
|
-
|
|
100
|
+
user_extensions = self.mapping.get("extensions", {})
|
|
101
|
+
# The default cs1/cs1Label pair is atomic: if a custom mapping
|
|
102
|
+
# touches either member, drop both defaults so a custom cs1 is
|
|
103
|
+
# not silently labeled "rawEvent" (nor a custom label attached
|
|
104
|
+
# to the default raw line).
|
|
105
|
+
if "cs1" in user_extensions or "cs1Label" in user_extensions:
|
|
106
|
+
merged.pop("cs1", None)
|
|
107
|
+
merged.pop("cs1Label", None)
|
|
108
|
+
merged.update(user_extensions)
|
|
82
109
|
for key, template in merged.items():
|
|
83
110
|
value = self._format(template, fields)
|
|
84
111
|
if value:
|
|
@@ -140,11 +140,12 @@ def _year_with_rollover(parsed: datetime, ref: datetime) -> datetime:
|
|
|
140
140
|
|
|
141
141
|
candidate = with_year(ref.year)
|
|
142
142
|
if candidate is None:
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
143
|
+
# Feb 29: pick the occurrence closest in time, not the first
|
|
144
|
+
# leap year found in either direction.
|
|
145
|
+
candidates = [c for off in range(-4, 5) if off and (c := with_year(ref.year + off))]
|
|
146
|
+
if not candidates:
|
|
147
|
+
return parsed
|
|
148
|
+
return min(candidates, key=lambda c: abs(c - ref))
|
|
148
149
|
if candidate - ref > timedelta(days=180):
|
|
149
150
|
return with_year(ref.year - 1) or candidate
|
|
150
151
|
if ref - candidate > timedelta(days=180):
|