syslog2cef 0.1.5__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. {syslog2cef-0.1.5/syslog2cef.egg-info → syslog2cef-0.2.0}/PKG-INFO +24 -1
  2. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/README.md +23 -0
  3. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/pyproject.toml +2 -2
  4. {syslog2cef-0.1.5 → syslog2cef-0.2.0/syslog2cef.egg-info}/PKG-INFO +24 -1
  5. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslog2cef.egg-info/SOURCES.txt +6 -0
  6. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/cef.py +32 -5
  7. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/custom.py +6 -5
  8. syslog2cef-0.2.0/syslogcef/dictionary.json +943 -0
  9. syslog2cef-0.2.0/syslogcef/dictionary.py +61 -0
  10. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/normalizer.py +10 -4
  11. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/parsers.py +124 -12
  12. syslog2cef-0.2.0/syslogcef/validation.py +138 -0
  13. syslog2cef-0.2.0/tests/test_dictionary.py +142 -0
  14. syslog2cef-0.2.0/tests/test_fuzz_grammar.py +293 -0
  15. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/tests/test_fuzz_properties.py +30 -31
  16. syslog2cef-0.1.5/syslogcef/validation.py +0 -177
  17. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/LICENSE +0 -0
  18. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/setup.cfg +0 -0
  19. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslog2cef.egg-info/dependency_links.txt +0 -0
  20. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslog2cef.egg-info/entry_points.txt +0 -0
  21. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslog2cef.egg-info/requires.txt +0 -0
  22. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslog2cef.egg-info/top_level.txt +0 -0
  23. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/__init__.py +0 -0
  24. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/__main__.py +0 -0
  25. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/adaptive.py +0 -0
  26. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/api.py +0 -0
  27. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/cli.py +0 -0
  28. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/mappings/__init__.py +0 -0
  29. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/mappings/cisco_asa.json +0 -0
  30. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/mappings/cisco_ios.json +0 -0
  31. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/mappings/f5.json +0 -0
  32. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/mappings/fortinet.json +0 -0
  33. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/mappings/linux.json +0 -0
  34. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/mappings/sophos.json +0 -0
  35. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/mappings/vmware.json +0 -0
  36. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/net.py +0 -0
  37. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/syslogcef/utils.py +0 -0
  38. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/tests/test_adaptive.py +0 -0
  39. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/tests/test_cef.py +0 -0
  40. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/tests/test_cli.py +0 -0
  41. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/tests/test_custom_parsers.py +0 -0
  42. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/tests/test_net.py +0 -0
  43. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/tests/test_normalizer.py +0 -0
  44. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/tests/test_parsers.py +0 -0
  45. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/tests/test_parsers_extra.py +0 -0
  46. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/tests/test_pipeline.py +0 -0
  47. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/tests/test_real_world.py +0 -0
  48. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/tests/test_utils.py +0 -0
  49. {syslog2cef-0.1.5 → syslog2cef-0.2.0}/tests/test_validation.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: syslog2cef
3
- Version: 0.1.5
3
+ Version: 0.2.0
4
4
  Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
5
  Author-email: Tamir Suliman <allamiro@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/allamiro/syslogcef
@@ -201,6 +201,29 @@ Field templates that reference missing keys resolve to an empty string
201
201
  rather than failing the event. See [docs/cef_fields.md](docs/cef_fields.md)
202
202
  for the full CEF extension dictionary.
203
203
 
204
+ ## Field Dictionary
205
+
206
+ `syslogcef/dictionary.json` is the single source of truth for CEF field
207
+ knowledge, derived from the ArcSight Extension Dictionary (see
208
+ [docs/cef_fields.md](docs/cef_fields.md)):
209
+
210
+ - **Key metadata** — data type, maximum length, and producer/consumer
211
+ scope for every CEF key. `--validate`/`--strict` check types and
212
+ lengths from it, and warn (without failing) when producer output sets
213
+ a consumer-side key such as `rawEvent`.
214
+ - **Field aliases** — common source-log names mapped to canonical CEF
215
+ keys: `srcip`/`source_ip` → `src`, `dstport` → `dpt`, `user` →
216
+ `suser`, `rcvdbyte` → `in`, `action` → `act`, and ~50 more. Aliases
217
+ are applied during normalization for **every** event — including
218
+ key=value pairs extracted from adaptively-parsed unknown formats — so
219
+ mappings and validation always see canonical names. Original keys are
220
+ preserved and an explicit canonical key is never overwritten.
221
+
222
+ This means a Fortinet-style `srcip=10.1.1.1 dstport=443` and an unknown
223
+ device emitting the same pairs behind an unrecognized prefix both end up
224
+ with `src` and `dpt` available to mapping templates, with no per-device
225
+ configuration.
226
+
204
227
  ## Custom Parsers
205
228
 
206
229
  When a device emits a format no built-in parser handles, add your own
@@ -178,6 +178,29 @@ Field templates that reference missing keys resolve to an empty string
178
178
  rather than failing the event. See [docs/cef_fields.md](docs/cef_fields.md)
179
179
  for the full CEF extension dictionary.
180
180
 
181
+ ## Field Dictionary
182
+
183
+ `syslogcef/dictionary.json` is the single source of truth for CEF field
184
+ knowledge, derived from the ArcSight Extension Dictionary (see
185
+ [docs/cef_fields.md](docs/cef_fields.md)):
186
+
187
+ - **Key metadata** — data type, maximum length, and producer/consumer
188
+ scope for every CEF key. `--validate`/`--strict` check types and
189
+ lengths from it, and warn (without failing) when producer output sets
190
+ a consumer-side key such as `rawEvent`.
191
+ - **Field aliases** — common source-log names mapped to canonical CEF
192
+ keys: `srcip`/`source_ip` → `src`, `dstport` → `dpt`, `user` →
193
+ `suser`, `rcvdbyte` → `in`, `action` → `act`, and ~50 more. Aliases
194
+ are applied during normalization for **every** event — including
195
+ key=value pairs extracted from adaptively-parsed unknown formats — so
196
+ mappings and validation always see canonical names. Original keys are
197
+ preserved and an explicit canonical key is never overwritten.
198
+
199
+ This means a Fortinet-style `srcip=10.1.1.1 dstport=443` and an unknown
200
+ device emitting the same pairs behind an unrecognized prefix both end up
201
+ with `src` and `dpt` available to mapping templates, with no per-device
202
+ configuration.
203
+
181
204
  ## Custom Parsers
182
205
 
183
206
  When a device emits a format no built-in parser handles, add your own
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "syslog2cef"
7
- version = "0.1.5"
7
+ version = "0.2.0"
8
8
  description = "Convert RFC3164/RFC5424 syslog lines into ArcSight CEF"
9
9
  authors = [{name = "Tamir Suliman", email = "allamiro@gmail.com"}]
10
10
  readme = "README.md"
@@ -41,7 +41,7 @@ package-dir = {"" = "."}
41
41
  packages = ["syslogcef", "syslogcef.mappings"]
42
42
 
43
43
  [tool.setuptools.package-data]
44
- syslogcef = ["mappings/*.json"]
44
+ syslogcef = ["mappings/*.json", "dictionary.json"]
45
45
 
46
46
  [tool.pytest.ini_options]
47
47
  addopts = "-ra"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: syslog2cef
3
- Version: 0.1.5
3
+ Version: 0.2.0
4
4
  Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
5
  Author-email: Tamir Suliman <allamiro@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/allamiro/syslogcef
@@ -201,6 +201,29 @@ Field templates that reference missing keys resolve to an empty string
201
201
  rather than failing the event. See [docs/cef_fields.md](docs/cef_fields.md)
202
202
  for the full CEF extension dictionary.
203
203
 
204
+ ## Field Dictionary
205
+
206
+ `syslogcef/dictionary.json` is the single source of truth for CEF field
207
+ knowledge, derived from the ArcSight Extension Dictionary (see
208
+ [docs/cef_fields.md](docs/cef_fields.md)):
209
+
210
+ - **Key metadata** — data type, maximum length, and producer/consumer
211
+ scope for every CEF key. `--validate`/`--strict` check types and
212
+ lengths from it, and warn (without failing) when producer output sets
213
+ a consumer-side key such as `rawEvent`.
214
+ - **Field aliases** — common source-log names mapped to canonical CEF
215
+ keys: `srcip`/`source_ip` → `src`, `dstport` → `dpt`, `user` →
216
+ `suser`, `rcvdbyte` → `in`, `action` → `act`, and ~50 more. Aliases
217
+ are applied during normalization for **every** event — including
218
+ key=value pairs extracted from adaptively-parsed unknown formats — so
219
+ mappings and validation always see canonical names. Original keys are
220
+ preserved and an explicit canonical key is never overwritten.
221
+
222
+ This means a Fortinet-style `srcip=10.1.1.1 dstport=443` and an unknown
223
+ device emitting the same pairs behind an unrecognized prefix both end up
224
+ with `src` and `dpt` available to mapping templates, with no per-device
225
+ configuration.
226
+
204
227
  ## Custom Parsers
205
228
 
206
229
  When a device emits a format no built-in parser handles, add your own
@@ -8,6 +8,8 @@ pyproject.toml
8
8
  ./syslogcef/cef.py
9
9
  ./syslogcef/cli.py
10
10
  ./syslogcef/custom.py
11
+ ./syslogcef/dictionary.json
12
+ ./syslogcef/dictionary.py
11
13
  ./syslogcef/net.py
12
14
  ./syslogcef/normalizer.py
13
15
  ./syslogcef/parsers.py
@@ -34,6 +36,8 @@ syslogcef/api.py
34
36
  syslogcef/cef.py
35
37
  syslogcef/cli.py
36
38
  syslogcef/custom.py
39
+ syslogcef/dictionary.json
40
+ syslogcef/dictionary.py
37
41
  syslogcef/net.py
38
42
  syslogcef/normalizer.py
39
43
  syslogcef/parsers.py
@@ -51,6 +55,8 @@ tests/test_adaptive.py
51
55
  tests/test_cef.py
52
56
  tests/test_cli.py
53
57
  tests/test_custom_parsers.py
58
+ tests/test_dictionary.py
59
+ tests/test_fuzz_grammar.py
54
60
  tests/test_fuzz_properties.py
55
61
  tests/test_net.py
56
62
  tests/test_normalizer.py
@@ -28,7 +28,11 @@ DEFAULT_MAPPING = {
28
28
  },
29
29
  "extensions": {
30
30
  "msg": "%(msg)s",
31
- "rawEvent": "%(raw)s",
31
+ # The raw line travels in a labeled custom string: rawEvent is a
32
+ # consumer-side key producers must not set (see docs/cef_fields.md),
33
+ # and every bundled mapping already uses this shape.
34
+ "cs1Label": "rawEvent",
35
+ "cs1": "%(raw)s",
32
36
  },
33
37
  }
34
38
 
@@ -69,16 +73,39 @@ class MappingResolver:
69
73
  severity_map = {**DEFAULT_MAPPING["severity_map"], **self.mapping.get("severity_map", {})}
70
74
  value = fields.get("severity")
71
75
  if value is None and "pri" in fields:
72
- _, severity_value = divmod(int(fields.get("pri", 0)), 8)
73
- value = severity_value
76
+ # "pri" may come from an untrusted kv pair (pri=zzz), not just
77
+ # a parsed <PRI> header; garbage must not abort the pipeline.
78
+ try:
79
+ _, value = divmod(int(fields.get("pri", 0)), 8)
80
+ except (TypeError, ValueError):
81
+ value = None
74
82
  if value is None:
75
83
  return "3"
76
- return severity_map.get(str(value), str(value))
84
+ mapped = str(severity_map.get(str(value), value))
85
+ # The CEF:0 header severity slot must be numeric 0-10; anything
86
+ # else (non-numeric kv severity, bad severity_map value) falls
87
+ # back to the default rather than corrupting the header. ASCII
88
+ # check required: str.isdigit() accepts Unicode digits like "²"
89
+ # that int() rejects.
90
+ # Length guard first: int() on a multi-thousand-digit string hits
91
+ # Python 3.11+'s integer-conversion limit and raises. Valid CEF
92
+ # severities are at most two characters ("10").
93
+ if not (mapped.isascii() and mapped.isdigit() and len(mapped) <= 2) or int(mapped) > 10:
94
+ return "3"
95
+ return mapped
77
96
 
78
97
  def resolve_raw_extensions(self, fields: Mapping[str, Any]) -> Dict[str, str]:
79
98
  raw = {}
80
99
  merged = DEFAULT_MAPPING["extensions"].copy()
81
- merged.update(self.mapping.get("extensions", {}))
100
+ user_extensions = self.mapping.get("extensions", {})
101
+ # The default cs1/cs1Label pair is atomic: if a custom mapping
102
+ # touches either member, drop both defaults so a custom cs1 is
103
+ # not silently labeled "rawEvent" (nor a custom label attached
104
+ # to the default raw line).
105
+ if "cs1" in user_extensions or "cs1Label" in user_extensions:
106
+ merged.pop("cs1", None)
107
+ merged.pop("cs1Label", None)
108
+ merged.update(user_extensions)
82
109
  for key, template in merged.items():
83
110
  value = self._format(template, fields)
84
111
  if value:
@@ -140,11 +140,12 @@ def _year_with_rollover(parsed: datetime, ref: datetime) -> datetime:
140
140
 
141
141
  candidate = with_year(ref.year)
142
142
  if candidate is None:
143
- for offset in (1, -1, 2, -2, 3, -3, 4, -4):
144
- candidate = with_year(ref.year + offset)
145
- if candidate is not None:
146
- return candidate
147
- return parsed
143
+ # Feb 29: pick the occurrence closest in time, not the first
144
+ # leap year found in either direction.
145
+ candidates = [c for off in range(-4, 5) if off and (c := with_year(ref.year + off))]
146
+ if not candidates:
147
+ return parsed
148
+ return min(candidates, key=lambda c: abs(c - ref))
148
149
  if candidate - ref > timedelta(days=180):
149
150
  return with_year(ref.year - 1) or candidate
150
151
  if ref - candidate > timedelta(days=180):