syslog2cef 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,202 @@
1
+ Metadata-Version: 2.4
2
+ Name: syslog2cef
3
+ Version: 0.1.0
4
+ Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
+ Author-email: Tamir Suliman <allamiro@gmail.com>
6
+ Project-URL: Homepage, https://github.com/allamiro/syslogcef
7
+ Project-URL: Issues, https://github.com/allamiro/syslogcef/issues
8
+ Keywords: syslog,cef,logging,security
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: Programming Language :: Python :: 3 :: Only
11
+ Classifier: License :: OSI Approved :: MIT License
12
+ Classifier: Topic :: System :: Logging
13
+ Requires-Python: >=3.9
14
+ Description-Content-Type: text/markdown
15
+ License-File: LICENSE
16
+ Provides-Extra: test
17
+ Requires-Dist: pytest>=7; extra == "test"
18
+ Dynamic: license-file
19
+
20
+ # syslogcef
21
+
22
+ syslogcef converts syslog events into ArcSight Common Event Format (CEF). It
23
+ is a small, dependency-free Python package with a command line interface,
24
+ built for feeding syslog data from network devices and Linux hosts into SIEM
25
+ platforms that consume CEF.
26
+
27
+ ## Features
28
+
29
+ - Automatic detection and parsing of multiple syslog dialects:
30
+ RFC3164 (BSD), RFC5424 (including structured data), rsyslog JSON and
31
+ file formats, and systemd journal exports (JSON, short, and ISO formats).
32
+ - Deterministic vendor/product mappings bundled for Cisco ASA, Cisco IOS,
33
+ F5 BIG-IP, generic Linux, and VMware ESXi, with automatic mapping
34
+ selection based on message content.
35
+ - Custom mappings supplied as JSON files or Python dictionaries.
36
+ - Escaping of untrusted log content in both CEF header and extension
37
+ fields, so crafted messages cannot forge header fields or split records.
38
+ - Streaming CLI with stdin/file input, tail (follow) mode across multiple
39
+ files, optional multiprocessing, and no runtime dependencies.
40
+ - Malformed lines never abort a stream: unparseable input falls back to a
41
+ raw-message event.
42
+
43
+ ## Installation
44
+
45
+ From PyPI (the distribution is named `syslog2cef` because the `syslogcef`
46
+ name on PyPI belongs to an unrelated project; the import package and CLI
47
+ are still `syslogcef`):
48
+
49
+ ```bash
50
+ pip install syslog2cef
51
+ ```
52
+
53
+ From source:
54
+
55
+ ```bash
56
+ git clone https://github.com/allamiro/syslogcef.git
57
+ cd syslogcef
58
+ pip install .
59
+ ```
60
+
61
+ An RPM package with a systemd service is also available from the GitHub
62
+ releases page; see "Running as a service" below.
63
+
64
+ ## Command Line Usage
65
+
66
+ ```bash
67
+ # Read from stdin, write CEF to stdout
68
+ syslogcef < /var/log/syslog
69
+
70
+ # Convert one or more files and write to an output file
71
+ syslogcef /var/log/messages /var/log/secure --output events.cef
72
+
73
+ # Follow files in real time (all files are tailed concurrently)
74
+ syslogcef /var/log/asa.log /var/log/messages --tail
75
+
76
+ # Force a parser and mapping instead of auto-detection
77
+ syslogcef asa.log --mode rfc3164 --mapping syslogcef/mappings/cisco_asa.json
78
+
79
+ # Use multiprocessing for high-volume batch conversion
80
+ syslogcef big.log --multiprocess --pool-size 4
81
+ ```
82
+
83
+ Options:
84
+
85
+ | Option | Description |
86
+ | ------ | ----------- |
87
+ | `paths` | Input files; stdin is used when omitted. |
88
+ | `-o, --output FILE` | Write CEF lines to a file instead of stdout. |
89
+ | `--mode MODE` | Parser override: `rfc3164`, `rfc5424`, `rsyslog_json`, `rsyslog_file`, `journald_json`, `journald_short`, `journald_iso`. Auto-detected when omitted. |
90
+ | `--mapping FILE` | Mapping JSON file. Auto-selected from message content when omitted. |
91
+ | `--tail` | Follow input files like `tail -f`. |
92
+ | `--multiprocess` | Convert lines using a process pool. |
93
+ | `--pool-size N` | Worker count for `--multiprocess` (default: CPU count minus one). |
94
+ | `--log-level LEVEL` | Python logging level (default `WARNING`). |
95
+
96
+ `python -m syslogcef` is equivalent to the `syslogcef` entry point.
97
+
98
+ ## Python API
99
+
100
+ High-level, one call per line:
101
+
102
+ ```python
103
+ from syslogcef import convert_line
104
+
105
+ line = "<166>Jan 1 12:34:56 fw01 %ASA-6-302013: Built inbound TCP connection src=10.0.0.1 dst=10.0.0.2"
106
+ print(convert_line(line))
107
+ ```
108
+
109
+ Lower-level pipeline when granular control is required:
110
+
111
+ ```python
112
+ from syslogcef import parse_syslog, normalize_event, to_cef
113
+
114
+ parsed = parse_syslog(line) # ParsedEvent: pri, timestamp, host, app, msg, ...
115
+ normalized = normalize_event(parsed) # adds key/value pairs, event codes, derived fields
116
+ cef = to_cef(normalized, mapping="my_mapping.json")
117
+ ```
118
+
119
+ Bundled mappings are importable from `syslogcef.mappings` (`CISCO_ASA`,
120
+ `CISCO_IOS`, `F5`, `LINUX`, `VMWARE`, or `load_mapping(name)`).
121
+
122
+ ## Mapping Files
123
+
124
+ A mapping is a JSON object that controls the CEF header and extension
125
+ fields. Values are Python %-format templates resolved against the
126
+ normalized event's fields:
127
+
128
+ ```json
129
+ {
130
+ "deviceVendor": "Cisco",
131
+ "deviceProduct": "ASA",
132
+ "deviceVersion": "auto",
133
+ "eventClassId": "asa.%(event_code)s",
134
+ "name": "%(message_short)s",
135
+ "severity_map": { "6": "2", "3": "6" },
136
+ "extensions": {
137
+ "src": "%(src)s",
138
+ "dst": "%(dst)s",
139
+ "cs1Label": "rawEvent",
140
+ "cs1": "%(raw_kv)s"
141
+ }
142
+ }
143
+ ```
144
+
145
+ - Header keys: `deviceVendor`, `deviceProduct`, `deviceVersion`,
146
+ `eventClassId`, `name`.
147
+ - `severity_map` translates syslog severity (0-7) to CEF severity (0-10);
148
+ unmapped values pass through.
149
+ - `extensions` maps CEF extension keys to templates. Extensions that
150
+ resolve to an empty value are omitted.
151
+ - Available template fields include `host`, `app`, `pid`, `msgid`, `msg`,
152
+ `message_short` (first 120 characters), `raw`, `raw_kv`, `event_code`,
153
+ `facility`, `severity`, `ts`, plus every key=value pair extracted from
154
+ the message and any RFC5424 structured-data or journald fields.
155
+
156
+ Field templates that reference missing keys resolve to an empty string
157
+ rather than failing the event. See [docs/cef_fields.md](docs/cef_fields.md)
158
+ for the full CEF extension dictionary.
159
+
160
+ ## Running as a Service
161
+
162
+ The RPM package installs a systemd unit and an environment file:
163
+
164
+ - `/etc/syslogcef/syslogcef.conf` — input file, output file, and extra
165
+ arguments for the converter.
166
+ - `syslogcef.service` — runs `syslogcef --tail` against the configured
167
+ input and appends CEF to the configured output.
168
+
169
+ ```bash
170
+ sudo dnf install syslogcef-*.rpm
171
+ sudo vi /etc/syslogcef/syslogcef.conf
172
+ sudo systemctl enable --now syslogcef
173
+ ```
174
+
175
+ See [packaging/rpm/](packaging/rpm/) for the spec file and build
176
+ instructions, including GPG signing of the RPM.
177
+
178
+ ## Security
179
+
180
+ Log content is treated as untrusted input. Header and extension values are
181
+ escaped per the CEF specification before rendering, and CR/LF are removed
182
+ from header fields so records cannot be split or spoofed. To report a
183
+ vulnerability, see [SECURITY.md](SECURITY.md) — please do not open public
184
+ issues for security reports.
185
+
186
+ ## Development
187
+
188
+ ```bash
189
+ git clone https://github.com/allamiro/syslogcef.git
190
+ cd syslogcef
191
+ python -m venv .venv
192
+ source .venv/bin/activate
193
+ pip install -e .[test]
194
+ pytest
195
+ ```
196
+
197
+ Contributions are welcome — see [CONTRIBUTING.md](CONTRIBUTING.md). Notable
198
+ changes are tracked in [CHANGELOG.md](CHANGELOG.md).
199
+
200
+ ## License
201
+
202
+ MIT — see [LICENSE](LICENSE). Copyright (c) Tamir Suliman.
@@ -0,0 +1,20 @@
1
+ syslog2cef-0.1.0.dist-info/licenses/LICENSE,sha256=l2nGswRGvDbaAf9OMG61hhS9jYY8GDgRtOYICQtvaE0,1070
2
+ syslogcef/__init__.py,sha256=Q_SFz4laNK4ipLeAS-PxHPgOGLWbSI2aeWuX0tPmC2E,496
3
+ syslogcef/__main__.py,sha256=nbdP6zFlOqcCirgxoIQ6HiWVhdh7URm5GOWr5_EwmBk,116
4
+ syslogcef/api.py,sha256=RdVZSo1J39agJoCzA7qANJENq9a7SwN8G-eBQU_Z34I,3304
5
+ syslogcef/cef.py,sha256=mThPrk6Fzcv2rBG96WQEhv_jnLUZr4_GUosvcs0ae9k,3352
6
+ syslogcef/cli.py,sha256=rFKkx9ffq-QANlDPfWbzuH1GWu2vMRkXTCACrfPCbUg,4794
7
+ syslogcef/normalizer.py,sha256=qmqMufqyldce5kJd0Dg-iCaR-mURx6YhHpRfp0FcsHU,2990
8
+ syslogcef/parsers.py,sha256=gK2zN66WUG5FJ6UZThb4I2fOMlRf5duFWjFlgGE8C8s,11088
9
+ syslogcef/utils.py,sha256=8UIif4b9Iloar7rTlebELXC80Wc1PihFufpfp9Foj9M,3699
10
+ syslogcef/mappings/__init__.py,sha256=X9Y7Suon2RXwMRdtK8Fpsm2ueBy-wDsbY67QG89H61U,852
11
+ syslogcef/mappings/cisco_asa.json,sha256=QbCeQWD999C08DjpMiPitB89Z_wLtNof-pMe8kVmVOU,483
12
+ syslogcef/mappings/cisco_ios.json,sha256=uHz5Y1bzfA1zoyByJnm6yVqEwbdmIUNrqU2kPRw6JgI,270
13
+ syslogcef/mappings/f5.json,sha256=h-19rqQ8c703jJwo2RN5fXx8cFpmZUdGF2i1x-Ra-t4,278
14
+ syslogcef/mappings/linux.json,sha256=fA5ZuxJXVzt5vlQV8kF4p5TJfyrLvX11c_G7owLR7Sc,272
15
+ syslogcef/mappings/vmware.json,sha256=kruNvpSb0wglZalAXXHLQurmw9bFlxxgI_hGrcpSWiQ,254
16
+ syslog2cef-0.1.0.dist-info/METADATA,sha256=W_DXPiJ62VFf2D1uFPMF8jJxMbVGDkWFshKuDttbWUc,6913
17
+ syslog2cef-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
18
+ syslog2cef-0.1.0.dist-info/entry_points.txt,sha256=-_AClsJ3zP6yWMf1MiRZLtdEF2olVWQKJhI6nWL-vMA,49
19
+ syslog2cef-0.1.0.dist-info/top_level.txt,sha256=7Xt2pSdcLZ9y3ufEbHHz47Bi3rXBVwZZ6nDFUNpvPIM,10
20
+ syslog2cef-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.0.0)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ syslogcef = syslogcef.cli:main
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2023 Tamir Suliman
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1 @@
1
+ syslogcef
syslogcef/__init__.py ADDED
@@ -0,0 +1,19 @@
1
+ """Core API for converting syslog lines into ArcSight CEF.
2
+
3
+ This package exposes a high level pipeline composed of four functions:
4
+
5
+ ``parse_syslog`` -> ``normalize_event`` -> ``to_cef`` -> ``convert_line``
6
+
7
+ Each function can be used independently or as part of the complete
8
+ conversion workflow.
9
+ """
10
+ from __future__ import annotations
11
+
12
+ from .api import convert_line, normalize_event, parse_syslog, to_cef
13
+
14
+ __all__ = [
15
+ "convert_line",
16
+ "normalize_event",
17
+ "parse_syslog",
18
+ "to_cef",
19
+ ]
syslogcef/__main__.py ADDED
@@ -0,0 +1,7 @@
1
+ from __future__ import annotations
2
+
3
+ from .cli import main
4
+
5
+
6
+ if __name__ == "__main__":
7
+ raise SystemExit(main())
syslogcef/api.py ADDED
@@ -0,0 +1,127 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ import logging
5
+ from dataclasses import dataclass, field
6
+ from datetime import datetime, timezone
7
+ from pathlib import Path
8
+ from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional
9
+
10
+ from .parsers import ParsedEvent, autodetect_and_parse
11
+ from .normalizer import NormalizedEvent, normalize
12
+ from .mappings import CISCO_ASA, CISCO_IOS, F5, LINUX, VMWARE
13
+ from .cef import CEFEvent, build_cef
14
+
15
+ logger = logging.getLogger(__name__)
16
+
17
+
18
+ @dataclass(slots=True)
19
+ class ParseResult:
20
+ """Container returned by :func:`parse_syslog`.
21
+
22
+ The ``raw`` attribute always contains the original syslog line.
23
+ """
24
+
25
+ event: NormalizedEvent
26
+ raw: str
27
+
28
+
29
+ def parse_syslog(
30
+ line: str,
31
+ *,
32
+ mode: Optional[str] = None,
33
+ now: Optional[datetime] = None,
34
+ ) -> ParsedEvent:
35
+ """Parse a raw syslog line.
36
+
37
+ Parameters
38
+ ----------
39
+ line:
40
+ Raw syslog string.
41
+ mode:
42
+ Optional parser mode. When ``None`` the parser auto-detects RFC3164,
43
+ RFC5424, rsyslog formats and journalctl exports.
44
+ now:
45
+ Optional datetime used when inferring missing year information.
46
+ """
47
+
48
+ parsed = autodetect_and_parse(line, mode=mode, now=now)
49
+ return parsed
50
+
51
+
52
+ def normalize_event(event: ParsedEvent | NormalizedEvent) -> NormalizedEvent:
53
+ """Normalize parsed syslog data.
54
+
55
+ The normalizer enriches the event with structured fields, sanitised
56
+ messages and derived metadata that the CEF renderer relies on.
57
+ """
58
+
59
+ if isinstance(event, NormalizedEvent):
60
+ return event
61
+ return normalize(event)
62
+
63
+
64
+ def _load_mapping(mapping: Mapping[str, Any] | Path | str | None) -> Mapping[str, Any]:
65
+ if mapping is None:
66
+ return {}
67
+ if isinstance(mapping, Mapping):
68
+ return mapping
69
+ path = Path(mapping)
70
+ with path.open("r", encoding="utf-8") as fp:
71
+ return json.load(fp)
72
+
73
+
74
+ def to_cef(
75
+ event: NormalizedEvent,
76
+ mapping: Mapping[str, Any] | Path | str | None = None,
77
+ ) -> str:
78
+ """Convert a normalized event into a CEF string."""
79
+
80
+ if mapping is None:
81
+ mapping_data = _guess_mapping(event)
82
+ else:
83
+ mapping_data = _load_mapping(mapping)
84
+ cef_event = build_cef(event, mapping_data)
85
+ return cef_event.render()
86
+
87
+
88
+ def convert_line(
89
+ line: str,
90
+ *,
91
+ mode: Optional[str] = None,
92
+ mapping: Mapping[str, Any] | Path | str | None = None,
93
+ now: Optional[datetime] = None,
94
+ ) -> str:
95
+ """Full pipeline that parses, normalizes and converts a syslog line."""
96
+
97
+ parsed = parse_syslog(line, mode=mode, now=now)
98
+ normalized = normalize_event(parsed)
99
+
100
+ return to_cef(normalized, mapping)
101
+
102
+
103
+ def _guess_mapping(event: NormalizedEvent) -> Mapping[str, Any]:
104
+ msg_upper = event.msg.upper()
105
+ app_upper = (event.app or "").upper()
106
+
107
+ if "%ASA-" in msg_upper or "ASA" in app_upper:
108
+ return CISCO_ASA
109
+ if "IOS" in app_upper or "%IOS-" in msg_upper:
110
+ return CISCO_IOS
111
+ if "BIG-IP" in msg_upper or "F5" in app_upper:
112
+ return F5
113
+ if "VMWARE" in msg_upper or "ESXI" in msg_upper or "VMWARE" in app_upper:
114
+ return VMWARE
115
+ return LINUX
116
+
117
+
118
+ __all__ = [
119
+ "CEFEvent",
120
+ "ParsedEvent",
121
+ "NormalizedEvent",
122
+ "ParseResult",
123
+ "convert_line",
124
+ "normalize_event",
125
+ "parse_syslog",
126
+ "to_cef",
127
+ ]
syslogcef/cef.py ADDED
@@ -0,0 +1,103 @@
1
+ from __future__ import annotations
2
+
3
+ import logging
4
+ from dataclasses import dataclass
5
+ from datetime import datetime
6
+ from typing import Any, Dict, Mapping, Optional
7
+
8
+ from .normalizer import NormalizedEvent
9
+ from .utils import cef_escape, cef_escape_header
10
+
11
+ logger = logging.getLogger(__name__)
12
+
13
+
14
+ DEFAULT_MAPPING = {
15
+ "deviceVendor": "Generic",
16
+ "deviceProduct": "Syslog",
17
+ "deviceVersion": "1.0",
18
+ "eventClassId": "syslog",
19
+ "name": "%(message_short)s",
20
+ "severity_map": {
21
+ "0": "10",
22
+ "1": "8",
23
+ "2": "7",
24
+ "3": "6",
25
+ "4": "5",
26
+ "5": "3",
27
+ "6": "2",
28
+ "7": "1",
29
+ },
30
+ "extensions": {
31
+ "msg": "%(msg)s",
32
+ "rawEvent": "%(raw)s",
33
+ },
34
+ }
35
+
36
+
37
+ @dataclass(slots=True)
38
+ class CEFEvent:
39
+ header: Dict[str, str]
40
+ extensions: Dict[str, str]
41
+
42
+ def render(self) -> str:
43
+ extension_str = " ".join(f"{k}={v}" for k, v in self.extensions.items())
44
+ return "CEF:0|{deviceVendor}|{deviceProduct}|{deviceVersion}|{eventClassId}|{name}|{severity}|{extension}".format(
45
+ extension=extension_str,
46
+ **self.header,
47
+ )
48
+
49
+
50
+ class MappingResolver:
51
+ def __init__(self, mapping: Mapping[str, Any]) -> None:
52
+ merged = DEFAULT_MAPPING | mapping
53
+ self.mapping = merged
54
+
55
+ def resolve_header(self, fields: Mapping[str, Any]) -> Dict[str, str]:
56
+ header = {}
57
+ for key in ["deviceVendor", "deviceProduct", "deviceVersion", "eventClassId", "name"]:
58
+ template = self.mapping.get(key, DEFAULT_MAPPING[key])
59
+ header[key] = cef_escape_header(self._format(template, fields))
60
+ severity = self._resolve_severity(fields)
61
+ header["severity"] = severity
62
+ return header
63
+
64
+ def _resolve_severity(self, fields: Mapping[str, Any]) -> str:
65
+ severity_map = {**DEFAULT_MAPPING["severity_map"], **self.mapping.get("severity_map", {})}
66
+ value = fields.get("severity")
67
+ if value is None and "pri" in fields:
68
+ _, severity_value = divmod(int(fields.get("pri", 0)), 8)
69
+ value = severity_value
70
+ if value is None:
71
+ return "3"
72
+ return severity_map.get(str(value), str(value))
73
+
74
+ def resolve_extensions(self, fields: Mapping[str, Any]) -> Dict[str, str]:
75
+ extensions = {}
76
+ merged = DEFAULT_MAPPING["extensions"].copy()
77
+ merged.update(self.mapping.get("extensions", {}))
78
+ for key, template in merged.items():
79
+ value = self._format(template, fields)
80
+ if value:
81
+ extensions[key] = cef_escape(str(value))
82
+ return extensions
83
+
84
+ def _format(self, template: str, fields: Mapping[str, Any]) -> str:
85
+ try:
86
+ return template % fields
87
+ except KeyError as exc:
88
+ logger.debug("Missing key %s for template %s", exc, template)
89
+ return ""
90
+ except (ValueError, TypeError) as exc:
91
+ logger.warning("Invalid template %r: %s", template, exc)
92
+ return ""
93
+
94
+
95
+ def build_cef(event: NormalizedEvent, mapping: Mapping[str, Any]) -> CEFEvent:
96
+ resolver = MappingResolver(mapping)
97
+ fields = event.as_field_dict()
98
+ header = resolver.resolve_header(fields)
99
+ extensions = resolver.resolve_extensions(fields)
100
+ return CEFEvent(header=header, extensions=extensions)
101
+
102
+
103
+ __all__ = ["CEFEvent", "build_cef"]
syslogcef/cli.py ADDED
@@ -0,0 +1,136 @@
1
+ from __future__ import annotations
2
+
3
+ import argparse
4
+ import logging
5
+ import sys
6
+ import time
7
+ from functools import partial
8
+ from multiprocessing import Pool, cpu_count
9
+ from pathlib import Path
10
+ from typing import Iterable, Iterator, Optional
11
+
12
+ from .api import convert_line
13
+
14
+ logger = logging.getLogger(__name__)
15
+
16
+
17
+ def iter_lines_from_sources(paths: Iterable[Path] | None) -> Iterator[str]:
18
+ if not paths:
19
+ for line in sys.stdin:
20
+ yield line.rstrip("\n")
21
+ return
22
+
23
+ for path in paths:
24
+ with path.open("r", encoding="utf-8", errors="replace") as handle:
25
+ for line in handle:
26
+ yield line.rstrip("\n")
27
+
28
+
29
+ def tail_file(path: Path, *, follow: bool = False) -> Iterator[str]:
30
+ with path.open("r", encoding="utf-8", errors="replace") as handle:
31
+ while True:
32
+ line = handle.readline()
33
+ if not line:
34
+ if not follow:
35
+ break
36
+ time.sleep(0.5)
37
+ continue
38
+ yield line.rstrip("\n")
39
+
40
+
41
+ def follow_files(
42
+ paths: Iterable[Path],
43
+ *,
44
+ poll_interval: float = 0.5,
45
+ max_idle_polls: Optional[int] = None,
46
+ ) -> Iterator[str]:
47
+ """Tail several files at once, round-robin, yielding lines as they appear.
48
+
49
+ ``max_idle_polls`` bounds how many consecutive empty polls are allowed
50
+ before the generator stops; ``None`` follows forever.
51
+ """
52
+
53
+ handles = [path.open("r", encoding="utf-8", errors="replace") for path in paths]
54
+ idle_polls = 0
55
+ try:
56
+ while True:
57
+ got_line = False
58
+ for handle in handles:
59
+ while True:
60
+ line = handle.readline()
61
+ if not line:
62
+ break
63
+ got_line = True
64
+ yield line.rstrip("\n")
65
+ if got_line:
66
+ idle_polls = 0
67
+ continue
68
+ idle_polls += 1
69
+ if max_idle_polls is not None and idle_polls >= max_idle_polls:
70
+ return
71
+ time.sleep(poll_interval)
72
+ finally:
73
+ for handle in handles:
74
+ handle.close()
75
+
76
+
77
+ def _convert_line_mp(line: str, mode: Optional[str], mapping: Optional[str]) -> str:
78
+ return convert_line(line, mode=mode, mapping=mapping)
79
+
80
+
81
+ def process_lines(
82
+ lines: Iterable[str],
83
+ *,
84
+ mode: Optional[str],
85
+ mapping: Optional[str],
86
+ use_multiprocessing: bool,
87
+ pool_size: Optional[int],
88
+ ) -> Iterator[str]:
89
+ if use_multiprocessing:
90
+ size = pool_size or max(1, cpu_count() - 1)
91
+ with Pool(size) as pool:
92
+ worker = partial(_convert_line_mp, mode=mode, mapping=mapping)
93
+ for cef in pool.imap(worker, lines):
94
+ yield cef
95
+ else:
96
+ for line in lines:
97
+ yield convert_line(line, mode=mode, mapping=mapping)
98
+
99
+
100
+ def main(argv: Optional[list[str]] = None) -> int:
101
+ parser = argparse.ArgumentParser(description="Convert syslog lines to ArcSight CEF")
102
+ parser.add_argument("paths", nargs="*", type=Path, help="Input files (defaults to stdin)")
103
+ parser.add_argument("-o", "--output", type=Path, help="Output file")
104
+ parser.add_argument("-a", "--append", action="store_true", help="Append to --output instead of truncating it")
105
+ parser.add_argument("--mode", choices=["rfc3164", "rfc5424", "rsyslog_json", "rsyslog_file", "journald_json", "journald_short", "journald_iso"], help="Parser mode override")
106
+ parser.add_argument("--mapping", type=str, help="Mapping JSON file")
107
+ parser.add_argument("--tail", action="store_true", help="Follow file like tail -f")
108
+ parser.add_argument("--multiprocess", action="store_true", help="Process lines using a process pool")
109
+ parser.add_argument("--pool-size", type=int, help="Number of worker processes for --multiprocess")
110
+ parser.add_argument("--log-level", default="WARNING", help="Logging level")
111
+
112
+ args = parser.parse_args(argv)
113
+ logging.basicConfig(level=getattr(logging, args.log_level.upper(), logging.WARNING))
114
+
115
+ mapping = args.mapping
116
+
117
+ outputs: Iterator[str]
118
+ if args.tail and args.paths:
119
+ outputs = process_lines(follow_files(args.paths), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size)
120
+ else:
121
+ outputs = process_lines(iter_lines_from_sources(args.paths), mode=args.mode, mapping=mapping, use_multiprocessing=args.multiprocess, pool_size=args.pool_size)
122
+
123
+ if args.output:
124
+ with args.output.open("a" if args.append else "w", encoding="utf-8") as fp:
125
+ for cef in outputs:
126
+ fp.write(cef + "\n")
127
+ if args.tail:
128
+ fp.flush()
129
+ else:
130
+ for cef in outputs:
131
+ print(cef)
132
+ return 0
133
+
134
+
135
+ if __name__ == "__main__":
136
+ raise SystemExit(main())