syslog2cef 0.1.3__tar.gz → 0.1.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. {syslog2cef-0.1.3/syslog2cef.egg-info → syslog2cef-0.1.4}/PKG-INFO +1 -1
  2. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/pyproject.toml +1 -1
  3. {syslog2cef-0.1.3 → syslog2cef-0.1.4/syslog2cef.egg-info}/PKG-INFO +1 -1
  4. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/syslogcef/cli.py +2 -1
  5. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/syslogcef/net.py +42 -3
  6. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/syslogcef/validation.py +24 -11
  7. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/tests/test_net.py +89 -0
  8. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/tests/test_validation.py +24 -1
  9. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/LICENSE +0 -0
  10. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/README.md +0 -0
  11. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/setup.cfg +0 -0
  12. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/syslog2cef.egg-info/SOURCES.txt +0 -0
  13. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/syslog2cef.egg-info/dependency_links.txt +0 -0
  14. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/syslog2cef.egg-info/entry_points.txt +0 -0
  15. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/syslog2cef.egg-info/requires.txt +0 -0
  16. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/syslog2cef.egg-info/top_level.txt +0 -0
  17. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/syslogcef/__init__.py +0 -0
  18. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/syslogcef/__main__.py +0 -0
  19. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/syslogcef/adaptive.py +0 -0
  20. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/syslogcef/api.py +0 -0
  21. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/syslogcef/cef.py +0 -0
  22. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/syslogcef/mappings/__init__.py +0 -0
  23. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/syslogcef/mappings/cisco_asa.json +0 -0
  24. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/syslogcef/mappings/cisco_ios.json +0 -0
  25. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/syslogcef/mappings/f5.json +0 -0
  26. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/syslogcef/mappings/fortinet.json +0 -0
  27. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/syslogcef/mappings/linux.json +0 -0
  28. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/syslogcef/mappings/sophos.json +0 -0
  29. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/syslogcef/mappings/vmware.json +0 -0
  30. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/syslogcef/normalizer.py +0 -0
  31. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/syslogcef/parsers.py +0 -0
  32. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/syslogcef/utils.py +0 -0
  33. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/tests/test_adaptive.py +0 -0
  34. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/tests/test_cef.py +0 -0
  35. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/tests/test_cli.py +0 -0
  36. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/tests/test_normalizer.py +0 -0
  37. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/tests/test_parsers.py +0 -0
  38. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/tests/test_parsers_extra.py +0 -0
  39. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/tests/test_pipeline.py +0 -0
  40. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/tests/test_real_world.py +0 -0
  41. {syslog2cef-0.1.3 → syslog2cef-0.1.4}/tests/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: syslog2cef
3
- Version: 0.1.3
3
+ Version: 0.1.4
4
4
  Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
5
  Author-email: Tamir Suliman <allamiro@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/allamiro/syslogcef
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "syslog2cef"
7
- version = "0.1.3"
7
+ version = "0.1.4"
8
8
  description = "Convert RFC3164/RFC5424 syslog lines into ArcSight CEF"
9
9
  authors = [{name = "Tamir Suliman", email = "allamiro@gmail.com"}]
10
10
  readme = "README.md"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: syslog2cef
3
- Version: 0.1.3
3
+ Version: 0.1.4
4
4
  Summary: Convert RFC3164/RFC5424 syslog lines into ArcSight CEF
5
5
  Author-email: Tamir Suliman <allamiro@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/allamiro/syslogcef
@@ -160,8 +160,9 @@ def main(argv: Optional[list[str]] = None) -> int:
160
160
  for cef in outputs:
161
161
  sender.send(cef)
162
162
  else:
163
+ stream_flush = bool(args.tail or args.listen)
163
164
  for cef in outputs:
164
- print(cef)
165
+ print(cef, flush=stream_flush)
165
166
  except CEFValidationError as exc:
166
167
  print(f"syslogcef: strict validation failed: {exc}", file=sys.stderr)
167
168
  return 1
@@ -16,6 +16,11 @@ logger = logging.getLogger(__name__)
16
16
 
17
17
  MAX_DATAGRAM = 65535
18
18
 
19
+ # A TCP client that streams data without ever sending a newline must not be
20
+ # able to grow its buffer without bound; connections exceeding this are
21
+ # dropped.
22
+ MAX_TCP_BUFFER = 1024 * 1024
23
+
19
24
 
20
25
  def parse_endpoint(spec: str) -> Tuple[str, str, int]:
21
26
  """Parse ``udp:514``, ``tcp:5514``, or ``udp:10.0.0.5:514``."""
@@ -119,6 +124,20 @@ def _listen_tcp(host, port, max_messages, ready_callback) -> Iterator[str]:
119
124
  count += 1
120
125
  continue
121
126
  buffers[sock] += data
127
+ if len(buffers[sock]) > MAX_TCP_BUFFER and b"\n" not in buffers[sock]:
128
+ peer = None
129
+ try:
130
+ peer = sock.getpeername()
131
+ except OSError:
132
+ pass
133
+ logger.warning(
134
+ "Dropping connection %s: %d bytes without a newline",
135
+ peer, len(buffers[sock]),
136
+ )
137
+ sel.unregister(sock)
138
+ sock.close()
139
+ buffers.pop(sock, None)
140
+ continue
122
141
  while b"\n" in buffers[sock]:
123
142
  raw, buffers[sock] = buffers[sock].split(b"\n", 1)
124
143
  line = raw.decode("utf-8", "replace").rstrip("\r")
@@ -159,7 +178,11 @@ class RateLimiter:
159
178
  now = self._clock()
160
179
  if now < self._next:
161
180
  self._sleep(self._next - now)
162
- self._next = max(self._next + self.interval, self._clock())
181
+ now = self._clock()
182
+ # Schedule the next slot one full interval after the later of the
183
+ # previous slot and now, so falling behind never permits
184
+ # back-to-back sends that exceed the configured rate.
185
+ self._next = max(self._next, now) + self.interval
163
186
 
164
187
 
165
188
  class SyslogSender:
@@ -225,6 +248,7 @@ class KafkaSender:
225
248
  def __init__(self, broker: str, topic: str, *, limiter: Optional[RateLimiter] = None, producer=None):
226
249
  self.topic = topic
227
250
  self.limiter = limiter
251
+ self._async_error: Optional[Exception] = None
228
252
  if producer is None:
229
253
  try:
230
254
  from kafka import KafkaProducer
@@ -235,10 +259,21 @@ class KafkaSender:
235
259
  producer = KafkaProducer(bootstrap_servers=broker)
236
260
  self.producer = producer
237
261
 
262
+ def _record_error(self, exc: Exception) -> None:
263
+ logger.error("Kafka delivery failed: %s", exc)
264
+ self._async_error = exc
265
+
238
266
  def send(self, record: str) -> None:
267
+ if self._async_error is not None:
268
+ raise ConnectionError(
269
+ f"Kafka delivery previously failed: {self._async_error}"
270
+ )
239
271
  if self.limiter:
240
272
  self.limiter.wait()
241
- self.producer.send(self.topic, record.encode("utf-8"))
273
+ future = self.producer.send(self.topic, record.encode("utf-8"))
274
+ add_errback = getattr(future, "add_errback", None)
275
+ if add_errback:
276
+ add_errback(self._record_error)
242
277
 
243
278
  def close(self) -> None:
244
279
  try:
@@ -247,13 +282,17 @@ class KafkaSender:
247
282
  close = getattr(self.producer, "close", None)
248
283
  if close:
249
284
  close()
285
+ if self._async_error is not None:
286
+ raise ConnectionError(
287
+ f"Kafka delivery failed: {self._async_error}"
288
+ )
250
289
 
251
290
 
252
291
  def create_sender(url: str, *, eps: Optional[float] = None):
253
292
  """Create a sender from udp://host:port, tcp://host:port, or kafka://broker/topic."""
254
293
 
255
294
  parsed = urlparse(url)
256
- limiter = RateLimiter(eps) if eps else None
295
+ limiter = RateLimiter(eps) if eps is not None else None
257
296
  if parsed.scheme in ("udp", "tcp"):
258
297
  if not parsed.hostname or not parsed.port:
259
298
  raise ValueError(f"Invalid send target {url!r}; expected {parsed.scheme}://HOST:PORT")
@@ -11,6 +11,7 @@ from __future__ import annotations
11
11
 
12
12
  import ipaddress
13
13
  import re
14
+ from datetime import datetime
14
15
  from dataclasses import dataclass
15
16
  from typing import Dict, List, Tuple
16
17
 
@@ -31,23 +32,28 @@ CEF_KEYS: Dict[str, Tuple[str, int]] = {
31
32
  "c6a1": ("ipv6", 0), "c6a2": ("ipv6", 0), "c6a3": ("ipv6", 0), "c6a4": ("ipv6", 0),
32
33
  "c6a1Label": (_S, 1023), "c6a2Label": (_S, 1023), "c6a3Label": (_S, 1023), "c6a4Label": (_S, 1023),
33
34
  "destinationDnsDomain": (_S, 255), "destinationServiceName": (_S, 1023),
34
- "destinationTranslatedAddress": ("ip", 0), "destinationTranslatedPort": ("port", 0),
35
+ "destinationTranslatedAddress": ("ipv4", 0), "destinationTranslatedPort": ("port", 0),
35
36
  "deviceDirection": ("int", 0), "deviceDnsDomain": (_S, 255),
36
37
  "deviceExternalId": (_S, 255), "deviceFacility": (_S, 1023),
37
38
  "deviceInboundInterface": (_S, 128), "deviceOutboundInterface": (_S, 128),
38
39
  "deviceNtDomain": (_S, 255), "devicePayloadId": (_S, 128),
39
- "deviceProcessName": (_S, 1023), "deviceTranslatedAddress": ("ip", 0),
40
+ "deviceProcessName": (_S, 1023), "deviceTranslatedAddress": ("ipv4", 0),
40
41
  "deviceCustomDate1": ("ts", 0), "deviceCustomDate2": ("ts", 0),
41
42
  "deviceCustomDate1Label": (_S, 1023), "deviceCustomDate2Label": (_S, 1023),
42
43
  "dhost": (_S, 1023), "dmac": ("mac", 0), "dntdom": (_S, 255),
43
44
  "dpid": ("int", 0), "dpriv": (_S, 1023), "dproc": (_S, 1023),
44
- "dpt": ("port", 0), "dst": ("ip", 0), "dtz": (_S, 255),
45
+ "dpt": ("port", 0), "dst": ("ipv4", 0), "dtz": (_S, 255),
45
46
  "duid": (_S, 1023), "duser": (_S, 1023),
46
- "dvc": ("ip", 0), "dvchost": (_S, 100), "dvcpid": ("int", 0),
47
+ "dvc": ("ipv4", 0), "dvchost": (_S, 100), "dvcpid": ("int", 0),
47
48
  "end": ("ts", 0), "externalId": (_S, 40),
48
49
  "fileCreateTime": ("ts", 0), "fileHash": (_S, 255), "fileId": (_S, 1023),
49
50
  "fileModificationTime": ("ts", 0), "filePath": (_S, 1023),
50
51
  "filePermission": (_S, 1023), "fileType": (_S, 1023),
52
+ "oldFileCreateTime": ("ts", 0), "oldFileHash": (_S, 255),
53
+ "oldFileId": (_S, 1023), "oldFileModificationTime": ("ts", 0),
54
+ "oldFileName": (_S, 1023), "oldFilePath": (_S, 1023),
55
+ "oldFilePermission": (_S, 1023), "oldFileSize": ("int", 0),
56
+ "oldFileType": (_S, 1023),
51
57
  "flexDate1": ("ts", 0), "flexDate1Label": (_S, 128),
52
58
  "flexString1": (_S, 1023), "flexString1Label": (_S, 128),
53
59
  "flexString2": (_S, 1023), "flexString2Label": (_S, 128),
@@ -60,16 +66,15 @@ CEF_KEYS: Dict[str, Tuple[str, int]] = {
60
66
  "rt": ("ts", 0), "start": ("ts", 0),
61
67
  "shost": (_S, 1023), "smac": ("mac", 0), "sntdom": (_S, 255),
62
68
  "sourceDnsDomain": (_S, 255), "sourceServiceName": (_S, 1023),
63
- "sourceTranslatedAddress": ("ip", 0), "sourceTranslatedPort": ("port", 0),
69
+ "sourceTranslatedAddress": ("ipv4", 0), "sourceTranslatedPort": ("port", 0),
64
70
  "spid": ("int", 0), "spriv": (_S, 1023), "sproc": (_S, 1023),
65
- "spt": ("port", 0), "src": ("ip", 0),
71
+ "spt": ("port", 0), "src": ("ipv4", 0),
66
72
  "suid": (_S, 1023), "suser": (_S, 1023),
67
73
  "type": ("int", 0),
68
74
  "rawEvent": (_S, 4000),
69
75
  }
70
76
 
71
77
  MAC_RE = re.compile(r"^[0-9a-fA-F]{2}(:[0-9a-fA-F]{2}){5}$")
72
- TS_RE = re.compile(r"^[A-Z][a-z]{2} \d{2} \d{4} \d{2}:\d{2}:\d{2}$")
73
78
 
74
79
 
75
80
  class CEFValidationError(ValueError):
@@ -124,6 +129,10 @@ def _check(key: str, value: str) -> str | None:
124
129
  if not _is_ip(value):
125
130
  return f"expected an IP address, got {value!r}"
126
131
  return None
132
+ if kind == "ipv4":
133
+ if not _is_ip(value, 4):
134
+ return f"expected an IPv4 address (use c6a1-c6a4 for IPv6), got {value!r}"
135
+ return None
127
136
  if kind == "ipv6":
128
137
  if not _is_ip(value, 6):
129
138
  return f"expected an IPv6 address, got {value!r}"
@@ -134,10 +143,14 @@ def _check(key: str, value: str) -> str | None:
134
143
  return None
135
144
  if kind == "ts":
136
145
  if value.isdigit():
137
- return None # epoch milliseconds
138
- if TS_RE.match(value):
139
- return None
140
- return f"expected 'MMM dd yyyy HH:mm:ss' or epoch milliseconds, got {value!r}"
146
+ if 10 <= len(value) <= 19:
147
+ return None # epoch seconds through nanoseconds
148
+ return f"epoch timestamp has implausible magnitude: {value!r}"
149
+ try:
150
+ datetime.strptime(value, "%b %d %Y %H:%M:%S")
151
+ except ValueError:
152
+ return f"expected 'MMM dd yyyy HH:mm:ss' or epoch milliseconds, got {value!r}"
153
+ return None
141
154
  return None
142
155
 
143
156
 
@@ -178,3 +178,92 @@ def test_kafka_sender_uses_injected_producer():
178
178
  sender.close()
179
179
  assert producer.sent == [("cef-events", b"CEF:0|a|b|1|c|d|5|")]
180
180
  assert producer.flushed
181
+
182
+
183
+ # --- round-2 review findings --------------------------------------------------
184
+
185
+ def test_tcp_flood_without_newline_drops_connection(monkeypatch):
186
+ import syslogcef.net as net
187
+
188
+ monkeypatch.setattr(net, "MAX_TCP_BUFFER", 4096)
189
+
190
+ def send(port):
191
+ flood = socket.create_connection(("127.0.0.1", port), timeout=5)
192
+ try:
193
+ flood.sendall(b"x" * 20000) # no newline: must be dropped, not buffered
194
+ except OSError:
195
+ pass
196
+ finally:
197
+ flood.close()
198
+ good = socket.create_connection(("127.0.0.1", port), timeout=5)
199
+ good.sendall(b"legit line\n")
200
+ good.close()
201
+
202
+ ready, thread = _run_when_ready(send)
203
+ lines = list(
204
+ listen_lines("tcp", "127.0.0.1", 0, max_messages=1, ready_callback=ready)
205
+ )
206
+ thread.join(5)
207
+ assert lines == ["legit line"]
208
+
209
+
210
+ def test_rate_limiter_keeps_spacing_after_falling_behind():
211
+ from syslogcef.net import RateLimiter
212
+
213
+ clock_value = [0.0]
214
+ sleeps = []
215
+
216
+ def clock():
217
+ return clock_value[0]
218
+
219
+ def sleep(seconds):
220
+ sleeps.append(seconds)
221
+ clock_value[0] += seconds
222
+
223
+ limiter = RateLimiter(1, clock=clock, sleep=sleep) # 1s interval
224
+ limiter.wait() # t=0, no sleep
225
+ clock_value[0] = 5.0 # long processing pause
226
+ limiter.wait() # behind schedule: no sleep, but next slot = 6.0
227
+ limiter.wait() # must sleep a full interval, not send immediately
228
+ assert sleeps == [1.0]
229
+
230
+
231
+ def test_eps_zero_is_rejected():
232
+ from syslogcef.net import create_sender
233
+
234
+ with pytest.raises(ValueError):
235
+ create_sender("udp://127.0.0.1:5514", eps=0)
236
+ with pytest.raises(ValueError):
237
+ create_sender("udp://127.0.0.1:5514", eps=-5)
238
+
239
+
240
+ def test_kafka_async_failure_surfaces():
241
+ from syslogcef.net import KafkaSender
242
+
243
+ class FakeFuture:
244
+ def __init__(self):
245
+ self.errbacks = []
246
+
247
+ def add_errback(self, fn):
248
+ self.errbacks.append(fn)
249
+
250
+ class FakeProducer:
251
+ def __init__(self):
252
+ self.futures = []
253
+
254
+ def send(self, topic, payload):
255
+ fut = FakeFuture()
256
+ self.futures.append(fut)
257
+ return fut
258
+
259
+ def flush(self):
260
+ pass
261
+
262
+ producer = FakeProducer()
263
+ sender = KafkaSender("broker:9092", "t", producer=producer)
264
+ sender.send("record one")
265
+ producer.futures[0].errbacks[0](RuntimeError("broker down"))
266
+ with pytest.raises(ConnectionError):
267
+ sender.send("record two")
268
+ with pytest.raises(ConnectionError):
269
+ sender.close()
@@ -8,7 +8,7 @@ from syslogcef.validation import CEFValidationError, validate_extensions
8
8
 
9
9
  def test_valid_extensions_produce_no_fatal_findings():
10
10
  findings = validate_extensions(
11
- {"src": "10.0.0.1", "dst": "2001:db8::1", "spt": "443", "msg": "hello", "smac": "00:0d:60:af:1b:61"}
11
+ {"src": "10.0.0.1", "dst": "10.0.0.2", "c6a1": "2001:db8::1", "spt": "443", "msg": "hello", "smac": "00:0d:60:af:1b:61"}
12
12
  )
13
13
  assert [f for f in findings if f.fatal] == []
14
14
 
@@ -64,3 +64,26 @@ def test_cli_strict_exits_nonzero(tmp_path):
64
64
  )
65
65
  assert result.returncode == 1
66
66
  assert b"strict validation failed" in result.stderr
67
+
68
+
69
+ # --- round-2 review findings --------------------------------------------------
70
+
71
+ def test_ipv4_only_fields_reject_ipv6():
72
+ findings = validate_extensions({"src": "2001:db8::1", "c6a1": "2001:db8::1"})
73
+ fatal = [f.key for f in findings if f.fatal]
74
+ assert fatal == ["src"]
75
+
76
+
77
+ def test_old_file_keys_validated():
78
+ findings = validate_extensions({"oldFileSize": "not-a-number", "oldFileHash": "abc"})
79
+ fatal = [f.key for f in findings if f.fatal]
80
+ assert fatal == ["oldFileSize"]
81
+
82
+
83
+ def test_timestamp_values_actually_parsed():
84
+ bad = validate_extensions({"rt": "Foo 99 2020 99:99:99"})
85
+ assert bad and bad[0].fatal
86
+ bad2 = validate_extensions({"rt": "123"}) # implausible epoch
87
+ assert bad2 and bad2[0].fatal
88
+ ok = validate_extensions({"rt": "Feb 29 2020 12:00:00"})
89
+ assert [f for f in ok if f.fatal] == []
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes