ciphergate-cli 0.7.0__py3-none-any.whl

ciphergate/readers/line_structure.py

@@ -0,0 +1,125 @@
+"""Shared span machinery for structured line formats (JSON-in-log, access logs).
+
+A structured line format parses a text line into *regions* (spans of recognised structure) made
+of *tokens* (keys and values with raw offsets). The pipeline scans the line exactly like
+``text`` and uses the parse only to add field context, to snap findings onto token boundaries,
+and to re-encode replacements so that the structure survives rewriting.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass(frozen=True, slots=True)
+class FieldToken:
+    """A key or value inside a parsed region. Spans are raw offsets in the line."""
+
+    kind: str  # "key", "string" (value) or "literal" (a value replaced as a whole)
+    start: int  # strings: the raw value body (inside any quotes); literals: the literal itself
+    end: int
+    pointer: str  # JSON Pointer-style field path (keys: the pointer of their value)
+    key: str | None  # nearest field name, None when the value has no name
+    value: Any  # decoded value (keys: the decoded key)
+    escaped: bool = False  # raw text differs from the decoded value
+    style: str = ""  # format-specific encoding of the token in the line (for escape/unescape)
+    hint: str | None = None  # canonical field name for field-name heuristics (e.g. CEF "suser")
+
+
+@dataclass(frozen=True, slots=True)
+class FieldRegion:
+    start: int
+    end: int
+    tokens: tuple[FieldToken, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class LineStructure:
+    """How one structured line format parses and re-encodes its tokens."""
+
+    parse: Callable[[str, int], list[FieldRegion]]  # (line, max_depth) -> regions
+    unescape: Callable[[FieldToken, str], Any]  # raw part of a token -> what a transform sees
+    escape: Callable[[FieldToken, str], str]  # replacement text -> raw text for the token
+
+
+def region_tokens(regions: list[FieldRegion]) -> list[FieldToken]:
+    return [t for r in regions for t in r.tokens]
+
+
+def token_at(tokens: list[FieldToken], start: int, end: int) -> FieldToken | None:
+    """The token whose span contains ``[start, end)`` (literals: any overlap), if any."""
+    for t in tokens:
+        if t.kind == "literal" and start < t.end and t.start < end:
+            return t
+        if t.kind != "literal" and t.start <= start and end <= t.end:
+            return t
+    return None
+
+
+def snap_span(regions: list[FieldRegion], start: int, end: int) -> list[tuple[int, int]]:
+    """Split a span so that no piece crosses structural syntax.
+
+    Parts outside every region are kept. Inside a region, the span becomes the spans of the
+    value tokens it overlaps (or key tokens, if it overlaps no value); literal tokens are taken
+    whole. A span over punctuation only covers every value of its region, whole; a region with
+    no values yields nothing. A span already inside one string token, or covering one literal
+    exactly, is returned unchanged.
+    """
+    pieces: list[tuple[int, int]] = []
+    cursor = start
+    for region in regions:
+        if region.end <= start or end <= region.start:
+            continue
+        if cursor < region.start:
+            pieces.append((cursor, region.start))
+        lo, hi = max(start, region.start), min(end, region.end)
+        inside = [t for t in region.tokens if t.kind != "key" and lo < t.end and t.start < hi]
+        if not inside:
+            inside = [t for t in region.tokens if t.kind == "key" and lo < t.end and t.start < hi]
+        if not inside:
+            # Only punctuation (e.g. a secret pattern that matched an opening brace): the
+            # finding is about the structure itself, so protect every value in it, whole.
+            lo, hi = region.start, region.end
+            inside = [t for t in region.tokens if t.kind != "key"]
+        for t in inside:
+            if t.kind == "literal":
+                pieces.append((t.start, t.end))
+            else:
+                pieces.append((max(lo, t.start), min(hi, t.end)))
+        cursor = region.end
+    if cursor < end:
+        pieces.append((cursor, end))
+    return [p for p in pieces if p[0] < p[1]]
+
+
+class SpanCodec:
+    """Decodes a span to the value a transform sees and re-encodes the replacement."""
+
+    def __init__(self, structure: LineStructure, regions: list[FieldRegion]) -> None:
+        self.structure = structure
+        self.tokens = region_tokens(regions)
+
+    def decode(self, start: int, end: int, raw: str) -> Any:
+        owner = token_at(self.tokens, start, end)
+        if owner is None:
+            return raw
+        if owner.kind == "literal" or (start, end) == (owner.start, owner.end):
+            return owner.value
+        return self.structure.unescape(owner, raw)
+
+    def encode(self, start: int, end: int, replacement: Any) -> str:
+        owner = token_at(self.tokens, start, end)
+        text = replacement if isinstance(replacement, str) else str(replacement)
+        if owner is None:
+            return text
+        return self.structure.escape(owner, text)
+
+
+def structure_still_valid(
+    structure: LineStructure, line: str, before: list[FieldRegion], max_depth: int
+) -> bool:
+    """After rewriting, the line must parse into at least as many regions and tokens."""
+    after = structure.parse(line, max_depth)
+    return len(after) >= len(before) and len(region_tokens(after)) >= len(region_tokens(before))

ciphergate/readers/logfmt_reader.py

@@ -0,0 +1,100 @@
+"""logfmt adapter: text/log lines with ``key=value`` fields (``level=info user_email=a@b``).
+
+Detection never relies on the parse alone. Every line is scanned exactly like ``text`` (including
+multi-line private key blocks); the parse only adds field context on top:
+
+* findings whose span lies inside a field value are attributed to that field (``field_name`` =
+  key, ``field_path`` = ``/<key>``), so policy ``field_names``/``field_paths`` rules apply;
+* field-name heuristics and policy classifications run on each key and, when they match, cover
+  the whole value span.
+
+Grammar: whitespace-separated tokens; a field is ``key=value`` with ``key`` matching
+``[A-Za-z_][A-Za-z0-9_.-]*`` and ``value`` either bare (up to whitespace, may be empty) or
+double-quoted with backslash escapes. Other tokens (timestamps, words) are free text. A line
+with an unterminated quote gets no field context (it is still scanned as text). Lines inside a
+PEM private key block are never parsed. Output keeps the original line; only finding spans are
+replaced, so quoting is preserved. ``drop_field`` is not supported (it becomes ``review``).
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from ciphergate.constants import InputFormat
+from ciphergate.readers.text_reader import TextReader
+
+_KEY_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_.\-]{0,255}")
+
+
+@dataclass(frozen=True, slots=True)
+class LogfmtField:
+    key: str
+    value: str  # unquoted/unescaped
+    start: int  # raw value span in the line (inside the quotes for quoted values)
+    end: int
+
+
+def parse_fields(line: str) -> list[LogfmtField] | None:
+    """Fields of one line, or ``None`` when quoting is malformed (no field context then)."""
+    fields: list[LogfmtField] = []
+    i, n = 0, len(line)
+    while i < n:
+        if line[i].isspace():
+            i += 1
+            continue
+        key_match = _KEY_RE.match(line, i)
+        eq = key_match.end() if key_match else i
+        if key_match and eq < n and line[eq] == "=":
+            v = eq + 1
+            if v < n and line[v] == '"':
+                parsed = _quoted(line, v)
+                if parsed is None:
+                    return None
+                value, end = parsed
+                if end < n and not line[end].isspace():
+                    return None  # text glued to a closing quote: ambiguous
+                fields.append(LogfmtField(key_match.group(), value, v + 1, end - 1))
+                i = end
+                continue
+            end = v
+            while end < n and not line[end].isspace():
+                end += 1
+            fields.append(LogfmtField(key_match.group(), line[v:end], v, end))
+            i = end
+            continue
+        while i < n and not line[i].isspace():  # free-text token
+            if line[i] == '"':
+                return None
+            i += 1
+    return fields
+
+
+def _quoted(line: str, start: int) -> tuple[str, int] | None:
+    """Parse a quoted value starting at ``start`` (the opening quote). Returns (value, end)."""
+    out: list[str] = []
+    i = start + 1
+    n = len(line)
+    while i < n:
+        ch = line[i]
+        if ch == "\\" and i + 1 < n:
+            nxt = line[i + 1]
+            out.append({"n": "\n", "t": "\t", "r": "\r"}.get(nxt, nxt))
+            i += 2
+            continue
+        if ch == '"':
+            return "".join(out), i + 1
+        out.append(ch)
+        i += 1
+    return None
+
+
+def field_for_span(fields: list[LogfmtField], span: tuple[int, int]) -> LogfmtField | None:
+    for f in fields:
+        if f.start <= span[0] and span[1] <= f.end:
+            return f
+    return None
+
+
+class LogfmtReader(TextReader):
+    format = InputFormat.LOGFMT

ciphergate/readers/stacktrace_reader.py

@@ -0,0 +1,103 @@
+"""Stack-trace-aware log input (``--format stacktrace``).
+
+Lines are read and scanned exactly like ``text``. On top of that, a small cross-line state machine
+recognises stack trace lines and tells the analyzer what each one is:
+
+* Python: ``Traceback (most recent call last):``, then ``File "...", line N, in f`` frames,
+  indented source/caret lines under a frame, and a final ``Type: message`` exception line;
+* JVM: ``at pkg.Class.method(File.java:12)`` frames, ``Caused by:`` / ``Suppressed:`` lines and
+  ``... N more`` markers;
+* Node.js: ``at fn (file.js:1:2)`` and ``at file.js:1:2`` frames.
+
+There is no lookahead and nothing is buffered: each line is classified when it is read, from the
+lines before it, so every line keeps the ``max_record_bytes`` limit and output is still written
+line by line. Frame lines are recognised on their own, without a header, so a truncated or
+interleaved trace still has every frame classified.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from ciphergate.constants import InputFormat
+from ciphergate.readers.text_reader import TextReader
+
+_PY_START_RE = re.compile(r"[ \t]*Traceback \(most recent call last\):[ \t]*")
+_PY_FRAME_RE = re.compile(r'[ \t]+(File "[^"\r\n]{1,4096}", line \d{1,9}(?:, in [^\r\n]{1,4096})?)')
+_PY_EXCEPTION_RE = re.compile(r"[A-Za-z_][\w.]{0,255}(?::[^\r\n]*)?")
+# JVM ("at a.B.c(B.java:1)", logback "~[app.jar:1.0]") and Node ("at fn (x.js:1:2)", "at x.js:1:2").
+_FRAME_RE = re.compile(
+    r"[ \t]+(at (?:[^()\r\n]{1,4096}\([^()\r\n]{0,4096}\)(?: ~?\[[^\[\]\r\n]{0,1024}\])?"
+    r"|(?:async )?[^\s()]{1,4096}:\d{1,9}:\d{1,9}))"
+)
+_NODE_LOCATION_RE = re.compile(r":\d{1,9}:\d{1,9}\)?$")  # JVM frames carry "File.java:12"
+_CAUSE_RE = re.compile(r"[ \t]*(Caused by|Suppressed): [^\r\n]*")
+_MORE_RE = re.compile(r"[ \t]+\.\.\. \d{1,9} (?:more|common frames omitted)")
+
+FRAME = "frame"
+SOURCE = "source"  # a Python source or caret line under a frame
+EXCEPTION = "exception"
+CONTEXT = "context"  # header and "... N more" lines: part of a trace, no content of their own
+
+
+@dataclass(frozen=True, slots=True)
+class TraceLine:
+    role: str
+    pointer: str
+    span: tuple[int, int] | None = None  # FRAME/SOURCE: the part to protect (after indentation)
+    family: str = ""
+
+
+@dataclass(slots=True)
+class TraceState:
+    """What the previous lines said: inside a Python traceback, or a run of JVM/Node frames."""
+
+    kind: str | None = None  # "python" or "frames"
+    frames: int = 0
+
+    def reset(self, kind: str | None = None) -> None:
+        self.kind, self.frames = kind, 0
+
+    def frame(self, family: str, span: tuple[int, int]) -> TraceLine:
+        line = TraceLine(FRAME, f"/stack/frames/{self.frames}", span, family)
+        self.frames += 1
+        return line
+
+
+def classify(text: str, state: TraceState) -> TraceLine | None:
+    """Classify one line (without its line ending) and advance ``state``."""
+    if _PY_START_RE.fullmatch(text):
+        state.reset("python")
+        return TraceLine(CONTEXT, "/stack")
+    if m := _PY_FRAME_RE.fullmatch(text):
+        if state.kind != "python":
+            state.reset("python")  # a frame without its header (e.g. a prefixed first line)
+        return state.frame("python_frame", m.span(1))
+    if state.kind == "python":
+        stripped = text.lstrip(" \t")
+        if stripped and stripped != text:
+            last = state.frames - 1
+            pointer = f"/stack/frames/{last}/source" if state.frames else "/stack/source"
+            span = (len(text) - len(stripped), len(text))
+            return TraceLine(SOURCE, pointer, span, "python_source")
+        state.reset()
+        if stripped and _PY_EXCEPTION_RE.fullmatch(text):
+            return TraceLine(EXCEPTION, "/stack/exception")
+        return None
+    if m := _FRAME_RE.fullmatch(text):
+        if state.kind != "frames":
+            state.reset("frames")
+        family = "node_frame" if _NODE_LOCATION_RE.search(text) else "jvm_frame"
+        return state.frame(family, m.span(1))
+    if state.kind == "frames":
+        if m := _CAUSE_RE.fullmatch(text):
+            return TraceLine(EXCEPTION, "/stack/" + m.group(1).lower().replace(" ", "_"))
+        if _MORE_RE.fullmatch(text):
+            return TraceLine(CONTEXT, "/stack")
+        state.reset()
+    return None
+
+
+class StackTraceReader(TextReader):
+    format = InputFormat.STACKTRACE

ciphergate/readers/syslog_reader.py

@@ -0,0 +1,309 @@
+"""syslog adapter: RFC 5424 and RFC 3164 lines, and CEF (bare or as a syslog message).
+
+* RFC 5424: ``<34>1 2026-01-05T10:00:00Z host app 42 ID7 [origin ip="192.0.2.1"] message``.
+  Header fields become ``/timestamp``, ``/hostname``, ``/app_name``, ``/procid``, ``/msgid``;
+  structured-data parameters become ``/sd/<sd-id>/<name>`` (escapes ``\\"``, ``\\\\``, ``\\]``);
+  the message is ``/msg``.
+* RFC 3164 / BSD, including files written by rsyslog without ``<PRI>`` and with RFC 3339
+  timestamps: ``Jan  5 10:00:00 host sshd[42]: message`` gives ``/timestamp``, ``/hostname``,
+  ``/app_name`` (the tag), ``/procid`` and ``/msg``. A line needs a ``tag:`` (or a CEF message)
+  to be recognised.
+* CEF: ``CEF:0|Vendor|Product|1.0|100|Name|5|suser=alice src=192.0.2.1``, on its own or as
+  the message of either syslog layout. Header fields become ``/cef/device_vendor`` ...
+  ``/cef/severity``, with CEF ``Name`` as ``/cef/event_name`` (escapes ``\\|``, ``\\\\``);
+  extensions become ``/cef/extension/<key>``
+  (escapes ``\\=``, ``\\\\``, ``\\n``, ``\\r``). Common short keys get a canonical name for the
+  field-name heuristics (``suser`` -> ``source_user_name``, ``src`` -> ``source_ip``), and
+  ``csN``/``cnN`` fields use their ``csNLabel`` value; the raw key stays the field name.
+
+Detection never relies on the parse alone: every line is scanned exactly like ``text`` and the
+parse only adds field context. Replacements are re-escaped for their field; header fields get
+``_`` for whitespace (an RFC 3164 tag also for anything outside ``[A-Za-z0-9_./-]``). Every
+rewritten line must still parse with as many fields, or the run fails. ``-`` (NILVALUE) fields
+get no field context. ``drop_field`` is not supported (it becomes ``review``).
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from ciphergate.constants import InputFormat
+from ciphergate.readers.json_reader import pointer_escape
+from ciphergate.readers.line_structure import FieldRegion, FieldToken, LineStructure
+from ciphergate.readers.text_reader import TextReader
+
+_PRI = r"<\d{1,3}>"
+_RFC5424_RE = re.compile(
+    _PRI + r"[1-9]\d{0,2} (\S{1,64}) (\S{1,255}) (\S{1,48}) (\S{1,128}) (\S{1,32}) "
+)
+_TS3164 = r"[A-Z][a-z]{2} {1,2}\d{1,2} \d{2}:\d{2}:\d{2}"
+_TS3339 = r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?(?:Z|[+-]\d{2}:\d{2})"
+_RFC3164_RE = re.compile(rf"(?:{_PRI})?({_TS3164}|{_TS3339}) (\S{{1,255}}) ")
+_TAG_RE = re.compile(r"([A-Za-z0-9_./-]{1,48})(?:\[([0-9A-Za-z_-]{1,32})\])?: ")
+_SD_NAME_RE = re.compile(r'[^\s="\]]{1,32}')
+_CEF_START = "CEF:"
+_CEF_VERSION_RE = re.compile(r"CEF:\d{1,2}(?:\.\d{1,2})?\|")
+_CEF_KEY_RE = re.compile(r"[A-Za-z0-9_.-]{1,64}=")
+_CEF_HEADERS = (
+    "device_vendor",
+    "device_product",
+    "device_version",
+    "signature_id",
+    "event_name",  # CEF "Name"; not "name", which the heuristics read as a person name
+    "severity",
+)
+_MAX_SD_ELEMENTS = 64
+_MAX_SD_PARAMS = 64
+_MAX_CEF_FIELDS = 256
+_MAX_VALUE = 16384
+
+# Canonical names for common CEF extension keys, used only for field-name heuristics.
+CEF_HINTS = {
+    "src": "source_ip",
+    "dst": "destination_ip",
+    "dvc": "device_ip",
+    "c6a1": "device_ipv6",
+    "c6a2": "source_ipv6",
+    "c6a3": "destination_ipv6",
+    "shost": "source_hostname",
+    "dhost": "destination_hostname",
+    "dvchost": "device_hostname",
+    "suser": "source_user_name",
+    "duser": "destination_user_name",
+    "suid": "source_user_id",
+    "duid": "destination_user_id",
+    "smac": "source_mac_address",
+    "dmac": "destination_mac_address",
+    "dvcmac": "device_mac_address",
+}
+_LABELLED = re.compile(r"(?:cs|cn|cfp|c6a|flexString|flexNumber|flexDate)\d{1,2}")
+
+# Characters an RFC 3164 tag may hold (other header fields only exclude whitespace).
+_TAG_SAFE = frozenset("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_./-")
+
+
+def _unescape_chars(raw: str, escapes: dict[str, str]) -> str:
+    if "\\" not in raw:
+        return raw
+    out: list[str] = []
+    i, n = 0, len(raw)
+    while i < n:
+        ch = raw[i]
+        if ch == "\\" and i + 1 < n and raw[i + 1] in escapes:
+            out.append(escapes[raw[i + 1]])
+            i += 2
+            continue
+        out.append(ch)
+        i += 1
+    return "".join(out)
+
+
+_UNESCAPES = {
+    "sd": {'"': '"', "\\": "\\", "]": "]"},
+    "cef_header": {"|": "|", "\\": "\\"},
+    "cef_ext": {"=": "=", "\\": "\\", "n": "\n", "r": "\r"},
+}
+
+
+def _decode(style: str, raw: str) -> str:
+    escapes = _UNESCAPES.get(style)
+    return _unescape_chars(raw, escapes) if escapes else raw
+
+
+def _escape(owner: FieldToken, text: str) -> str:
+    style = owner.style
+    if style == "sd":
+        return "".join("\\" + ch if ch in '"\\]' else _no_control(ch) for ch in text)
+    if style == "cef_header":
+        return "".join("\\" + ch if ch in "|\\" else _no_control(ch) for ch in text)
+    if style == "cef_ext":
+        table = {"\\": "\\\\", "=": "\\=", "\n": "\\n", "\r": "\\r"}
+        return "".join(table.get(ch, _no_control(ch)) for ch in text)
+    if style == "tag":
+        return "".join(ch if ch in _TAG_SAFE else "_" for ch in text) or "_"
+    if style == "bare":
+        return "".join("_" if ch.isspace() else _no_control(ch) for ch in text) or "_"
+    return "".join(_no_control(ch) for ch in text)  # free-text message
+
+
+def _no_control(ch: str) -> str:
+    return " " if ord(ch) < 0x20 or ord(ch) == 0x7F else ch
+
+
+def _unescape(owner: FieldToken, raw: str) -> Any:
+    return _decode(owner.style, raw)
+
+
+class _Tokens:
+    def __init__(self, line: str) -> None:
+        self.line = line
+        self.items: list[FieldToken] = []
+
+    def value(
+        self,
+        pointer: str,
+        key: str,
+        start: int,
+        end: int,
+        style: str,
+        hint: str | None = None,
+        *,
+        nil: bool = True,
+    ) -> None:
+        raw = self.line[start:end]
+        if raw == "" or (nil and raw == "-"):
+            return  # NILVALUE / empty: nothing to protect
+        value = _decode(style, raw)
+        self.items.append(
+            FieldToken("string", start, end, pointer, key, value, value != raw, style, hint)
+        )
+
+    def key(self, pointer: str, name: str, start: int, end: int) -> None:
+        self.items.append(FieldToken("key", start, end, pointer, name, name))
+
+
+def _structured_data(t: _Tokens, i: int) -> int | None:
+    """Parse STRUCTURED-DATA at ``i``; returns its end, or ``None`` when malformed."""
+    line, n = t.line, len(t.line)
+    if line.startswith("-", i) and (i + 1 == n or line[i + 1] == " "):
+        return i + 1
+    elements = 0
+    while i < n and line[i] == "[":
+        elements += 1
+        if elements > _MAX_SD_ELEMENTS:
+            return None
+        sd_id = _SD_NAME_RE.match(line, i + 1)
+        if sd_id is None:
+            return None
+        base = f"/sd/{pointer_escape(sd_id.group())}"
+        i = sd_id.end()
+        params = 0
+        while i < n and line[i] == " ":
+            params += 1
+            name = _SD_NAME_RE.match(line, i + 1)
+            if params > _MAX_SD_PARAMS or name is None:
+                return None
+            eq = name.end()
+            if not line.startswith('="', eq):
+                return None
+            j, limit = eq + 2, min(n, eq + 2 + _MAX_VALUE)
+            while j < limit and line[j] != '"':
+                j += 2 if line[j] == "\\" else 1
+            if j >= limit:
+                return None
+            pointer = f"{base}/{pointer_escape(name.group())}"
+            t.key(pointer, name.group(), name.start(), name.end())
+            t.value(pointer, name.group(), eq + 2, j, "sd", nil=False)
+            i = j + 1
+        if i >= n or line[i] != "]":
+            return None
+        i += 1
+    return i if elements else None
+
+
+def _cef(t: _Tokens, start: int) -> int | None:
+    """Parse a CEF record starting at ``start``; returns its end, or ``None`` if it is not CEF."""
+    line, n = t.line, len(t.line)
+    head = _CEF_VERSION_RE.match(line, start)
+    if head is None:
+        return None
+    i = head.end()
+    for name in _CEF_HEADERS:
+        j, limit = i, min(n, i + _MAX_VALUE)
+        while j < limit and line[j] != "|":
+            j += 2 if line[j] == "\\" else 1
+        if j >= limit:
+            return None
+        t.value(f"/cef/{name}", name, i, j, "cef_header", nil=False)
+        i = j + 1
+    _cef_extension(t, i, n)
+    return n
+
+
+def _cef_extension(t: _Tokens, start: int, end: int) -> None:
+    line = t.line
+    # A key starts the extension or follows a space; "=" inside values must be escaped.
+    keys: list[tuple[int, int]] = []  # (key start, value start)
+    i = start
+    while i < end and len(keys) < _MAX_CEF_FIELDS:
+        if i == start or line[i - 1] == " ":
+            m = _CEF_KEY_RE.match(line, i, end)
+            if m is not None:
+                keys.append((i, m.end()))
+                i = m.end()
+                continue
+        i += 1
+    labels: dict[str, str] = {}
+    spans: list[tuple[str, int, int]] = []
+    for index, (key_start, value_start) in enumerate(keys):
+        value_end = keys[index + 1][0] - 1 if index + 1 < len(keys) else end
+        while value_end > value_start and line[value_end - 1] == " ":
+            value_end -= 1
+        name = line[key_start : value_start - 1]
+        spans.append((name, value_start, value_end))
+        if name.endswith("Label"):
+            labels[name[: -len("Label")]] = _decode("cef_ext", line[value_start:value_end])
+    for name, value_start, value_end in spans:
+        hint = CEF_HINTS.get(name)
+        if hint is None and _LABELLED.fullmatch(name):
+            hint = labels.get(name)
+        pointer = f"/cef/extension/{pointer_escape(name)}"
+        t.key(pointer, name, value_start - len(name) - 1, value_start - 1)
+        t.value(pointer, name, value_start, value_end, "cef_ext", hint, nil=False)
+
+
+def _message(t: _Tokens, start: int) -> None:
+    """A syslog MSG: CEF if it is one, else free text."""
+    line = t.line
+    if line.startswith("", start):
+        start += 1
+    if line.startswith(_CEF_START, start) and _cef(t, start) is not None:
+        return
+    t.value("/msg", "msg", start, len(line), "msg", nil=False)
+
+
+def parse_regions(line: str, max_depth: int = 64) -> list[FieldRegion]:
+    """The syslog/CEF fields of ``line`` as one region, or ``[]`` for other layouts."""
+    t = _Tokens(line)
+    if line.startswith(_CEF_START):
+        if _cef(t, 0) is None:
+            return []
+        return [FieldRegion(0, len(line), tuple(t.items))]
+    m = _RFC5424_RE.match(line)
+    if m is not None:
+        for group, name in enumerate(("timestamp", "hostname", "app_name", "procid", "msgid"), 1):
+            t.value("/" + name, name, m.start(group), m.end(group), "bare")
+        end = _structured_data(t, m.end())
+        if end is None:
+            return []
+        if end < len(line):
+            if line[end] != " ":
+                return []
+            _message(t, end + 1)
+        return [FieldRegion(0, len(line), tuple(t.items))]
+    m = _RFC3164_RE.match(line)
+    if m is None:
+        return []
+    t.value("/timestamp", "timestamp", m.start(1), m.end(1), "bare")
+    t.value("/hostname", "hostname", m.start(2), m.end(2), "bare")
+    rest = m.end()
+    if line.startswith(_CEF_START, rest):
+        if _cef(t, rest) is None:
+            return []
+        return [FieldRegion(0, len(line), tuple(t.items))]
+    tag = _TAG_RE.match(line, rest)
+    if tag is None:
+        return []
+    t.value("/app_name", "app_name", tag.start(1), tag.end(1), "tag")
+    if tag.group(2) is not None:
+        t.value("/procid", "procid", tag.start(2), tag.end(2), "tag")
+    _message(t, tag.end())
+    return [FieldRegion(0, len(line), tuple(t.items))]
+
+
+SYSLOG_STRUCTURE = LineStructure(parse=parse_regions, unescape=_unescape, escape=_escape)
+
+
+class SyslogReader(TextReader):
+    format = InputFormat.SYSLOG