ciphergate-cli 0.7.0__py3-none-any.whl

ciphergate/readers/__init__.py

@@ -0,0 +1,113 @@
+"""Format adapters (readers + writers): CSV, JSON, JSONL/NDJSON, text/log and log formats."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import IO
+
+from ciphergate.constants import InputFormat
+from ciphergate.errors import InputFormatError, UsageError
+from ciphergate.readers.accesslog_reader import ACCESSLOG_STRUCTURE, AccessLogReader
+from ciphergate.readers.base import Reader, ReaderOptions, Record, Writer
+from ciphergate.readers.csv_reader import CSVReader, CSVWriter
+from ciphergate.readers.json_reader import JSONReader, JSONWriter
+from ciphergate.readers.jsonl_reader import JSONLReader, JSONLWriter
+from ciphergate.readers.jsonlog_reader import JSONLOG_STRUCTURE, JsonLogReader
+from ciphergate.readers.line_structure import LineStructure
+from ciphergate.readers.logfmt_reader import LogfmtReader
+from ciphergate.readers.stacktrace_reader import StackTraceReader
+from ciphergate.readers.syslog_reader import SYSLOG_STRUCTURE, SyslogReader
+from ciphergate.readers.text_reader import TextReader, TextWriter
+
+# Line formats whose lines are parsed into fields (see readers/line_structure.py).
+STRUCTURED_LINE_FORMATS: dict[InputFormat, LineStructure] = {
+    InputFormat.JSONLOG: JSONLOG_STRUCTURE,
+    InputFormat.ACCESSLOG: ACCESSLOG_STRUCTURE,
+    InputFormat.SYSLOG: SYSLOG_STRUCTURE,
+}
+
+EXTENSION_FORMATS: dict[str, InputFormat] = {
+    ".csv": InputFormat.CSV,
+    ".json": InputFormat.JSON,
+    ".jsonl": InputFormat.JSONL,
+    ".ndjson": InputFormat.JSONL,
+    ".txt": InputFormat.TEXT,
+    ".log": InputFormat.TEXT,
+}
+
+_JSON_PREFIXES = (
+    b"{",
+    b"[",
+    b'"',
+    b"-",
+    b"true",
+    b"false",
+    b"null",
+    *(bytes([d]) for d in b"0123456789"),
+)
+
+
+def detect_format(path: Path, explicit: InputFormat | None = None) -> InputFormat:
+    """Explicit ``--format`` wins; else use the extension, refusing contradicting content."""
+    if explicit is not None:
+        return explicit
+    fmt = EXTENSION_FORMATS.get(path.suffix.lower())
+    if fmt is None:
+        raise UsageError(
+            "cannot infer the input format from this file extension; pass --format "
+            + "|".join(InputFormat),
+            code="format_required",
+        )
+    if fmt in (InputFormat.JSON, InputFormat.JSONL):
+        with path.open("rb") as fh:
+            head = fh.read(4096).lstrip(b"\xef\xbb\xbf \t\r\n")
+        if head and not head.startswith(_JSON_PREFIXES):
+            raise InputFormatError(
+                "file content does not look like JSON despite its extension; pass --format",
+                code="format_mismatch",
+            )
+    return fmt
+
+
+def open_reader(path: Path, fmt: InputFormat, options: ReaderOptions) -> Reader:
+    if fmt == InputFormat.CSV:
+        return CSVReader(path, options)
+    if fmt == InputFormat.JSON:
+        return JSONReader(path, options)
+    if fmt == InputFormat.JSONL:
+        return JSONLReader(path, options)
+    if fmt == InputFormat.LOGFMT:
+        return LogfmtReader(path, options)
+    if fmt == InputFormat.JSONLOG:
+        return JsonLogReader(path, options)
+    if fmt == InputFormat.ACCESSLOG:
+        return AccessLogReader(path, options)
+    if fmt == InputFormat.SYSLOG:
+        return SyslogReader(path, options)
+    if fmt == InputFormat.STACKTRACE:
+        return StackTraceReader(path, options)
+    return TextReader(path, options)
+
+
+def make_writer(stream: IO[str], reader: Reader, dropped_columns: set[int]) -> Writer:
+    if isinstance(reader, CSVReader):
+        return CSVWriter(stream, reader, dropped_columns)
+    if isinstance(reader, JSONReader):
+        return JSONWriter(stream, reader)
+    if isinstance(reader, JSONLReader):
+        return JSONLWriter(stream)
+    if isinstance(reader, TextReader):
+        return TextWriter(stream, reader)
+    raise TypeError("unsupported reader")
+
+
+__all__ = [
+    "EXTENSION_FORMATS",
+    "Reader",
+    "ReaderOptions",
+    "Record",
+    "Writer",
+    "detect_format",
+    "make_writer",
+    "open_reader",
+]

ciphergate/readers/accesslog_reader.py

@@ -0,0 +1,244 @@
+"""Access-log adapter: NCSA common/combined lines as written by nginx and Apache httpd.
+
+``192.0.2.10 - alice [05/Jan/2026:10:00:00 +0000] "GET /p?email=a%40b HTTP/1.1" 200 12 "ref" "ua"``
+
+Detection never relies on the parse alone. Every line is scanned exactly like ``text``; the
+parse only adds field context on top:
+
+* fields get nginx variable names as field paths: ``/remote_addr``, ``/ident``,
+  ``/remote_user``, ``/time_local``, ``/request`` (split into ``/request/method``,
+  ``/request/path``, ``/request/protocol`` when it has the usual three parts), ``/status``,
+  ``/body_bytes_sent``, ``/http_referer`` and ``/http_user_agent``;
+* query strings of the request target and the referer become fields too
+  (``/request/query/<name>``), so field-name heuristics and ``field_names`` rules apply to
+  parameters such as ``email`` or ``access_token``;
+* log escapes (``\\x22``, ``\\"``) and percent-encoding are decoded and the decoded value is
+  scanned as well, because the raw-text scan cannot see through them.
+
+A ``-`` field means "no value" and gets no field context. Output keeps the line as it was; only
+protected spans change. Replacements are re-encoded for their field (percent-encoding in paths
+and query strings, ``\\xHH`` log escapes elsewhere), and the line must still parse with as many
+fields afterwards, or the run fails. Only lines that *start* with the common/combined layout are
+recognised; anything after the user agent (e.g. ``$http_x_forwarded_for``) and lines in other
+layouts get text-mode scanning only. ``drop_field`` is not supported (it becomes ``review``).
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from ciphergate.constants import InputFormat
+from ciphergate.readers.json_reader import pointer_escape
+from ciphergate.readers.line_structure import FieldRegion, FieldToken, LineStructure
+from ciphergate.readers.text_reader import TextReader
+
+# Three space-free fields, then the bracketed time. Classes never overlap the separators, so
+# matching is linear.
+_HEAD_RE = re.compile(r'([^\s"]{1,1024}) ([^\s"]{1,1024}) ([^\s"]{1,1024}) \[([^\]\n]{1,128})\] ')
+_SHORT_RE = re.compile(r' ([^\s"]{1,64}) ([^\s"]{1,64})')
+_REQUEST_RE = re.compile(r"([^ ]{1,64}) ([^ ]{1,8192}) ([^ ]{1,64})")
+_MAX_QUOTED = 16384
+_LOG_ESCAPES = {'"': '"', "\\": "\\", "n": "\n", "t": "\t", "r": "\r"}
+# Characters kept literally when percent-encoding a replacement (markers stay readable).
+_URL_SAFE = frozenset(
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~!$'()*,:@/[]"
+)
+_HEX = "0123456789abcdefABCDEF"
+
+
+def _quoted(line: str, i: int) -> int | None:
+    """End offset of the quoted field whose opening quote is at ``i`` (after the closing quote)."""
+    if i >= len(line) or line[i] != '"':
+        return None
+    j, limit = i + 1, min(len(line), i + 1 + _MAX_QUOTED)
+    while j < limit:
+        ch = line[j]
+        if ch == '"':
+            return j + 1
+        j += 2 if ch == "\\" else 1
+    return None
+
+
+def _log_unescape(raw: str) -> str:
+    """Undo nginx (``\\xHH``) and Apache (``\\"``, ``\\\\``, ``\\n``) log escapes."""
+    if "\\" not in raw:
+        return raw
+    out: list[str] = []
+    i, n = 0, len(raw)
+    while i < n:
+        ch = raw[i]
+        if ch == "\\" and i + 1 < n:
+            nxt = raw[i + 1]
+            if nxt == "x" and i + 3 < n and raw[i + 2] in _HEX and raw[i + 3] in _HEX:
+                out.append(chr(int(raw[i + 2 : i + 4], 16)))
+                i += 4
+                continue
+            if nxt in _LOG_ESCAPES:
+                out.append(_LOG_ESCAPES[nxt])
+                i += 2
+                continue
+        out.append(ch)
+        i += 1
+    return "".join(out)
+
+
+def _percent_decode(text: str, *, plus: bool) -> str:
+    """Percent-decoding (``+`` is a space in query strings); invalid UTF-8 becomes U+FFFD."""
+    if "%" not in text and not (plus and "+" in text):
+        return text
+    data = bytearray()
+    i, n = 0, len(text)
+    while i < n:
+        ch = text[i]
+        if ch == "%" and i + 2 < n and text[i + 1] in _HEX and text[i + 2] in _HEX:
+            data.append(int(text[i + 1 : i + 3], 16))
+            i += 3
+            continue
+        data.extend(b" " if plus and ch == "+" else ch.encode("utf-8"))
+        i += 1
+    return data.decode("utf-8", errors="replace")
+
+
+def _kind(pointer: str) -> str:
+    """How a field is encoded in the line, from its path."""
+    if "/query/" in pointer or pointer == "/request/path":
+        return "url"
+    if pointer == "/http_referer":
+        return "referer"
+    if pointer in ("/request/method", "/request/protocol"):
+        return "request_part"
+    if pointer in ("/request", "/http_user_agent"):
+        return "quoted"
+    if pointer == "/time_local":
+        return "time"
+    return "bare"
+
+
+def _decode(pointer: str, raw: str) -> str:
+    text = _log_unescape(raw)
+    if _kind(pointer) == "url":
+        return _percent_decode(text, plus="/query/" in pointer)
+    return text
+
+
+def _hex_escape(ch: str) -> str:
+    return "".join(f"\\x{b:02X}" for b in ch.encode("utf-8"))
+
+
+def _escape(owner: FieldToken, text: str) -> str:
+    kind = _kind(owner.pointer)
+    if kind == "url":
+        return "".join(
+            ch if ch in _URL_SAFE else "".join(f"%{b:02X}" for b in ch.encode("utf-8"))
+            for ch in text
+        )
+    special = {
+        "quoted": '"\\',
+        "referer": '"\\',
+        "request_part": '"\\ ',
+        "time": '"\\]',
+        "bare": '"\\',
+    }[kind]
+    out: list[str] = []
+    for ch in text:
+        if ch in special or ord(ch) < 0x20 or ord(ch) == 0x7F or (kind == "bare" and ch.isspace()):
+            out.append(_hex_escape(ch))
+        elif kind == "referer" and ch in "?#":
+            out.append(f"%{ord(ch):02X}")
+        else:
+            out.append(ch)
+    return "".join(out)
+
+
+def _unescape(owner: FieldToken, raw: str) -> Any:
+    return _decode(owner.pointer, raw)
+
+
+def _field(
+    tokens: list[FieldToken], line: str, pointer: str, key: str, start: int, end: int
+) -> None:
+    raw = line[start:end]
+    if raw in ("", "-"):
+        return  # "-" is the log's "no value"
+    value = _decode(pointer, raw)
+    tokens.append(FieldToken("string", start, end, pointer, key, value, value != raw))
+
+
+def _query(tokens: list[FieldToken], base: str, line: str, start: int, end: int) -> None:
+    """``name=value&...`` between ``start`` and ``end`` as key and value tokens."""
+    i = start
+    while i <= end:
+        amp = line.find("&", i, end)
+        stop = end if amp < 0 else amp
+        if stop > i:
+            eq = line.find("=", i, stop)
+            key_end = stop if eq < 0 else eq
+            name_raw = line[i:key_end]
+            name = _percent_decode(_log_unescape(name_raw), plus=True)
+            pointer = f"{base}/query/{pointer_escape(name)}"
+            tokens.append(FieldToken("key", i, key_end, pointer, name, name, name != name_raw))
+            if eq >= 0:
+                raw = line[eq + 1 : stop]
+                value = _decode(pointer, raw)
+                tokens.append(
+                    FieldToken("string", eq + 1, stop, pointer, name, value, value != raw)
+                )
+        i = stop + 1
+
+
+def _target(tokens: list[FieldToken], line: str, start: int, end: int) -> None:
+    q = line.find("?", start, end)
+    _field(tokens, line, "/request/path", "path", start, end if q < 0 else q)
+    if q >= 0:
+        _query(tokens, "/request", line, q + 1, end)
+
+
+def parse_regions(line: str, max_depth: int = 64) -> list[FieldRegion]:
+    """The access-log fields of ``line`` as one region, or ``[]`` if it is not in that layout."""
+    head = _HEAD_RE.match(line)
+    if head is None:
+        return []
+    request_end = _quoted(line, head.end())
+    if request_end is None:
+        return []
+    short = _SHORT_RE.match(line, request_end)
+    if short is None:
+        return []
+    tokens: list[FieldToken] = []
+    for group, name in enumerate(("remote_addr", "ident", "remote_user", "time_local"), 1):
+        _field(tokens, line, "/" + name, name, head.start(group), head.end(group))
+    req_start, req_end = head.end() + 1, request_end - 1
+    request = _REQUEST_RE.fullmatch(line, req_start, req_end)
+    if request is None:
+        _field(tokens, line, "/request", "request", req_start, req_end)
+    else:
+        _field(tokens, line, "/request/method", "method", request.start(1), request.end(1))
+        _target(tokens, line, request.start(2), request.end(2))
+        _field(tokens, line, "/request/protocol", "protocol", request.start(3), request.end(3))
+    _field(tokens, line, "/status", "status", short.start(1), short.end(1))
+    _field(tokens, line, "/body_bytes_sent", "body_bytes_sent", short.start(2), short.end(2))
+    end = short.end()
+    referer_end = _quoted(line, end + 1) if line.startswith(' "', end) else None
+    if referer_end is not None:
+        agent_end = _quoted(line, referer_end + 1) if line.startswith(' "', referer_end) else None
+        if agent_end is not None:
+            ref_start, ref_stop = end + 2, referer_end - 1
+            q = line.find("?", ref_start, ref_stop)
+            _field(
+                tokens, line, "/http_referer", "http_referer", ref_start, ref_stop if q < 0 else q
+            )
+            if q >= 0:
+                _query(tokens, "/http_referer", line, q + 1, ref_stop)
+            _field(
+                tokens, line, "/http_user_agent", "http_user_agent", referer_end + 2, agent_end - 1
+            )
+            end = agent_end
+    return [FieldRegion(0, end, tuple(tokens))]
+
+
+ACCESSLOG_STRUCTURE = LineStructure(parse=parse_regions, unescape=_unescape, escape=_escape)
+
+
+class AccessLogReader(TextReader):
+    format = InputFormat.ACCESSLOG

ciphergate/readers/base.py

@@ -0,0 +1,131 @@
+"""Format adapter contracts shared by all readers/writers."""
+
+from __future__ import annotations
+
+import codecs
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import IO, Any, Protocol
+
+from ciphergate.config import RuntimeSettings
+from ciphergate.constants import InputFormat
+from ciphergate.errors import InputFormatError, InputLimitError, UsageError
+from ciphergate.models.finding import ScanItem
+from ciphergate.models.policy import PolicySettings
+
+# Readers split lines on bytes, so only ASCII-compatible encodings are supported.
+SUPPORTED_ENCODINGS = frozenset({"utf-8", "utf-8-sig", "ascii", "latin-1", "cp1252"})
+
+
+def normalize_encoding(name: str) -> str:
+    try:
+        canonical = codecs.lookup(name).name
+    except LookupError:
+        raise UsageError("unknown encoding", code="unsupported_encoding") from None
+    aliases = {
+        "utf-8": "utf-8",
+        "utf-8-sig": "utf-8-sig",
+        "ascii": "ascii",
+        "iso8859-1": "latin-1",
+        "latin-1": "latin-1",
+        "cp1252": "cp1252",
+    }
+    if canonical not in aliases:
+        raise UsageError(
+            "unsupported encoding; supported: " + ", ".join(sorted(SUPPORTED_ENCODINGS)),
+            code="unsupported_encoding",
+        )
+    return aliases[canonical]
+
+
+@dataclass(slots=True)
+class Record:
+    index: int
+    line: int | None
+    data: Any
+    line_ending: str = ""
+    malformed: bool = False
+    error_code: str | None = None
+
+
+@dataclass(slots=True)
+class ReadStats:
+    bytes_read: int = 0
+    records: int = 0
+    blank_lines: int = 0
+    parse_errors: list[tuple[int | None, str]] = field(default_factory=list)
+
+
+class Reader(Protocol):
+    format: InputFormat
+    stats: ReadStats
+
+    def records(self) -> Iterator[Record]: ...
+
+    def iter_items(self, record: Record) -> Iterator[ScanItem]: ...
+
+
+class Writer(Protocol):
+    def write_record(self, record: Record, data: Any) -> None: ...
+
+    def close(self) -> None: ...
+
+
+@dataclass(frozen=True, slots=True)
+class ReaderOptions:
+    runtime: RuntimeSettings
+    policy_settings: PolicySettings
+    encoding: str = "utf-8"
+
+
+def read_bounded_lines(
+    stream: IO[bytes], options: ReaderOptions, stats: ReadStats
+) -> Iterator[tuple[int, bytes]]:
+    """Yield (line_number, raw line bytes incl. terminator) enforcing per-line and total limits."""
+    max_line = options.runtime.max_record_bytes
+    line_no = 0
+    while True:
+        raw = stream.readline(max_line + 2)
+        if not raw:
+            return
+        line_no += 1
+        stats.bytes_read += len(raw)
+        if stats.bytes_read > options.runtime.max_input_bytes:
+            raise InputLimitError("input exceeds max_input_bytes", code="size_limit_exceeded")
+        content_len = len(raw.rstrip(b"\r\n"))
+        if content_len > max_line:
+            raise InputLimitError(
+                f"line {line_no} exceeds max_record_bytes ({max_line})", code="record_too_large"
+            )
+        yield line_no, raw
+
+
+def split_line_ending(text: str) -> tuple[str, str]:
+    if text.endswith("\r\n"):
+        return text[:-2], "\r\n"
+    if text.endswith("\n") or text.endswith("\r"):
+        return text[:-1], text[-1]
+    return text, ""
+
+
+def decode_line(raw: bytes, encoding: str, line_no: int, *, first: bool) -> str:
+    if first and raw.startswith(codecs.BOM_UTF8) and encoding.startswith("utf-8"):
+        raw = raw[len(codecs.BOM_UTF8) :]
+    try:
+        return raw.decode(encoding.replace("-sig", ""))
+    except UnicodeDecodeError:
+        raise InputFormatError(
+            f"line {line_no} is not valid {encoding}", code="decode_error"
+        ) from None
+
+
+def has_utf8_bom(path: Path) -> bool:
+    with path.open("rb") as fh:
+        return fh.read(3) == codecs.BOM_UTF8
+
+
+def detect_newline(path: Path) -> str:
+    with path.open("rb") as fh:
+        chunk = fh.read(64 * 1024)
+    return "\r\n" if b"\r\n" in chunk else "\n"