ciphergate-cli 0.7.0__py3-none-any.whl

ciphergate/readers/csv_reader.py

@@ -0,0 +1,241 @@
+"""CSV adapter built on the standard ``csv`` module (never naive comma splitting).
+
+* Header row is required. Header cells are scanned like JSON keys (``header_items``, record index
+  ``-1``): a label can itself contain data. Duplicate headers are rejected by default; with
+  ``csv_duplicate_headers: disambiguate`` they are labelled ``name#2`` internally while the
+  original header text is written back unchanged.
+* Empty header cells are labelled ``#<column number>`` and are not used for field-name heuristics.
+* Rows with a different width than the header are parse errors (fail, or drop when configured).
+* BOM and CRLF/LF line endings are preserved on output. All values stay text (CSV is untyped).
+"""
+
+from __future__ import annotations
+
+import csv
+from collections.abc import Generator, Iterator
+from contextlib import contextmanager
+from pathlib import Path
+from typing import IO, Any
+
+from ciphergate.constants import InputFormat
+from ciphergate.errors import InputFormatError, InputLimitError
+from ciphergate.models.finding import ScanItem
+from ciphergate.readers.base import (
+    ReaderOptions,
+    ReadStats,
+    Record,
+    detect_newline,
+    has_utf8_bom,
+)
+
+
+@contextmanager
+def _field_limit(limit: int) -> Iterator[None]:
+    previous = csv.field_size_limit()
+    csv.field_size_limit(limit)
+    try:
+        yield
+    finally:
+        csv.field_size_limit(previous)
+
+
+class _CountingText:
+    """Wraps a text stream, counting encoded bytes so the total input limit is enforced."""
+
+    def __init__(self, inner: IO[str], encoding: str, limit: int, stats: ReadStats) -> None:
+        self._inner = inner
+        self._encoding = encoding
+        self._limit = limit
+        self._stats = stats
+
+    def readline(self, size: int = -1) -> str:
+        line = self._inner.readline(size)
+        self._stats.bytes_read += len(line.encode(self._encoding, errors="replace"))
+        if self._stats.bytes_read > self._limit:
+            raise InputLimitError("input exceeds max_input_bytes", code="size_limit_exceeded")
+        return line
+
+    def __iter__(self) -> Iterator[str]:
+        return self
+
+    def __next__(self) -> str:
+        line = self.readline()
+        if not line:
+            raise StopIteration
+        return line
+
+
+class CSVReader:
+    format = InputFormat.CSV
+
+    def __init__(self, path: Path, options: ReaderOptions) -> None:
+        self.path = path
+        self.options = options
+        self.stats = ReadStats()
+        self.header: list[str] | None = None
+        self.labels: list[str] = []
+        self.field_names: list[str | None] = []
+        self.has_bom = options.encoding.startswith("utf-8") and has_utf8_bom(path)
+        self.newline = detect_newline(path)
+        self.delimiter = options.runtime.csv_delimiter
+
+    def _build_labels(self, header: list[str]) -> None:
+        labels: list[str] = []
+        names: list[str | None] = []
+        counts: dict[str, int] = {}
+        for i, name in enumerate(header):
+            if name == "":
+                labels.append(f"#{i + 1}")
+                names.append(None)
+                continue
+            counts[name] = counts.get(name, 0) + 1
+            if counts[name] > 1:
+                if self.options.policy_settings.csv_duplicate_headers == "reject":
+                    raise InputFormatError(
+                        f"duplicate CSV header in column {i + 1}", code="csv_duplicate_header"
+                    )
+                labels.append(f"{name}#{counts[name]}")
+            else:
+                labels.append(name)
+            names.append(name)
+        self.labels, self.field_names = labels, names
+
+    def header_items(self) -> Iterator[ScanItem]:
+        """Header cells as key items. Reads only up to the first data row to learn the header."""
+        rows = self.records()
+        try:
+            next(rows, None)
+        finally:
+            rows.close()
+        for col, name in enumerate(self.header or []):
+            if name:
+                yield ScanItem(
+                    record_index=-1,
+                    line=1,
+                    field_path=self.labels[col],
+                    field_name=None,
+                    value=name,
+                    column_index=col,
+                    in_key=True,
+                )
+
+    def records(self) -> Generator[Record, None, None]:
+        self.header = None
+        self.stats = ReadStats()
+        encoding = (
+            "utf-8-sig" if self.options.encoding.startswith("utf-8") else self.options.encoding
+        )
+        runtime = self.options.runtime
+        try:
+            with (
+                _field_limit(runtime.max_record_bytes),
+                self.path.open("r", encoding=encoding, newline="", errors="strict") as fh,
+            ):
+                counted = _CountingText(fh, encoding, runtime.max_input_bytes, self.stats)
+                reader = csv.reader(counted, delimiter=self.delimiter, strict=True)
+                index = 0
+                for row in reader:
+                    line = reader.line_num
+                    if self.header is None:
+                        self.header = row
+                        self._build_labels(row)
+                        continue
+                    if not row:
+                        self.stats.blank_lines += 1
+                        if self.options.policy_settings.preserve_blank_lines:
+                            yield Record(index=-1, line=line, data=[])
+                        continue
+                    if len(row) != len(self.header):
+                        self.stats.parse_errors.append((line, "csv_row_width"))
+                        if self.options.policy_settings.fail_on_parse_error:
+                            raise InputFormatError(
+                                f"CSV row ending at line {line} has {len(row)} fields, header has "
+                                f"{len(self.header)}",
+                                code="csv_row_width",
+                            )
+                        yield Record(
+                            index=index,
+                            line=line,
+                            data=row,
+                            malformed=True,
+                            error_code="csv_row_width",
+                        )
+                    else:
+                        yield Record(index=index, line=line, data=row)
+                    index += 1
+                    self.stats.records = index
+        except UnicodeDecodeError:
+            raise InputFormatError(
+                f"input is not valid {self.options.encoding}", code="decode_error"
+            ) from None
+        except csv.Error as exc:
+            line = None
+            if "field larger than field limit" in str(exc):
+                raise InputLimitError(
+                    "a CSV field exceeds max_record_bytes", code="record_too_large"
+                ) from None
+            raise InputFormatError(
+                f"malformed CSV near line {line or 'unknown'}", code="csv_syntax"
+            ) from None
+
+    def iter_items(self, record: Record) -> Iterator[ScanItem]:
+        if record.malformed or record.index < 0:
+            return
+        for col, value in enumerate(record.data):
+            yield ScanItem(
+                record_index=record.index,
+                line=record.line,
+                field_path=self.labels[col],
+                field_name=self.field_names[col],
+                value=value,
+                column_index=col,
+            )
+
+
+class CSVWriter:
+    """Writes the header lazily: the reader only knows it once pass two has started reading."""
+
+    def __init__(self, stream: IO[str], reader: CSVReader, dropped_columns: set[int]) -> None:
+        self._stream = stream
+        self._reader = reader
+        self._dropped = dropped_columns
+        self._keep: list[int] | None = None
+        self._writer = csv.writer(
+            stream,
+            delimiter=reader.delimiter,
+            lineterminator=reader.newline,
+            quoting=csv.QUOTE_MINIMAL,
+        )
+        # QUOTE_MINIMAL only quotes characters of ``lineterminator``; a lone CR/LF inside a field
+        # would otherwise be written unquoted and corrupt the value.
+        self._quote_all = csv.writer(
+            stream,
+            delimiter=reader.delimiter,
+            lineterminator=reader.newline,
+            quoting=csv.QUOTE_ALL,
+        )
+
+    def _writerow(self, row: list[Any]) -> None:
+        needs_quotes = any(isinstance(v, str) and ("\r" in v or "\n" in v) for v in row)
+        (self._quote_all if needs_quotes else self._writer).writerow(row)
+
+    def _ensure_header(self) -> list[int]:
+        if self._keep is None:
+            header = self._reader.header
+            self._keep = [i for i in range(len(header or [])) if i not in self._dropped]
+            if self._reader.has_bom:
+                self._stream.write("")
+            if header is not None:
+                self._writerow([header[i] for i in self._keep])
+        return self._keep
+
+    def write_record(self, record: Record, data: Any) -> None:
+        keep = self._ensure_header()
+        if record.index < 0:
+            self._writerow([])
+            return
+        self._writerow([data[i] for i in keep])
+
+    def close(self) -> None:
+        self._ensure_header()
+        self._stream.flush()

ciphergate/readers/json_reader.py

@@ -0,0 +1,184 @@
+"""JSON adapter (whole-document parse, bounded by ``max_json_document_bytes``).
+
+* A top-level array is treated as a sequence of records (``/0``, ``/1``, ...); any other top-level
+  value is a single record.
+* Paths are RFC 6901 JSON Pointers (``~`` -> ``~0``, ``/`` -> ``~1``).
+* Nesting depth is checked by a string-aware pre-scan before parsing.
+* Duplicate object keys and NaN/Infinity are rejected (strict JSON).
+* Parser error messages are replaced with position-only messages (never source excerpts).
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterator
+from pathlib import Path
+from typing import IO, Any
+
+from ciphergate.constants import InputFormat
+from ciphergate.errors import InputFormatError, InputLimitError
+from ciphergate.models.finding import ScanItem
+from ciphergate.readers.base import ReaderOptions, ReadStats, Record, detect_newline
+
+
+class _DuplicateKey(Exception):
+    pass
+
+
+class _BadConstant(Exception):
+    pass
+
+
+def pointer_escape(segment: str) -> str:
+    return segment.replace("~", "~0").replace("/", "~1")
+
+
+def check_depth(text: str, max_depth: int) -> None:
+    depth = 0
+    in_string = False
+    escape = False
+    for ch in text:
+        if in_string:
+            if escape:
+                escape = False
+            elif ch == "\\":
+                escape = True
+            elif ch == '"':
+                in_string = False
+            continue
+        if ch == '"':
+            in_string = True
+        elif ch in "[{":
+            depth += 1
+            if depth > max_depth:
+                raise InputLimitError(
+                    f"JSON nesting exceeds max depth {max_depth}", code="json_too_deep"
+                )
+        elif ch in "]}":
+            depth -= 1
+
+
+def _pairs_hook(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
+    out: dict[str, Any] = {}
+    for key, value in pairs:
+        if key in out:
+            raise _DuplicateKey
+        out[key] = value
+    return out
+
+
+def _reject_constant(_: str) -> Any:
+    raise _BadConstant
+
+
+def parse_json_text(text: str, max_depth: int, *, where: str = "document") -> Any:
+    check_depth(text, max_depth)
+    try:
+        return json.loads(text, object_pairs_hook=_pairs_hook, parse_constant=_reject_constant)
+    except _DuplicateKey:
+        raise InputFormatError(
+            f"duplicate object key in JSON {where}", code="json_duplicate_key"
+        ) from None
+    except _BadConstant:
+        raise InputFormatError(
+            f"NaN/Infinity are not valid JSON ({where})", code="json_invalid_constant"
+        ) from None
+    except json.JSONDecodeError as exc:
+        raise InputFormatError(
+            f"invalid JSON {where} at line {exc.lineno}, column {exc.colno}", code="json_syntax"
+        ) from None
+    except RecursionError:
+        raise InputLimitError("JSON nesting too deep", code="json_too_deep") from None
+
+
+def iter_json_items(
+    node: Any,
+    pointer: str,
+    field_name: str | None,
+    record_index: int,
+    line: int | None,
+    scan_keys: bool,
+) -> Iterator[ScanItem]:
+    if isinstance(node, dict):
+        for key, value in node.items():
+            child = f"{pointer}/{pointer_escape(key)}"
+            if scan_keys:
+                yield ScanItem(record_index, line, child, None, key, in_key=True)
+            yield from iter_json_items(value, child, key, record_index, line, scan_keys)
+    elif isinstance(node, list):
+        for i, value in enumerate(node):
+            yield from iter_json_items(
+                value, f"{pointer}/{i}", field_name, record_index, line, scan_keys
+            )
+    else:
+        yield ScanItem(record_index, line, pointer, field_name, node)
+
+
+class JSONReader:
+    format = InputFormat.JSON
+
+    def __init__(self, path: Path, options: ReaderOptions) -> None:
+        self.path = path
+        self.options = options
+        self.stats = ReadStats()
+        self.top_level_array = False
+        self.newline = detect_newline(path)
+
+    def records(self) -> Iterator[Record]:
+        runtime = self.options.runtime
+        limit = min(runtime.max_json_document_bytes, runtime.max_input_bytes)
+        with self.path.open("rb") as fh:
+            data = fh.read(limit + 1)
+        if len(data) > limit:
+            raise InputLimitError(
+                f"JSON document exceeds {limit} bytes", code="size_limit_exceeded"
+            )
+        self.stats.bytes_read = len(data)
+        encoding = self.options.encoding
+        try:
+            text = data.decode("utf-8-sig" if encoding.startswith("utf-8") else encoding)
+        except UnicodeDecodeError:
+            raise InputFormatError(f"input is not valid {encoding}", code="decode_error") from None
+        doc = parse_json_text(text, runtime.max_json_depth)
+        if isinstance(doc, list):
+            self.top_level_array = True
+            for i, element in enumerate(doc):
+                self.stats.records = i + 1
+                yield Record(index=i, line=None, data=element)
+        else:
+            self.stats.records = 1
+            yield Record(index=0, line=None, data=doc)
+
+    def iter_items(self, record: Record) -> Iterator[ScanItem]:
+        prefix = f"/{record.index}" if self.top_level_array else ""
+        yield from iter_json_items(
+            record.data,
+            prefix,
+            None,
+            record.index,
+            None,
+            self.options.policy_settings.scan_json_keys,
+        )
+
+
+class JSONWriter:
+    """Buffers records and writes one UTF-8 JSON document (indent 2) on close."""
+
+    def __init__(self, stream: IO[str], reader: JSONReader) -> None:
+        self._stream = stream
+        self._reader = reader
+        self._newline = reader.newline
+        self._items: list[Any] = []
+
+    def write_record(self, record: Record, data: Any) -> None:
+        self._items.append(data)
+
+    def close(self) -> None:
+        doc: Any = (
+            self._items
+            if self._reader.top_level_array
+            else (self._items[0] if self._items else None)
+        )
+        text = json.dumps(doc, ensure_ascii=False, indent=2, allow_nan=False)
+        self._stream.write(text.replace("\n", self._newline) + self._newline)
+        self._stream.flush()

ciphergate/readers/jsonl_reader.py

@@ -0,0 +1,97 @@
+"""JSONL / NDJSON adapter: one JSON value per non-empty line, streamed with bounded memory.
+
+Blank lines are dropped from output unless ``preserve_blank_lines`` is set. Malformed lines fail the
+run by default; with ``fail_on_parse_error: false`` + ``malformed_record_action: drop_record`` they
+are dropped and counted. A malformed line is never copied unchanged into the output.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterator
+from pathlib import Path
+from typing import IO, Any
+
+from ciphergate.constants import InputFormat
+from ciphergate.errors import InputFormatError
+from ciphergate.models.finding import ScanItem
+from ciphergate.readers.base import (
+    ReaderOptions,
+    ReadStats,
+    Record,
+    decode_line,
+    read_bounded_lines,
+    split_line_ending,
+)
+from ciphergate.readers.json_reader import iter_json_items, parse_json_text
+
+
+class JSONLReader:
+    format = InputFormat.JSONL
+
+    def __init__(self, path: Path, options: ReaderOptions) -> None:
+        self.path = path
+        self.options = options
+        self.stats = ReadStats()
+
+    def records(self) -> Iterator[Record]:
+        settings = self.options.policy_settings
+        index = 0
+        with self.path.open("rb") as fh:
+            for line_no, raw in read_bounded_lines(fh, self.options, self.stats):
+                text = decode_line(raw, self.options.encoding, line_no, first=line_no == 1)
+                content, ending = split_line_ending(text)
+                if not content.strip():
+                    self.stats.blank_lines += 1
+                    if settings.preserve_blank_lines:
+                        yield Record(index=-1, line=line_no, data=content, line_ending=ending)
+                    continue
+                try:
+                    data = parse_json_text(
+                        content, self.options.runtime.max_json_depth, where=f"line {line_no}"
+                    )
+                except InputFormatError as exc:
+                    self.stats.parse_errors.append((line_no, exc.code))
+                    if settings.fail_on_parse_error:
+                        raise InputFormatError(
+                            f"invalid JSON on line {line_no}", code=exc.code
+                        ) from None
+                    yield Record(
+                        index=index,
+                        line=line_no,
+                        data=None,
+                        line_ending=ending,
+                        malformed=True,
+                        error_code=exc.code,
+                    )
+                else:
+                    yield Record(index=index, line=line_no, data=data, line_ending=ending)
+                index += 1
+                self.stats.records = index
+
+    def iter_items(self, record: Record) -> Iterator[ScanItem]:
+        if record.malformed or record.index < 0:
+            return
+        yield from iter_json_items(
+            record.data,
+            "",
+            None,
+            record.index,
+            record.line,
+            self.options.policy_settings.scan_json_keys,
+        )
+
+
+class JSONLWriter:
+    def __init__(self, stream: IO[str]) -> None:
+        self._stream = stream
+
+    def write_record(self, record: Record, data: Any) -> None:
+        ending = record.line_ending or "\n"
+        if record.index < 0:
+            self._stream.write(ending)
+            return
+        self._stream.write(json.dumps(data, ensure_ascii=False, allow_nan=False) + ending)
+
+    def close(self) -> None:
+        self._stream.flush()

ciphergate/readers/jsonlog_reader.py

@@ -0,0 +1,178 @@
+"""JSON-in-log adapter: text/log lines that embed JSON objects (``ERROR {"user": ...}``).
+
+Detection never relies on the parse alone. Every line is scanned exactly like ``text``; the
+parse only adds field context on top, for each JSON object found in the line:
+
+* findings inside a value are attributed to it (``field_path`` = JSON Pointer inside the object,
+  ``field_name`` = its key), so policy ``field_names``/``field_paths`` rules apply;
+* field-name heuristics and policy classifications run on every key and cover the whole value;
+* string values that contain JSON escapes (``\\u0040``) are also decoded and scanned, because the
+  raw-text scan cannot see through escapes; a match there covers the whole value.
+
+Output keeps the line as it was; only protected spans change. Replacements inside a JSON string
+are JSON-escaped, and a replaced number/boolean/null becomes a JSON string, so every embedded
+object stays valid JSON; this is re-checked after rewriting and a violation fails the run.
+
+Only objects (``{...}``) are recognised, at most ``MAX_CANDIDATES`` start positions per line,
+nested at most ``max_depth`` levels. Text that is not valid JSON gets no field context (it is
+still scanned as text). ``drop_field`` is not supported (it becomes ``review``).
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+from ciphergate.constants import InputFormat
+from ciphergate.readers.json_reader import pointer_escape
+from ciphergate.readers.line_structure import FieldRegion, FieldToken, LineStructure
+from ciphergate.readers.text_reader import TextReader
+
+MAX_CANDIDATES = 16
+_WS = " \t\r\n"
+_NUMBER_RE = re.compile(r"-?(?:0|[1-9][0-9]{0,400})(?:\.[0-9]{1,400})?(?:[eE][+-]?[0-9]{1,5})?")
+_LITERALS = (("true", True), ("false", False), ("null", None))
+
+
+class _Invalid(Exception):
+    pass
+
+
+class _Parser:
+    def __init__(self, line: str, max_depth: int) -> None:
+        self.s = line
+        self.n = len(line)
+        self.max_depth = max_depth
+        self.tokens: list[FieldToken] = []
+
+    def ws(self, i: int) -> int:
+        while i < self.n and self.s[i] in _WS:
+            i += 1
+        return i
+
+    def string(self, i: int) -> tuple[str, int, bool]:
+        """Parse a JSON string at ``i`` (the opening quote); returns (value, end, escaped)."""
+        j = i + 1
+        escaped = False
+        while j < self.n:
+            ch = self.s[j]
+            if ch == '"':
+                raw = self.s[i : j + 1]
+                try:
+                    value = json.loads(raw)
+                except ValueError:
+                    raise _Invalid from None
+                return value, j + 1, escaped
+            if ch == "\\":
+                escaped = True
+                j += 2
+                continue
+            if ord(ch) < 0x20:
+                raise _Invalid
+            j += 1
+        raise _Invalid
+
+    def value(self, i: int, pointer: str, key: str | None, depth: int) -> int:
+        if i >= self.n:
+            raise _Invalid
+        ch = self.s[i]
+        if ch == "{":
+            return self.obj(i, pointer, depth + 1)
+        if ch == "[":
+            return self.array(i, pointer, key, depth + 1)
+        if ch == '"':
+            value, end, escaped = self.string(i)
+            self.tokens.append(FieldToken("string", i + 1, end - 1, pointer, key, value, escaped))
+            return end
+        for text, literal in _LITERALS:
+            if self.s.startswith(text, i):
+                self.tokens.append(FieldToken("literal", i, i + len(text), pointer, key, literal))
+                return i + len(text)
+        m = _NUMBER_RE.match(self.s, i)
+        if not m:
+            raise _Invalid
+        number = json.loads(m.group())
+        self.tokens.append(FieldToken("literal", i, m.end(), pointer, key, number))
+        return m.end()
+
+    def obj(self, i: int, pointer: str, depth: int) -> int:
+        if depth > self.max_depth:
+            raise _Invalid
+        i = self.ws(i + 1)
+        if i < self.n and self.s[i] == "}":
+            return i + 1
+        while True:
+            if i >= self.n or self.s[i] != '"':
+                raise _Invalid
+            key, key_end, escaped = self.string(i)
+            child = f"{pointer}/{pointer_escape(key)}"
+            self.tokens.append(FieldToken("key", i + 1, key_end - 1, child, key, key, escaped))
+            i = self.ws(key_end)
+            if i >= self.n or self.s[i] != ":":
+                raise _Invalid
+            i = self.value(self.ws(i + 1), child, key, depth)
+            i = self.ws(i)
+            if i < self.n and self.s[i] == ",":
+                i = self.ws(i + 1)
+                continue
+            if i < self.n and self.s[i] == "}":
+                return i + 1
+            raise _Invalid
+
+    def array(self, i: int, pointer: str, key: str | None, depth: int) -> int:
+        if depth > self.max_depth:
+            raise _Invalid
+        i = self.ws(i + 1)
+        if i < self.n and self.s[i] == "]":
+            return i + 1
+        index = 0
+        while True:
+            i = self.ws(self.value(i, f"{pointer}/{index}", key, depth))
+            index += 1
+            if i < self.n and self.s[i] == ",":
+                i = self.ws(i + 1)
+                continue
+            if i < self.n and self.s[i] == "]":
+                return i + 1
+            raise _Invalid
+
+
+def parse_regions(line: str, max_depth: int = 64) -> list[FieldRegion]:
+    """JSON objects embedded in ``line`` (non-overlapping, left to right)."""
+    regions: list[FieldRegion] = []
+    i = 0
+    attempts = 0
+    while attempts < MAX_CANDIDATES:
+        start = line.find("{", i)
+        if start < 0:
+            break
+        attempts += 1
+        parser = _Parser(line, max_depth)
+        try:
+            end = parser.obj(start, "", 1)
+        except (_Invalid, RecursionError):
+            i = start + 1
+            continue
+        regions.append(FieldRegion(start, end, tuple(parser.tokens)))
+        i = end
+    return regions
+
+
+def _unescape(owner: FieldToken, raw: str) -> Any:
+    try:
+        return json.loads(f'"{raw}"')
+    except ValueError:
+        return raw
+
+
+def _escape(owner: FieldToken, text: str) -> str:
+    quoted = json.dumps(text, ensure_ascii=False)
+    return quoted if owner.kind == "literal" else quoted[1:-1]
+
+
+JSONLOG_STRUCTURE = LineStructure(parse=parse_regions, unescape=_unescape, escape=_escape)
+
+
+class JsonLogReader(TextReader):
+    format = InputFormat.JSONLOG