mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/ux/author_yaml.py

@@ -0,0 +1,597 @@
+"""A bounded, fail-closed reader for the YAML subset a person writes a Recipe in.
+
+Nobody hand-maintains ten kilobytes of JSON twice. YAML is the authoring surface; JSON stays the
+contract. This module reads exactly enough YAML for that job and refuses everything else **by
+name**, because a reader that guesses at a Recipe is worse than one that stops: whatever comes out
+of here is what `mr-data approve` binds a fingerprint to.
+
+What it reads:
+
+* names with values under them, at any consistent indentation
+* lists, including lists of names-with-values and names with lists under them
+* plain values -- text, whole numbers, decimals, ``true``, ``false``, ``null`` and ``~``
+* text in double quotes (with ``\\"``, ``\\\\``, ``\\n`` and ``\\t``) or single quotes (with ``''``)
+* ``#`` notes, at the end of a line or on a line of their own
+* one optional ``---`` before the first line of content
+* ``[]`` and ``{}`` for a list or a group with nothing in it -- the only bracketed forms accepted
+
+Everything else raises :class:`AuthorError` naming the construct and the line it is on: anchors,
+aliases, merge keys, tags, bracketed lists and groups with contents in them, a second document,
+tabs used for indentation, block text markers, an indentation step that never opened, a repeated
+name, and anything the line scan does not positively recognise.
+
+Three properties this file is responsible for:
+
+* **No third-party loader exists here.** Nothing can call one by mistake or construct an object
+  from a tag, and this boundary adds no parser dependency.
+* **No expansion vector.** Anchors and aliases are refused outright, so the repeated-expansion
+  denial of service has nothing to work with. Depth and input size are capped besides.
+* **When in doubt, refuse.** Every accepted form is written down below. A line that matches none of
+  them is ``AUTHOR_YAML_UNSUPPORTED`` with its own text quoted back, never a best-effort guess.
+
+The value this returns is still only a value. :func:`mostlyright.data_harness.recipe
+.parse_frozen_recipe` is the second gate, and the plain-language summary `mr-data approve` prints
+is the third.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from mostlyright.data_harness.recipe import RecipeError
+from mostlyright.data_harness.ux.path_kind import UNKNOWN_KIND, name_of_kind, presence_at
+from mostlyright.data_harness.ux.plain_file import (
+    DANGLING,
+    MISSING,
+    NOT_PLAIN,
+    PlainFileRefusal,
+    read_plain_file,
+)
+
+# One megabyte of hand-written YAML is far more Recipe than anyone writes; above it the input is
+# refused before a single byte is decoded.
+MAX_INPUT_BYTES = 1 << 20
+
+# The checked-in example Recipe nests six deep. Thirty-two leaves room for anything a person means
+# and stops a file whose only purpose is to nest.
+MAX_DEPTH = 32
+
+_DOCUMENT_MARKER = "---"
+_DOCUMENT_END = "..."
+
+_ESCAPES = {'"': '"', "\\": "\\", "n": "\n", "t": "\t"}
+
+_INTEGER = re.compile(r"[-+]?[0-9]+")
+_DECIMAL = re.compile(
+    r"(?:[-+]?(?:[0-9]+\.[0-9]*|\.[0-9]+)(?:[eE][-+]?[0-9]+)?)|(?:[-+]?[0-9]+[eE][-+]?[0-9]+)"
+)
+
+# Spellings that are almost certainly meant as one of the three constants. Reading them as text
+# would be a silent surprise, so they are refused and the writer is told which spelling to use.
+# The lower-case words `on`, `off`, `yes`, `no` and `none` are ordinary text here and stay text --
+# the checked-in example Recipe uses `none` as a value and `on` as a name.
+_AMBIGUOUS = frozenset({"True", "TRUE", "False", "FALSE", "Null", "NULL", "None"})
+
+_SEQUENCE = "sequence"
+_MAPPING = "mapping"
+_VALUE = "value"
+
+_PLAIN_FORMS = (
+    "write text, a whole number, a decimal, true, false, or null, or put the text in quotes"
+)
+
+
+class AuthorError(RecipeError):
+    """One YAML line the authoring reader will not read, named and located.
+
+    It rides on :class:`RecipeError` on purpose: the command line already catches that class, so
+    an authoring refusal reaches a person through the same path a Recipe refusal does without a
+    second command-boundary translation.
+    """
+
+    def __init__(self, code: str, line: int, detail: str) -> None:
+        self.line = line
+        super().__init__(code, f"line {line}" if line else "recipe", detail)
+
+
+@dataclass(frozen=True)
+class _Token:
+    """One thing found on one line: a list dash, a name, or a value standing alone.
+
+    ``indent`` is the column the thing starts at, so ``- name: value`` yields a dash at the line's
+    own indentation and a name two columns further in. That is what lets one rule -- deeper column
+    means underneath -- build the whole tree.
+    """
+
+    kind: str
+    indent: int
+    line: int
+    key: str | None = None
+    text: str | None = None
+
+
+def read_author_source(path: Path) -> bytes:
+    """Read one authoring file, refusing an oversized one before any of it is read.
+
+    The rule itself -- open once, ask the descriptor what it is, read that descriptor under the
+    bound -- lives in :mod:`ux.plain_file`, because ``peek`` needs the identical rule and the
+    second copy of it is where this defect came back. What stays here is the wording: a Recipe is
+    written in a file, and the three sentences below say so in the words this command uses.
+    """
+
+    try:
+        return read_plain_file(path, max_bytes=MAX_INPUT_BYTES)
+    except PlainFileRefusal as refusal:
+        # Raised here rather than returned from a helper: the sweep that proves every typed code
+        # names its fix reads `raise` statements out of the live source, so a code that reaches a
+        # person only through a returned object is a code that sweep cannot see.
+        if refusal.reason == DANGLING:
+            # A dangling link is present even though its target is absent, so missing-path advice
+            # would misstate the observation. The noun comes from the mode already held by the
+            # rule, not a second filesystem lookup.
+            raise AuthorError(
+                "AUTHOR_YAML_DANGLING",
+                0,
+                f"there is {name_of_kind(refusal.mode) or UNKNOWN_KIND} at {refusal.path}, "
+                "and it leads "
+                "nowhere; repoint it at your Recipe, or remove it and write one there",
+            ) from None
+        if refusal.reason == MISSING:
+            raise AuthorError(
+                "AUTHOR_YAML_MISSING",
+                0,
+                f"there is {presence_at(path) or UNKNOWN_KIND} at {path}; check the spelling and "
+                "the folder you are in",
+            ) from None
+        if refusal.reason == NOT_PLAIN:
+            raise AuthorError(
+                "AUTHOR_YAML_TARGET",
+                0,
+                f"{path} is not a plain file; a Recipe is written in a file, not in a folder, a "
+                "pipe, or a device",
+            ) from None
+        raise AuthorError(
+            "AUTHOR_YAML_SIZE",
+            0,
+            f"{path} is larger than {MAX_INPUT_BYTES} bytes; write the Recipe in a smaller file",
+        ) from None
+
+
+def parse_author_yaml(raw: bytes | str) -> Any:
+    """Read one authoring file and return the plain value it describes.
+
+    Returns ``None`` for a file holding nothing but notes; naming what a Recipe is missing is
+    ``parse_frozen_recipe``'s job, not this one's.
+    """
+
+    tokens = _tokenise(_decode(raw))
+    if not tokens:
+        return None
+    first = tokens[0]
+    if first.indent != 0:
+        raise AuthorError(
+            "AUTHOR_YAML_INDENT",
+            first.line,
+            "the first line of content is indented; start it at the left margin",
+        )
+    if first.kind == _VALUE:
+        raise AuthorError(
+            "AUTHOR_YAML_UNSUPPORTED",
+            first.line,
+            f"a Recipe is a set of names with values under them; write name: value, not "
+            f"{_quote_back(first.text)}",
+        )
+    value, index = _build(tokens, 0, 0, 1)
+    if index < len(tokens):
+        raise AuthorError(
+            "AUTHOR_YAML_INDENT",
+            tokens[index].line,
+            "this line does not line up with anything above it; check its indentation",
+        )
+    return value
+
+
+# ------------------------------------------------------------------------------------------------
+# Bytes to lines
+# ------------------------------------------------------------------------------------------------
+
+
+def _decode(raw: bytes | str) -> str:
+    if isinstance(raw, str):
+        data = raw.encode("utf-8", errors="surrogatepass")
+    else:
+        data = raw
+    if len(data) > MAX_INPUT_BYTES:
+        raise AuthorError(
+            "AUTHOR_YAML_SIZE",
+            0,
+            f"the Recipe file is larger than {MAX_INPUT_BYTES} bytes; write a smaller one",
+        )
+    if isinstance(raw, str):
+        return raw
+    try:
+        return data.decode("utf-8")
+    except UnicodeDecodeError as exc:
+        raise AuthorError(
+            "AUTHOR_YAML_ENCODING",
+            0,
+            "the Recipe file is not plain UTF-8 text; save it as UTF-8 and try again",
+        ) from exc
+
+
+def _tokenise(text: str) -> list[_Token]:
+    """Turn every line of content into the things standing on it, left to right."""
+
+    tokens: list[_Token] = []
+    marker_seen = False
+    content_seen = False
+    for number, source in enumerate(text.split("\n"), start=1):
+        line = source[:-1] if source.endswith("\r") else source
+        body_start = len(line) - len(line.lstrip(" \t"))
+        if "\t" in line[:body_start]:
+            raise AuthorError(
+                "AUTHOR_YAML_TAB",
+                number,
+                "this line is indented with a tab; use spaces",
+            )
+        stripped = line[body_start:].rstrip()
+        if not stripped or stripped.startswith("#"):
+            continue
+        if stripped == _DOCUMENT_MARKER:
+            if content_seen or marker_seen:
+                raise AuthorError(
+                    "AUTHOR_YAML_MULTIDOC",
+                    number,
+                    "one Recipe per file; --- may appear once, before the first line of content",
+                )
+            marker_seen = True
+            continue
+        if stripped == _DOCUMENT_END or stripped.startswith(f"{_DOCUMENT_END} "):
+            raise AuthorError(
+                "AUTHOR_YAML_MULTIDOC",
+                number,
+                "one Recipe per file; write no ... end marker",
+            )
+        content_seen = True
+        _tokenise_line(tokens, line, body_start, number)
+    return tokens
+
+
+def _tokenise_line(tokens: list[_Token], line: str, indent: int, number: int) -> None:
+    column = indent
+    rest = line[indent:].rstrip()
+    while rest.startswith("-") and (len(rest) == 1 or rest[1] in " \t"):
+        if len(rest) > 1 and rest[1] == "\t":
+            raise AuthorError(
+                "AUTHOR_YAML_TAB",
+                number,
+                "there is a tab after this list dash; use spaces",
+            )
+        tokens.append(_Token(_SEQUENCE, column, number))
+        step = 1
+        while step < len(rest) and rest[step] == " ":
+            step += 1
+        column += step
+        rest = rest[step:]
+        if not rest:
+            return
+    if not rest or rest.startswith("#"):
+        return
+    if rest.startswith("\t"):
+        raise AuthorError("AUTHOR_YAML_TAB", number, "there is a tab here; use spaces")
+    if rest == "?" or rest.startswith("? "):
+        # The marker for a name written on its own line, with its value on the next. Caught here
+        # rather than left to the build so the refusal names the line the writer typed it on.
+        raise AuthorError(
+            "AUTHOR_YAML_UNSUPPORTED",
+            number,
+            "? starts a name written on a line of its own, which is not read here; write "
+            "name: value on one line",
+        )
+    key, value = _split_name(rest, number)
+    if key is None:
+        tokens.append(_Token(_VALUE, column, number, text=rest))
+        return
+    _check_name(key, number)
+    tokens.append(_Token(_MAPPING, column, number, key=key, text=value))
+
+
+def _split_name(rest: str, number: int) -> tuple[str | None, str | None]:
+    """Split ``name: value`` at the colon that ends the name, or report there is no name here.
+
+    The colon that ends a name is the first one followed by a space or by the end of the line, so
+    a value holding a time or an address keeps its own colons.
+    """
+
+    if rest[0] in "\"'":
+        name, end = _read_quoted(rest, number)
+        tail = rest[end:]
+        if tail.startswith(":") and (len(tail) == 1 or tail[1] == " "):
+            return name, _inline_value(tail[1:])
+        return None, None
+    for index, character in enumerate(rest):
+        if character == ":" and (index + 1 == len(rest) or rest[index + 1] == " "):
+            return rest[:index].rstrip(), _inline_value(rest[index + 1 :])
+    return None, None
+
+
+def _inline_value(text: str) -> str | None:
+    """What follows the colon, or nothing when only a note follows it."""
+
+    value = text.strip()
+    if not value or value.startswith("#"):
+        return None
+    return value
+
+
+def _check_name(key: str, number: int) -> None:
+    if not key:
+        raise AuthorError("AUTHOR_YAML_UNSUPPORTED", number, "this line has a colon but no name")
+    if key == "<<":
+        raise AuthorError(
+            "AUTHOR_YAML_MERGE_KEY",
+            number,
+            "<< copies one group of names into another, which is not read here; write the names "
+            "out in full",
+        )
+    _refuse_construct(key[0], key, number)
+
+
+# ------------------------------------------------------------------------------------------------
+# Values
+# ------------------------------------------------------------------------------------------------
+
+
+def _scalar(text: str | None, number: int) -> Any:
+    """One plain value, or a typed refusal naming what was written instead."""
+
+    if not text:
+        return None
+    if text[0] in "\"'":
+        value, end = _read_quoted(text, number)
+        _require_note_only(text[end:], number)
+        return value
+    bare = _strip_note(text)
+    if not bare:
+        return None
+    if bare == "[]":
+        return []
+    if bare == "{}":
+        return {}
+    _refuse_construct(bare[0], bare, number)
+    if bare in ("null", "~"):
+        return None
+    if bare == "true":
+        return True
+    if bare == "false":
+        return False
+    if bare in _AMBIGUOUS:
+        raise AuthorError(
+            "AUTHOR_YAML_AMBIGUOUS",
+            number,
+            f"{bare} could be a value or the word itself; write true, false or null in lower "
+            f"case, or put {bare} in quotes to mean the word",
+        )
+    if _INTEGER.fullmatch(bare):
+        return _whole_number(bare, number)
+    if _DECIMAL.fullmatch(bare):
+        return float(bare)
+    return bare
+
+
+def _whole_number(bare: str, number: int) -> int:
+    """One whole number, or the same refusal by name that every other unreadable value gets.
+
+    The only conversion in this reader that the interpreter can refuse: CPython will not turn a
+    decimal string longer than ``sys.get_int_max_str_digits()`` digits into an int, and says so
+    with a bare ``ValueError``. That class is not an ``AuthorError``, is not caught at the command
+    line, and reached the terminal as a Python traceback with no code and no fix -- out of the one
+    module whose whole contract is refusing by name with the line it is on.
+    """
+
+    try:
+        return int(bare)
+    except ValueError as exc:
+        raise AuthorError(
+            "AUTHOR_YAML_NUMBER",
+            number,
+            f"a whole number written with {len(bare.lstrip('+-'))} digits is longer than this "
+            "reader converts; put it in quotes to keep it as text, or write a shorter number",
+        ) from exc
+
+
+def _refuse_construct(head: str, written: str, number: int) -> None:
+    """Refuse, by name, every construct this reader does not read."""
+
+    if head == "&":
+        raise AuthorError(
+            "AUTHOR_YAML_ANCHOR",
+            number,
+            f"& marks a spot to copy from later, which is not read here; write the value out in "
+            f"full instead of {_quote_back(written)}",
+        )
+    if head == "*":
+        raise AuthorError(
+            "AUTHOR_YAML_ALIAS",
+            number,
+            f"* copies a value marked earlier, which is not read here; write the value out in "
+            f"full instead of {_quote_back(written)}",
+        )
+    if head == "!":
+        raise AuthorError(
+            "AUTHOR_YAML_TAG",
+            number,
+            f"! names a type to build, which is never read here; write the plain value instead of "
+            f"{_quote_back(written)}",
+        )
+    if head in "{[":
+        raise AuthorError(
+            "AUTHOR_YAML_FLOW",
+            number,
+            f"a list or group written inside brackets is not read here; write one entry per line "
+            f"under its name instead of {_quote_back(written)}",
+        )
+    if head in "|>":
+        raise AuthorError(
+            "AUTHOR_YAML_BLOCK_SCALAR",
+            number,
+            f"{head} starts text carried over several lines, which is not read here; write the "
+            f"text on one line in quotes, using \\n where you want a line break, instead of "
+            f"{_quote_back(written)}",
+        )
+    if head in "?%@`":
+        raise AuthorError(
+            "AUTHOR_YAML_UNSUPPORTED",
+            number,
+            f"{_quote_back(written)} starts with {head}, which is not read here; {_PLAIN_FORMS}",
+        )
+
+
+def _read_quoted(text: str, number: int) -> tuple[str, int]:
+    """Read one quoted value, returning it and the column just past its closing quote."""
+
+    quote = text[0]
+    out: list[str] = []
+    index = 1
+    while index < len(text):
+        character = text[index]
+        if quote == '"' and character == "\\":
+            if index + 1 >= len(text):
+                break
+            following = text[index + 1]
+            mapped = _ESCAPES.get(following)
+            if mapped is None:
+                raise AuthorError(
+                    "AUTHOR_YAML_QUOTE",
+                    number,
+                    f"\\{following} is not one of the four escapes read here; "
+                    f'write \\" \\\\ \\n or \\t',
+                )
+            out.append(mapped)
+            index += 2
+            continue
+        if character == quote:
+            if quote == "'" and index + 1 < len(text) and text[index + 1] == "'":
+                out.append("'")
+                index += 2
+                continue
+            return "".join(out), index + 1
+        out.append(character)
+        index += 1
+    raise AuthorError(
+        "AUTHOR_YAML_QUOTE",
+        number,
+        f"this {quote} is never closed; close it on the same line",
+    )
+
+
+def _require_note_only(tail: str, number: int) -> None:
+    remainder = tail.strip()
+    if remainder and not remainder.startswith("#"):
+        raise AuthorError(
+            "AUTHOR_YAML_UNSUPPORTED",
+            number,
+            f"{_quote_back(remainder)} follows the closing quote; put the whole value inside the "
+            f"quotes, or start a note with #",
+        )
+
+
+def _strip_note(text: str) -> str:
+    """Drop an end-of-line note. A # with no space before it is part of the word."""
+
+    for index in range(1, len(text)):
+        if text[index] == "#" and text[index - 1] in " \t":
+            return text[:index].rstrip()
+    return text
+
+
+def _quote_back(written: str | None) -> str:
+    return repr(written if written is not None else "")
+
+
+# ------------------------------------------------------------------------------------------------
+# Lines to a value
+# ------------------------------------------------------------------------------------------------
+
+
+def _build(tokens: list[_Token], index: int, column: int, depth: int) -> tuple[Any, int]:
+    if depth > MAX_DEPTH:
+        raise AuthorError(
+            "AUTHOR_YAML_DEPTH",
+            tokens[index].line,
+            f"this is nested more than {MAX_DEPTH} deep; a Recipe never needs that",
+        )
+    token = tokens[index]
+    if token.kind == _SEQUENCE:
+        return _build_sequence(tokens, index, column, depth)
+    if token.kind == _MAPPING:
+        return _build_mapping(tokens, index, column, depth)
+    return _scalar(token.text, token.line), index + 1
+
+
+def _build_mapping(
+    tokens: list[_Token], index: int, column: int, depth: int
+) -> tuple[dict[str, Any], int]:
+    result: dict[str, Any] = {}
+    while index < len(tokens):
+        token = tokens[index]
+        if token.indent < column:
+            break
+        if token.indent > column:
+            raise AuthorError(
+                "AUTHOR_YAML_INDENT",
+                token.line,
+                "this line is indented further in than the line above it opened; line it up with "
+                "the names it belongs to",
+            )
+        if token.kind != _MAPPING:
+            raise AuthorError(
+                "AUTHOR_YAML_INDENT",
+                token.line,
+                "a list entry stands where a name was expected; give the list a name, or indent "
+                "it under the one it belongs to",
+            )
+        name = token.key or ""
+        if name in result:
+            raise AuthorError(
+                "AUTHOR_YAML_DUPLICATE_KEY",
+                token.line,
+                f"{name} is named twice in the same group; every name is written once",
+            )
+        index += 1
+        if token.text is not None:
+            result[name] = _scalar(token.text, token.line)
+            continue
+        result[name], index = _under(tokens, index, column, depth)
+    return result, index
+
+
+def _under(tokens: list[_Token], index: int, column: int, depth: int) -> tuple[Any, int]:
+    """What sits under a name that carries no value of its own."""
+
+    if index < len(tokens) and tokens[index].indent > column:
+        return _build(tokens, index, tokens[index].indent, depth + 1)
+    # A list may sit flush with the name it belongs to; that is the form most people type, and it
+    # means what the indented form means.
+    if index < len(tokens) and tokens[index].indent == column and tokens[index].kind == _SEQUENCE:
+        return _build_sequence(tokens, index, column, depth + 1)
+    return None, index
+
+
+def _build_sequence(
+    tokens: list[_Token], index: int, column: int, depth: int
+) -> tuple[list[Any], int]:
+    items: list[Any] = []
+    while (
+        index < len(tokens) and tokens[index].kind == _SEQUENCE and tokens[index].indent == column
+    ):
+        index += 1
+        if index < len(tokens) and tokens[index].indent > column:
+            item, index = _build(tokens, index, tokens[index].indent, depth + 1)
+        else:
+            item = None
+        items.append(item)
+    return items, index