ymlplus 0.1.0__py3-none-any.whl

yamlplus/_syntax.py

@@ -0,0 +1,511 @@
+"""Literal-profile syntax recognition, without constructing a document value."""
+
+import ast
+import base64
+from dataclasses import dataclass
+import math
+import re
+
+from .errors import ValidationError
+from ._identifiers import (KEYWORDS, PREFIXES, continuation, fold, key_error,
+                           reserved_fold, start)
+from ._numbers import WARN_DIGITS, int_from_decimal
+
+_C32 = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
+_C32_VALUES = {ch: index for index, ch in enumerate(_C32)} | {"O": 0, "I": 1, "L": 1}
+_RADIXES = {"b": (2, "01"), "q": (4, "0-3"), "o": (8, "0-7"),
+            "x": (16, "0-9a-fA-F"), "z": (32, "0-9A-HJ-NP-TV-Za-hj-np-tv-zOoIiLl")}
+_STREAMS = {
+    "bin": (1, "01", False, {}, ""),
+    "dna": (2, "TCAG", True, {}, ""),
+    "oct": (3, "01234567", False, {}, ""),
+    "hex": (4, "0123456789ABCDEF", True, {}, ""),
+    "c32": (5, _C32, True, {"O": "0", "I": "1", "L": "1"}, "-"),
+}
+_ESCAPES = {"\n": "", "\\": "\\", "'": "'", '"': '"', "a": "\a", "b": "\b",
+            "f": "\f", "n": "\n", "r": "\r", "t": "\t", "v": "\v"}
+_PUNCTUATION = {":": "COLON", "=": "EQUAL", ",": "COMMA",
+                "[": "LBRACKET", "]": "RBRACKET", "{": "LBRACE", "}": "RBRACE"}
+
+
+def _digit(character):
+    return bool(character) and "0" <= character <= "9"
+
+
+@dataclass
+class Token:
+    kind: str
+    value: object
+    line: int
+    col: int
+    prefix: str = ""
+    lexeme: str = ""
+
+
+class _Lexer:
+    def __init__(self, text, max_digits, source):
+        self.text, self.max_digits, self.source = text, max_digits, source
+        self.pos, self.line, self.col = 0, 1, 0
+        self.indents, self.brackets, self.based = [0], [], False
+        self.tokens, self.warnings = [], []
+
+    def peek(self, offset=0):
+        return self.text[self.pos + offset:self.pos + offset + 1]
+
+    def advance(self, count=1):
+        text = self.text[self.pos:self.pos + count]
+        for ch in text:
+            if ch == "\n":
+                self.line, self.col = self.line + 1, 0
+            else:
+                self.col += 1
+        self.pos += count
+        return text
+
+    def fail(self, message, line=None, col=None):
+        raise ValidationError(message, self.line if line is None else line,
+                              self.col if col is None else col, self.source)
+
+    def emit(self, kind, value=None, *, line=None, col=None, prefix="", lexeme=""):
+        self.tokens.append(Token(kind, value, self.line if line is None else line,
+                                 self.col if col is None else col, prefix, lexeme))
+
+    def tokenize(self):
+        line_start = True
+        while self.pos < len(self.text):
+            if line_start and not self.brackets:
+                if self.line_start():
+                    continue
+                line_start = False
+            ch = self.peek()
+            if ch == "\n":
+                if not self.brackets:
+                    self.emit("NEWLINE")
+                    line_start = True
+                self.advance()
+            elif ch in (" ", "\t"):
+                self.advance()
+            elif ch == "#":
+                while self.peek() and self.peek() != "\n":
+                    self.advance()
+            else:
+                self.token()
+        if self.brackets:
+            self.fail("unclosed bracket at end of document")
+        if self.tokens and self.tokens[-1].kind not in ("NEWLINE", "DEDENT"):
+            self.emit("NEWLINE")
+        while len(self.indents) > 1:
+            self.indents.pop()
+            self.emit("DEDENT")
+        self.emit("EOF")
+        for index, token in enumerate(self.tokens[:-1]):
+            if token.kind == "STRING" and self.tokens[index + 1].kind == "COLON":
+                token.kind = "QKEY"
+        return self.tokens
+
+    def line_start(self):
+        width = 0
+        while self.peek() in (" ", "\t"):
+            if self.peek() == "\t":
+                self.fail("indentation must use spaces, not tabs")
+            self.advance()
+            width += 1
+        if self.peek() in ("", "\n"):
+            if self.peek():
+                self.advance()
+            return True
+        if self.peek() == "#":
+            while self.peek() and self.peek() != "\n":
+                self.advance()
+            if self.peek():
+                self.advance()
+            return True
+        if not self.based:
+            self.indents, self.based = [width], True
+        if width < self.indents[0]:
+            self.fail("indentation falls left of the document's base column")
+        if width > self.indents[-1]:
+            self.indents.append(width)
+            self.emit("INDENT")
+        else:
+            while width < self.indents[-1]:
+                self.indents.pop()
+                self.emit("DEDENT")
+            if width != self.indents[-1]:
+                self.fail("indentation matches no open block")
+        while self.peek() == "-" and self.peek(1) in ("", " ", "\n"):
+            column = self.col
+            self.emit("DASH")
+            self.advance()
+            if self.peek() == " ":
+                self.advance()
+            self.indents.append(column + 2)
+            self.emit("INDENT")
+        return False
+
+    def token(self):
+        ch, line, col = self.peek(), self.line, self.col
+        if ch in _PUNCTUATION:
+            if ch in "[{":
+                self.brackets.append(ch)
+            elif ch in "]}":
+                if not self.brackets or self.brackets[-1] != {"}": "{", "]": "["}[ch]:
+                    self.fail("unmatched closing bracket")
+                self.brackets.pop()
+            self.emit(_PUNCTUATION[ch], ch, lexeme=ch)
+            self.advance()
+        elif ch in ("'", '"'):
+            self.string("", line, col)
+        elif (_digit(ch) or ch in ("+", "-", ".") and _digit(self.peek(1))
+              or ch in ("+", "-") and self.peek(1) == "." and _digit(self.peek(2))):
+            self.number(line, col)
+        elif start(ch):
+            begin = self.pos
+            self.advance()
+            while continuation(self.peek()):
+                self.advance()
+            name = self.text[begin:self.pos]
+            reserved = reserved_fold(name)
+            if reserved is not None:
+                self.fail(f"reserved spelling {reserved!r} must be written exactly", line, col)
+            if self.peek() in ("'", '"'):
+                if name not in PREFIXES:
+                    self.fail(f"unsupported literal prefix {name!r}", line, col)
+                self.string(name, line, col)
+            elif name in KEYWORDS:
+                self.emit("KEYWORD", KEYWORDS[name], line=line, col=col, lexeme=name)
+            else:
+                self.emit("NAME", fold(name), line=line, col=col, lexeme=name)
+        else:
+            self.fail(f"unexpected character {ch!r}; only literal profile syntax is allowed")
+
+    def number(self, line, col):
+        begin = self.pos
+        if self.peek() in ("+", "-"):
+            self.advance()
+        radix = self.peek() == "0" and self.peek(1).lower() in _RADIXES
+        floating = False
+        if radix:
+            self.advance(2)
+            while self.peek() and (self.peek().isascii() and self.peek().isalnum() or self.peek() == "_"):
+                self.advance()
+        else:
+            while _digit(self.peek()) or self.peek() == "_":
+                self.advance()
+            if self.peek() == ".":
+                floating = True
+                self.advance()
+                while _digit(self.peek()) or self.peek() == "_":
+                    self.advance()
+            if self.peek() in ("e", "E") and (_digit(self.peek(1)) or
+                    self.peek(1) in ("+", "-") and _digit(self.peek(2))):
+                floating = True
+                self.advance()
+                if self.peek() in ("+", "-"):
+                    self.advance()
+                while _digit(self.peek()) or self.peek() == "_":
+                    self.advance()
+        if continuation(self.peek()) or self.peek().isalnum():
+            self.fail("a number cannot be followed by name characters", line, col)
+        spelling = self.text[begin:self.pos]
+        sign = spelling[:1] if spelling[:1] in ("+", "-") else ""
+        body = spelling[len(sign):]
+        if radix:
+            base, digits = _RADIXES[body[1].lower()]
+            rest = body[2:]
+            if not re.fullmatch(rf"_?[{digits}]+(?:_[{digits}]+)*", rest):
+                self.fail("malformed radix integer", line, col)
+            rest = rest.replace("_", "")
+            if base == 32:
+                value = 0
+                for ch in rest.upper():
+                    value = value * 32 + _C32_VALUES[ch]
+            else:
+                value = int(rest, base)
+        elif not floating:
+            if not re.fullmatch(r"0(?:_?0)*|[1-9](?:_?[0-9])*", body):
+                self.fail("malformed decimal integer", line, col)
+            digits = body.replace("_", "")
+            if len(digits) > self.max_digits:
+                self.fail("decimal integer exceeds max_digits", line, col)
+            if len(digits) > WARN_DIGITS:
+                where = f"{self.source}:{line}:{col}" if self.source else f"line {line}, column {col}"
+                self.warnings.append(f"warning: {where}: integer has {len(digits)} decimal digits; conversion is costly")
+            value = int_from_decimal(digits)
+        else:
+            try:
+                value = ast.literal_eval(body)
+            except (SyntaxError, ValueError):
+                self.fail("malformed floating-point literal", line, col)
+            if not isinstance(value, float) or not math.isfinite(value):
+                self.fail("a float must be finite", line, col)
+        if sign == "-":
+            value = -value
+        self.emit("FLOAT" if floating else "INT", value, line=line, col=col, lexeme=spelling)
+
+    def string(self, prefix, line, col):
+        begin = self.pos - len(prefix)
+        quote = self.peek()
+        triple = self.peek(1) == quote and self.peek(2) == quote
+        self.advance(3 if triple else 1)
+        body, positions = self.string_body(quote, triple, line, col)
+        is_bytes = prefix not in ("", "r")
+        if prefix == "b64":
+            for index, ch in enumerate(body):
+                if ch not in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/= \t\n":
+                    self.fail("invalid base64 symbol", *positions[index])
+            compact = "".join(ch for ch in body if ch not in " \t\n")
+            if len(compact) % 4:
+                self.fail("base64 requires padding to a multiple of four symbols", line, col)
+            try:
+                value = base64.b64decode(compact, validate=True)
+            except ValueError:
+                self.fail("invalid base64 payload", line, col)
+            if base64.b64encode(value).decode("ascii") != compact:
+                self.fail("non-canonical base64 padding or spare bits", line, col)
+        elif prefix in _STREAMS:
+            value = self.stream(prefix, body, positions, line, col)
+        elif prefix in ("r", "rb"):
+            if is_bytes and any(ord(ch) > 127 for ch in body):
+                self.fail("raw bytes require ASCII source characters", line, col)
+            value = body.encode("ascii") if is_bytes else body
+        else:
+            value = self.escapes(body, positions, is_bytes, line, col)
+        self.emit("BYTES" if is_bytes else "STRING", value, line=line, col=col,
+                  prefix=prefix, lexeme=self.text[begin:self.pos])
+
+    def string_body(self, quote, triple, line, col):
+        out, positions = [], []
+        closer, anchor, pristine = quote * (3 if triple else 1), None, True
+        while True:
+            if not self.peek():
+                self.fail("unterminated literal", line, col)
+            if self.text.startswith(closer, self.pos):
+                if triple and anchor is not None and self.col < anchor:
+                    self.fail("closing quote outdents past the literal's indentation")
+                self.advance(len(closer))
+                return "".join(out), positions
+            if self.peek() == "\\" and self.peek(1) not in ("", "\n"):
+                for _ in range(2):
+                    positions.append((self.line, self.col))
+                    out.append(self.advance())
+                pristine = False
+            elif self.peek() == "\n":
+                if not triple:
+                    self.fail("a single-line literal cannot contain a raw newline", line, col)
+                here = self.line, self.col
+                self.advance()
+                if not pristine:
+                    out.append("\n")
+                    positions.append(here)
+                pristine = False
+                width = 0
+                while self.peek() == " ":
+                    self.advance()
+                    width += 1
+                if self.peek() in ("", "\n") or self.text.startswith(closer, self.pos):
+                    continue
+                if anchor is None:
+                    anchor = width
+                elif width < anchor:
+                    self.fail("literal content outdents past its indentation anchor")
+                else:
+                    out.append(" " * (width - anchor))
+                    positions.extend((self.line, anchor + index) for index in range(width - anchor))
+            else:
+                positions.append((self.line, self.col))
+                out.append(self.advance())
+                pristine = False
+
+    def escapes(self, body, positions, is_bytes, line, col):
+        out, index = [], 0
+
+        def emit(point, at, literal=False):
+            if is_bytes and (point > 255 or literal and point > 127):
+                self.fail("bytes require ASCII source and escape values within 0..255", *positions[at])
+            out.append(point)
+
+        while index < len(body):
+            if body[index] != "\\":
+                emit(ord(body[index]), index, literal=True)
+                index += 1
+                continue
+            begin = index
+            index += 1
+            if index >= len(body):
+                self.fail("dangling backslash", line, col)
+            escape = body[index]
+            if escape in _ESCAPES:
+                for ch in _ESCAPES[escape]:
+                    emit(ord(ch), begin)
+                index += 1
+            elif escape == "x":
+                digits = body[index + 1:index + 3]
+                if len(digits) != 2 or any(ch not in "0123456789abcdefABCDEF" for ch in digits):
+                    self.fail("incomplete hexadecimal escape", *positions[begin])
+                emit(int(digits, 16), begin)
+                index += 3
+            elif escape in "01234567":
+                end = index
+                while end < len(body) and end - index < 3 and body[end] in "01234567":
+                    end += 1
+                emit(int(body[index:end], 8), begin)
+                index = end
+            elif escape in ("u", "U") and not is_bytes:
+                width = 4 if escape == "u" else 8
+                digits = body[index + 1:index + width + 1]
+                if len(digits) != width or any(ch not in "0123456789abcdefABCDEF" for ch in digits):
+                    self.fail("incomplete Unicode escape", *positions[begin])
+                point = int(digits, 16)
+                if point > 0x10FFFF or 0xD800 <= point <= 0xDFFF:
+                    self.fail("an escape must name a Unicode scalar value", *positions[begin])
+                emit(point, begin)
+                index += width + 1
+            else:
+                self.fail(f"unknown escape \\{escape}", *positions[begin])
+        return bytes(out) if is_bytes else "".join(chr(point) for point in out)
+
+    def stream(self, prefix, body, positions, line, col):
+        bits, alphabet, folds, aliases, ignored = _STREAMS[prefix]
+        symbols = []
+        for index, ch in enumerate(body):
+            if ch in " \t\n" or ch in ignored:
+                continue
+            key = ch.upper() if folds else ch
+            key = aliases.get(key, key)
+            value = alphabet.find(key)
+            if value < 0:
+                self.fail(f"invalid {prefix} symbol", *positions[index])
+            symbols.append(value)
+        total, number = len(symbols) * bits, 0
+        for value in symbols:
+            number = number << bits | value
+        if 8 % bits == 0:
+            count = -(-total // 8)
+            return (number << (8 * count - total)).to_bytes(count, "big")
+        count = total // 8
+        if -(-8 * count // bits) != len(symbols):
+            self.fail("stream symbol count does not encode a whole byte payload", line, col)
+        spare = total - 8 * count
+        if number & ((1 << spare) - 1):
+            self.fail("stream padding bits must be zero", line, col)
+        return (number >> spare).to_bytes(count, "big")
+
+
+class _Parser:
+    def __init__(self, tokens, source):
+        self.tokens, self.source, self.index = tokens, source, 0
+
+    def peek(self, offset=0):
+        return self.tokens[min(self.index + offset, len(self.tokens) - 1)]
+
+    def take(self, kind):
+        if self.peek().kind == kind:
+            token = self.peek()
+            if kind != "EOF":
+                self.index += 1
+            return token
+        return None
+
+    def fail(self, message, token=None):
+        token = token or self.peek()
+        raise ValidationError(message, token.line, token.col, self.source)
+
+    def expect(self, kind):
+        token = self.take(kind)
+        if token is None:
+            self.fail(f"expected {kind.lower()}, found {self.peek().kind.lower()}")
+        return token
+
+    def at_key(self):
+        return self.peek().kind in ("NAME", "QKEY") and self.peek(1).kind == "COLON"
+
+    def key(self, seen):
+        token = self.peek()
+        if token.kind not in ("NAME", "QKEY"):
+            self.fail("a mapping key must be a name or a quoted string")
+        self.index += 1
+        if token.kind == "QKEY" and token.prefix:
+            self.fail("a quoted key cannot have a literal prefix", token)
+        why = key_error(token.value)
+        if why:
+            self.fail(why, token)
+        if token.value in seen:
+            self.fail("duplicate key", token)
+        seen.add(token.value)
+        self.expect("COLON")
+        self.take("EQUAL")
+
+    def value(self, depth=0):
+        if depth > 128:
+            self.fail("source nesting exceeds this implementation's ceiling of 128")
+        if self.at_key():
+            seen = set()
+            while self.at_key():
+                self.key(seen)
+                if self.take("NEWLINE"):
+                    self.expect("INDENT")
+                    self.value(depth + 1)
+                    self.expect("DEDENT")
+                else:
+                    self.flow(depth + 1)
+                    self.expect("NEWLINE")
+        elif self.peek().kind == "DASH":
+            while self.take("DASH"):
+                self.expect("INDENT")
+                deeper = False
+                if self.take("NEWLINE"):
+                    deeper = self.take("INDENT") is not None
+                self.value(depth + 1)
+                if deeper:
+                    self.expect("DEDENT")
+                self.expect("DEDENT")
+        else:
+            self.flow(depth + 1)
+            self.expect("NEWLINE")
+
+    def flow(self, depth):
+        if depth > 128:
+            self.fail("source nesting exceeds this implementation's ceiling of 128")
+        if self.take("LBRACKET"):
+            if self.take("RBRACKET"):
+                return
+            while True:
+                self.flow(depth + 1)
+                if self.take("RBRACKET"):
+                    return
+                self.expect("COMMA")
+                if self.take("RBRACKET"):
+                    return
+        elif self.take("LBRACE"):
+            seen = set()
+            if self.take("RBRACE"):
+                return
+            while True:
+                self.key(seen)
+                self.flow(depth + 1)
+                if self.take("RBRACE"):
+                    return
+                self.expect("COMMA")
+                if self.take("RBRACE"):
+                    return
+        elif self.peek().kind in ("STRING", "BYTES", "INT", "FLOAT", "KEYWORD"):
+            self.index += 1
+        else:
+            self.fail("expected a literal, list, or mapping; bare text and computation are not profile values")
+
+    def parse(self):
+        if self.peek().kind == "EOF":
+            self.fail("a document must contain one value")
+        self.value()
+        self.expect("EOF")
+
+
+def validate(text, max_digits, source=None):
+    lexer = _Lexer(text, max_digits, source)
+    tokens = lexer.tokenize()
+    try:
+        _Parser(tokens, source).parse()
+    except RecursionError:
+        raise ValidationError("source nesting exceeds the parser's stack capacity", source=source) from None
+    return tokens, lexer.warnings