ymlplus 0.1.0__py3-none-any.whl

yamlplus/__init__.py

@@ -0,0 +1,16 @@
+"""YamlPlus validation and profile-conformant YAML I/O through PyYAML.
+
+Reading uses native safe YAML semantics. Output is validated against the
+literal profile before return. Identifier syntax is pinned to Unicode 14.0.0.
+"""
+
+from ._xid import UNICODE_VERSION
+from .errors import ValidationError
+from .api import check, dump, dumps, is_yamlplus, load, loads, normalize
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "load", "loads", "dump", "dumps", "check", "is_yamlplus", "normalize",
+    "ValidationError", "UNICODE_VERSION",
+]

yamlplus/__main__.py

@@ -0,0 +1,8 @@
+"""Validate or format YamlPlus with ``python -m yamlplus [--write] [FILE]``."""
+
+import sys
+
+from .api import main
+
+if __name__ == "__main__":
+    sys.exit(main())

yamlplus/_identifiers.py

@@ -0,0 +1,72 @@
+"""Pinned profile identifiers and exact string-key validation."""
+
+from bisect import bisect_right
+from unicodedata import decomposition, normalize
+
+from ._xid import XID_START, XID_CONTINUE
+
+_START = tuple(lo for lo, _ in XID_START)
+_CONTINUE = tuple(lo for lo, _ in XID_CONTINUE)
+KEYWORDS = {"True": True, "False": False, "None": None}
+PREFIXES = frozenset({"b", "rb", "r", "bin", "dna", "oct", "hex", "c32", "b64"})
+_RESERVED_PREFIXES = PREFIXES | {"y", "yb", "yt", "ry"}
+
+
+def _in(character, lows, ranges):
+    if not character:
+        return False
+    point = ord(character)
+    index = bisect_right(lows, point) - 1
+    return index >= 0 and point <= ranges[index][1]
+
+
+def start(character):
+    return character == "_" or _in(character, _START, XID_START)
+
+
+def continuation(character):
+    return _in(character, _CONTINUE, XID_CONTINUE)
+
+
+def identifier(value):
+    return bool(value) and start(value[0]) and all(continuation(ch) for ch in value[1:])
+
+
+def fold(value):
+    out = []
+    for ch in value:
+        mapped = decomposition(ch)
+        if mapped.startswith(("<wide>", "<narrow>")):
+            out.extend(chr(int(part, 16)) for part in mapped.split()[1:])
+        else:
+            out.append(ch)
+    return normalize("NFC", "".join(out))
+
+
+def dunder(value):
+    return value.startswith("__") and value.endswith("__")
+
+
+def reserved_fold(value):
+    folded = fold(value)
+    return (folded if folded != value and
+            (folded in KEYWORDS or folded in _RESERVED_PREFIXES or dunder(folded)) else None)
+
+
+def key_error(value):
+    if not isinstance(value, str):
+        return "a key must be a string"
+    if not value:
+        return "a key must be nonempty"
+    if value.startswith("@"):
+        return "keys beginning with '@' are reserved"
+    if dunder(value):
+        return "keys beginning and ending with '__' are reserved"
+    if any(0xD800 <= ord(ch) <= 0xDFFF for ch in value):
+        return "a key cannot contain a lone surrogate"
+    return ""
+
+
+def plain_key(value):
+    return (identifier(value) and value not in KEYWORDS and not dunder(value)
+            and reserved_fold(value) is None and fold(value) == value)

yamlplus/_io.py

@@ -0,0 +1,262 @@
+"""Safe PyYAML input and profile-conformant, bounded host-value output."""
+
+import base64
+import math
+import sys
+
+import yaml
+from yaml.events import ScalarEvent
+
+from .errors import ValidationError
+from ._identifiers import key_error, plain_key
+from ._numbers import WARN_DIGITS, decimal_digits, decimal_text
+from ._pyyaml import Loader
+from ._validation import (Notices, analyze, callback_option, digit_option,
+                          implicit_type, key_warning, send)
+
+MAX_DEPTH = 128
+MAX_SIZE = 4 * 1024 * 1024
+
+
+def read(text, source=None):
+    try:
+        return yaml.load(text, Loader=Loader)
+    except yaml.YAMLError as error:
+        mark = getattr(error, "problem_mark", None)
+        raise ValidationError(getattr(error, "problem", None) or "YAML value construction failed",
+                              mark.line + 1 if mark else 1, mark.column if mark else 0, source) from None
+    except RecursionError:
+        raise ValidationError("YAML construction exceeds the runtime's stack capacity", source=source) from None
+
+
+class _Dumper(yaml.SafeDumper):
+    ESCAPE_REPLACEMENTS = {
+        key: value for key, value in yaml.SafeDumper.ESCAPE_REPLACEMENTS.items()
+        if value not in ("e", "N", "_", "L", "P")
+    } | {"\0": "x00"}
+
+    def ignore_aliases(self, data):
+        return True
+
+    def increase_indent(self, flow=False, indentless=False):
+        return super().increase_indent(flow, indentless=False)
+
+    def expect_document_end(self):
+        super().expect_document_end()
+        self.open_ended = False
+
+    def check_simple_key(self):
+        ordinary = super().check_simple_key()
+        return ordinary or isinstance(self.event, ScalarEvent) and self.event.tag == "tag:yaml.org,2002:str"
+
+    def represent_mapping(self, tag, mapping, flow_style=None):
+        node = super().represent_mapping(tag, mapping, flow_style)
+        for key, _ in node.value:
+            key.style = None if plain_key(key.value) and not implicit_type(key.value) else '"'
+        return node
+
+
+def _text(dumper, value):
+    return dumper.represent_scalar("tag:yaml.org,2002:str", value, style='"')
+
+
+def _none(dumper, value):
+    return dumper.represent_scalar("tag:yaml.org,2002:str", "None")
+
+
+def _bytes(dumper, value):
+    literal = 'b64"' + base64.b64encode(value).decode("ascii") + '"'
+    return dumper.represent_scalar("tag:yaml.org,2002:str", literal)
+
+
+def _bool(dumper, value):
+    return dumper.represent_scalar("tag:yaml.org,2002:bool", "True" if value else "False")
+
+
+def _int(dumper, value):
+    return dumper.represent_scalar("tag:yaml.org,2002:int", decimal_text(value))
+
+
+for _type, _represent in ((str, _text), (type(None), _none), (bytes, _bytes),
+                          (bool, _bool), (int, _int)):
+    _Dumper.add_representer(_type, _represent)
+
+
+class _Path:
+    def __init__(self, parent=None, step="root"):
+        self.parent, self.step = parent, step
+
+    def child(self, step):
+        return _Path(self, step)
+
+    def __str__(self):
+        out, node = [], self
+        while node is not None:
+            out.append(node.step)
+            node = node.parent
+        return "".join(reversed(out))
+
+
+def _key_step(key, index):
+    if not isinstance(key, str):
+        return f"[entry {index} key]"
+    text = repr(str.__getitem__(key, slice(0, 64)))
+    return f"[{text}{'...' if str.__len__(key) > 64 else ''}]"
+
+
+class _Budget:
+    def __init__(self, maximum):
+        self.maximum, self.used = maximum, 0
+
+    def spend(self, amount, path):
+        if amount > self.maximum - self.used:
+            raise ValidationError(f"serialized UTF-8 exceeds max_size of {self.maximum} bytes", path=str(path))
+        self.used += amount
+
+    def text(self, value, path):
+        if len(value) > self.maximum - self.used:
+            self.spend(len(value), path)
+        for start in range(0, len(value), 4096):
+            try:
+                size = len(value[start:start + 4096].encode("utf-8"))
+            except UnicodeEncodeError:
+                raise ValidationError("text cannot contain a lone surrogate", path=str(path)) from None
+            self.spend(size, path)
+
+
+class _Output:
+    def __init__(self, maximum):
+        self.budget, self.parts = _Budget(maximum), []
+
+    def write(self, text):
+        self.budget.text(text, "root")
+        self.parts.append(text)
+        return len(text)
+
+    def flush(self):
+        pass
+
+
+def options(max_digits, max_depth, max_size, warn, info):
+    max_digits = digit_option(max_digits)
+    for name, value in (("max_depth", max_depth), ("max_size", max_size)):
+        if not isinstance(value, int) or isinstance(value, bool):
+            raise TypeError(f"{name} must be an integer")
+        if value < 1:
+            raise ValueError(f"{name} must be positive")
+    callback_option(warn, "warn")
+    callback_option(info, "info")
+    return max_digits, int.__int__(max_depth), int.__int__(max_size)
+
+
+def _prepare_host(value, max_digits, max_depth, max_size):
+    budget, notices, warnings = _Budget(max_size), Notices(), []
+    active, root = {}, []
+
+    def children(original, out, path):
+        if isinstance(out, list):
+            for index, child in enumerate(list.__iter__(original)):
+                yield child, path.child(f"[{index}]"), out.append
+        else:
+            for index, (key, child) in enumerate(dict.items(original)):
+                child_path = path.child(_key_step(key, index))
+                if not isinstance(key, str):
+                    raise TypeError(f"at {child_path}: a key must be a string")
+                key = str.__str__(key)
+                why = key_error(key)
+                if why:
+                    raise ValidationError(why, path=str(child_path))
+                if len(key) > 1024:
+                    raise ValidationError("an implicit key's source spelling must fit within 1,024 characters", path=str(child_path))
+                budget.text(key, child_path)
+                kind = implicit_type(key)
+                budget.spend(2 if plain_key(key) and not kind else 4, child_path)
+                if kind:
+                    warnings.append(key_warning(key, kind, f"at {child_path}", writing=True))
+
+                def put(item, key=key, out=out):
+                    out[key] = item
+
+                yield child, child_path, put
+
+    pending = [("value", value, _Path(), 0, root.append)]
+    while pending:
+        task = pending.pop()
+        if task[0] == "children":
+            _, original, path, depth, iterator = task
+            try:
+                child, child_path, put = next(iterator)
+            except StopIteration:
+                del active[id(original)]
+                continue
+            pending.append(task)
+            pending.append(("value", child, child_path, depth, put))
+            continue
+        _, original, path, depth, put = task
+        if original is None:
+            budget.spend(4, path)
+            notices.add("None", f"at {path}")
+            put(None)
+        elif isinstance(original, bool):
+            budget.spend(4 if original else 5, path)
+            put(original)
+        elif isinstance(original, int):
+            bits = int.bit_length(original)
+            if bits > (max_digits * 3322 + 999) // 1000:
+                raise ValidationError("integer exceeds max_digits", path=str(path))
+            value = int.__int__(original)
+            count = decimal_digits(value)
+            if count > max_digits:
+                raise ValidationError("integer exceeds max_digits", path=str(path))
+            budget.spend(count + (value < 0), path)
+            if count > WARN_DIGITS:
+                warnings.append(f"warning: at {path}: integer has {count} decimal digits; conversion is costly")
+            put(value)
+        elif isinstance(original, float):
+            value = float.__float__(original)
+            if not math.isfinite(value):
+                raise ValidationError("a float must be finite", path=str(path))
+            budget.spend(len(repr(value)), path)
+            put(value)
+        elif isinstance(original, str):
+            if str.__len__(original) > budget.maximum - budget.used:
+                budget.spend(str.__len__(original), path)
+            value = str.__str__(original)
+            budget.text(value, path)
+            budget.spend(2, path)
+            put(value)
+        elif isinstance(original, bytes):
+            budget.spend(4 * ((bytes.__len__(original) + 2) // 3) + 5, path)
+            notices.add("bytes", f"at {path}")
+            put(bytes.__bytes__(original))
+        elif isinstance(original, (dict, list)):
+            if id(original) in active:
+                raise ValidationError(f"container cycle reaches ancestor at {active[id(original)]}", path=str(path))
+            depth += 1
+            if depth > max_depth:
+                raise ValidationError(f"container depth exceeds max_depth of {max_depth}", path=str(path))
+            empty = not (dict.__len__(original) if isinstance(original, dict) else list.__len__(original))
+            budget.spend(2 if empty else 1, path)
+            out = {} if isinstance(original, dict) else []
+            put(out)
+            active[id(original)] = path
+            pending.append(("children", original, path, depth, iter(children(original, out, path))))
+        else:
+            raise TypeError(f"at {path}: {type(original).__name__} is not supported profile data")
+    return root[0], warnings, list(notices.messages())
+
+
+def write(value, max_digits, max_depth, max_size, warn, info):
+    value, warnings, notices = _prepare_host(value, max_digits, max_depth, max_size)
+    output = _Output(max_size)
+    try:
+        yaml.dump(value, stream=output, Dumper=_Dumper, sort_keys=False,
+                  default_flow_style=False, allow_unicode=True, width=sys.maxsize,
+                  explicit_start=False, explicit_end=False)
+    except RecursionError:
+        raise ValidationError("YAML emission exceeds the runtime's stack capacity", path="root") from None
+    text = "".join(output.parts)
+    analyze(text, max_digits)
+    send(warn, warnings)
+    send(info, notices)
+    return text

yamlplus/_numbers.py

@@ -0,0 +1,40 @@
+"""Bounded decimal conversion independent of Python's process-wide digit cap."""
+
+WARN_DIGITS = 4300
+REFUSE_DIGITS = 4 * WARN_DIGITS
+_CHUNK = 500
+
+
+def int_from_decimal(digits):
+    if len(digits) <= _CHUNK:
+        return int(digits)
+    half = len(digits) // 2
+    return (int_from_decimal(digits[:-half]) * 10 ** half
+            + int_from_decimal(digits[-half:]))
+
+
+def decimal_text(value):
+    if value < 0:
+        return "-" + decimal_text(-value)
+    if value < 10 ** _CHUNK:
+        return str(value)
+    half = max(_CHUNK, (value.bit_length() * 30103 // 100000) // 2)
+    high, low = divmod(value, 10 ** half)
+    return decimal_text(high) + decimal_text(low).rjust(half, "0")
+
+
+def decimal_digits(value):
+    value = abs(value)
+    if value < 10 ** _CHUNK:
+        return len(str(value))
+    estimate = value.bit_length() * 30103 // 100000
+    return estimate + (value >= 10 ** estimate)
+
+
+def float_text(value):
+    text = repr(value)
+    if "e" in text:
+        mantissa, exponent = text.split("e")
+        if "." not in mantissa:
+            text = f"{mantissa}.0e{exponent}"
+    return text

yamlplus/_pyyaml.py

@@ -0,0 +1,53 @@
+"""Private safe loader with profile-compatible whitespace and decimal handling."""
+
+import yaml
+
+from ._numbers import int_from_decimal
+
+
+class Loader(yaml.SafeLoader):
+    def scan_to_next_token(self):
+        # Profile validation has already excluded indentation tabs.
+        while True:
+            super().scan_to_next_token()
+            if self.peek() != "\t":
+                return
+            while self.peek() == "\t":
+                self.forward()
+
+    def scan_plain_spaces(self, indent, start_mark):
+        length = 0
+        while self.peek(length) in " \t":
+            length += 1
+        whitespace = self.prefix(length)
+        self.forward(length)
+        if self.peek() not in "\r\n\x85

":
+            return [whitespace] if whitespace else []
+        line_break = self.scan_line_break()
+        self.allow_simple_key = True
+        breaks = []
+        while True:
+            if (self.prefix(3) in ("---", "...") and
+                    self.peek(3) in "\0 \t\r\n\x85

"):
+                return None
+            if self.peek() in " \t":
+                self.forward()
+            elif self.peek() in "\r\n\x85

":
+                breaks.append(self.scan_line_break())
+            else:
+                break
+        if line_break != "\n":
+            return [line_break, *breaks]
+        return breaks or [" "]
+
+
+def _integer(loader, node):
+    text = loader.construct_scalar(node).replace("_", "")
+    unsigned = text.lstrip("+-")
+    if unsigned.isascii() and unsigned.isdecimal() and (len(unsigned) == 1 or unsigned[0] != "0"):
+        value = int_from_decimal(unsigned)
+        return -value if text.startswith("-") else value
+    return yaml.SafeLoader.construct_yaml_int(loader, node)
+
+
+Loader.add_constructor("tag:yaml.org,2002:int", _integer)