links-notation 0.22.0__tar.gz → 0.24.0__tar.gz

{links_notation-0.22.0/links_notation.egg-info → links_notation-0.24.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: links-notation
-Version: 0.22.0
+Version: 0.24.0
 Summary: Python implementation of the Links Notation parser
 Author-email: "Link.Foundation" <drakonard@gmail.com>
 License-Expression: Unlicense
@@ -22,18 +22,18 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Provides-Extra: test
-Requires-Dist: pytest>=9.1; extra == "test"
-Requires-Dist: pytest-timeout>=2.4; extra == "test"
+Requires-Dist: pytest>=9.1.1; extra == "test"
+Requires-Dist: pytest-timeout>=2.4.0; extra == "test"
 Provides-Extra: lint
-Requires-Dist: black>=26.5; extra == "lint"
-Requires-Dist: isort>=9.0; extra == "lint"
-Requires-Dist: flake8>=7.3; extra == "lint"
+Requires-Dist: black>=26.10.0; extra == "lint"
+Requires-Dist: isort>=9.0.2; extra == "lint"
+Requires-Dist: flake8>=7.4.1; extra == "lint"
 Provides-Extra: dev
-Requires-Dist: pytest>=9.1; extra == "dev"
-Requires-Dist: pytest-timeout>=2.4; extra == "dev"
-Requires-Dist: black>=26.5; extra == "dev"
-Requires-Dist: isort>=9.0; extra == "dev"
-Requires-Dist: flake8>=7.3; extra == "dev"
+Requires-Dist: pytest>=9.1.1; extra == "dev"
+Requires-Dist: pytest-timeout>=2.4.0; extra == "dev"
+Requires-Dist: black>=26.10.0; extra == "dev"
+Requires-Dist: isort>=9.0.2; extra == "dev"
+Requires-Dist: flake8>=7.4.1; extra == "dev"
 Dynamic: license-file
 
 # Links Notation Parser for Python
@@ -306,7 +306,7 @@ python -m build
 
 This project uses [Black](https://github.com/psf/black) for code formatting,
 [isort](https://pycqa.github.io/isort/) for import sorting, and
-[flake8](https://flake8.pycqa.org/) for linting.
+[flake8](https://flake8.pycqa.org/en/latest/) for linting.
 
 Install linting tools:
 
@@ -354,3 +354,6 @@ This project is released into the public domain under the [Unlicense](../LICENSE
 - [Main Repository](https://github.com/link-foundation/links-notation)
 - [PyPI Package](https://pypi.org/project/links-notation/)
 - [Documentation](https://link-foundation.github.io/links-notation/)
+
+Exact Unicode identifiers and values round-trip with the [reference literal contract](../docs/protocol/reference-literals.md)
+introduced in 0.24.0, including mixed quotes, empty strings, controls and multiline text.

{links_notation-0.22.0 → links_notation-0.24.0}/README.md

@@ -268,7 +268,7 @@ python -m build
 
 This project uses [Black](https://github.com/psf/black) for code formatting,
 [isort](https://pycqa.github.io/isort/) for import sorting, and
-[flake8](https://flake8.pycqa.org/) for linting.
+[flake8](https://flake8.pycqa.org/en/latest/) for linting.
 
 Install linting tools:
 
@@ -316,3 +316,6 @@ This project is released into the public domain under the [Unlicense](../LICENSE
 - [Main Repository](https://github.com/link-foundation/links-notation)
 - [PyPI Package](https://pypi.org/project/links-notation/)
 - [Documentation](https://link-foundation.github.io/links-notation/)
+
+Exact Unicode identifiers and values round-trip with the [reference literal contract](../docs/protocol/reference-literals.md)
+introduced in 0.24.0, including mixed quotes, empty strings, controls and multiline text.

{links_notation-0.22.0 → links_notation-0.24.0}/links_notation/__init__.py

@@ -10,11 +10,23 @@ from importlib.metadata import PackageNotFoundError
 from importlib.metadata import version as _distribution_version
 from pathlib import Path
 
+from .binary import (
+    ArityRange,
+    BinaryLinoCodec,
+    BinaryLinoOptions,
+    DecodeLimits,
+    External,
+    LinksPacket,
+    Section,
+)
+from .binary import format_document as format_binary_document
+from .binary import format_reference as format_binary_reference
 from .comments import strip_comments
 from .format_config import FormatConfig
 from .formatter import format_links
 from .link import Link
 from .parser import DEFAULT_MAX_DEPTH, ParseError, Parser
+from .reference_literal import decode_reference_literal, encode_reference_literal
 from .stream_parser import StreamParseError, StreamParser, StreamPosition, parse_async_chunks, parse_chunks
 
 
@@ -47,6 +59,17 @@ def _read_version() -> str:
 __version__ = _read_version()
 
 __all__ = [
+    "decode_reference_literal",
+    "encode_reference_literal",
+    "ArityRange",
+    "BinaryLinoCodec",
+    "BinaryLinoOptions",
+    "DecodeLimits",
+    "External",
+    "LinksPacket",
+    "Section",
+    "format_binary_document",
+    "format_binary_reference",
     "DEFAULT_MAX_DEPTH",
     "Link",
     "ParseError",

links_notation-0.24.0/links_notation/binary.py

@@ -0,0 +1,624 @@
+"""Binary links notation v1. See docs/protocol/binary-links-notation.md.
+
+The packet layer is independent of the document mapping. Native Link models
+are preserved; parse_document explicitly selects link-cli canonical groups.
+All integers on the wire are unsigned 64-bit values.
+"""
+
+import io
+import re
+from dataclasses import dataclass, field
+from typing import Optional
+
+from .link import Link
+from .parser import Parser
+from .reference_literal import format_reference
+
+U64 = (1 << 64) - 1
+WIDTHS = (1, 2, 4, 8)
+
+
+def _require(condition, message):
+    if not condition:
+        raise ValueError(message)
+
+
+def _u64(value):
+    _require(isinstance(value, int) and not isinstance(value, bool) and 0 <= value <= U64, "expected uint64")
+    return value
+
+
+@dataclass(frozen=True)
+class External:
+    """Hybrid external reference; bare integers denote internal addresses."""
+
+    value: int
+
+    def __post_init__(self):
+        _u64(self.value)
+
+
+@dataclass(frozen=True)
+class ArityRange:
+    min: int = 2
+    max: Optional[int] = 2
+
+    def __post_init__(self):
+        _require(1 <= _u64(self.min) <= U64 >> 4, "invalid minimum arity")
+        _require(self.max is None or _u64(self.max) >= self.min, "invalid maximum arity")
+
+    def contains(self, length):
+        return length >= self.min and (self.max is None or length <= self.max)
+
+    @property
+    def fixed(self):
+        return self.min == self.max
+
+    @classmethod
+    def parse(cls, text):
+        _require(re.fullmatch(r"[0-9]+(?:\.\.[0-9]*)?", text), "invalid arity")
+        parts = text.split("..")
+        return cls(int(parts[0]), int(parts[-1]) if parts[-1] else None)
+
+
+@dataclass(frozen=True)
+class DecodeLimits:
+    """Work budgets applied to both byte/model input and encoding."""
+
+    max_links: int = 1 << 22
+    max_references: int = 1 << 24
+    max_nodes: int = 1 << 22
+    max_string_bytes: int = 64 << 20
+    max_depth: int = 64
+
+    def __post_init__(self):
+        for value in vars(self).values():
+            _u64(value)
+
+    @classmethod
+    def unlimited(cls):
+        """For trusted input; caller must provide adequate memory/stack."""
+        return cls(U64, U64, U64, U64, U64)
+
+
+@dataclass(frozen=True)
+class BinaryLinoOptions:
+    external_references: bool = False
+    arity: ArityRange = field(default_factory=ArityRange)
+    packed_widths: bool = False
+
+    @classmethod
+    def of_packet(cls, packet):
+        lengths = [2] + [len(link) for _, link in packet.links()]
+        return cls(
+            packet.external_references,
+            ArityRange(min(lengths), max(lengths)),
+            len({section.width for section in packet.sections}) > 1,
+        )
+
+
+@dataclass
+class Section:
+    gap: int
+    arity: ArityRange
+    width: int
+    links: list
+
+
+def _width(reference, external):
+    value = reference.value if isinstance(reference, External) else _u64(reference)
+    _require(not isinstance(reference, External) or external, "external reference disabled")
+    for width in WIDTHS:
+        bits = width * 8 - int(external or isinstance(reference, External))
+        if value < 1 << bits:
+            return width
+    raise ValueError("reference exceeds capacity")
+
+
+def _leb(out, value):
+    value = _u64(value)
+    while value >= 128:
+        out.append((value & 127) | 128)
+        value >>= 7
+    out.append(value)
+
+
+class _Reader:
+    def __init__(self, stream):
+        self.stream = stream
+
+    def raw(self, count):
+        chunks = bytearray()
+        while len(chunks) < count:
+            part = self.stream.read(count - len(chunks))
+            _require(part, "unexpected end of packet")
+            chunks.extend(part)
+        return int.from_bytes(chunks, "little")
+
+    def leb(self):
+        value = 0
+        for shift in range(0, 64, 7):
+            byte = self.raw(1)
+            payload = byte & 127
+            _require(shift != 63 or payload <= 1, "LEB128 overflow")
+            value |= payload << shift
+            if byte < 128:
+                return value
+        raise ValueError("LEB128 overflow")
+
+
+@dataclass
+class LinksPacket:
+    external_references: bool = False
+    sections: list = field(default_factory=list)
+
+    def links(self):
+        result, address = [], 1
+        for section in self.sections:
+            address = _u64(address + _u64(section.gap))
+            for link in section.links:
+                result.append((address, list(link)))
+                address = _u64(address + 1)
+        return result
+
+    def validate(self, limits=DecodeLimits()):
+        _require(len(self.sections) <= limits.max_links, "too many sections")
+        links = self.links()
+        _require(len(links) <= limits.max_links, "too many links")
+        references = 0
+        for section in self.sections:
+            _require(section.width in WIDTHS, "invalid reference width")
+            # Also validate mutable in-memory sections/packets.
+            ArityRange(section.arity.min, section.arity.max)
+            for link in section.links:
+                _require(section.arity.contains(len(link)), "link outside arity range")
+                references += len(link)
+                _require(references <= limits.max_references, "too many references")
+                for reference in link:
+                    _require(_width(reference, self.external_references) <= section.width, "reference outside width")
+        return self
+
+    def to_bytes(self, limits=DecodeLimits()):
+        self.validate(limits)
+        compact = not self.sections or (
+            len(self.sections) == 1
+            and self.sections[0].gap == 5
+            and self.sections[0].arity == ArityRange()
+            and bool(self.sections[0].links)
+        )
+        out = bytearray([0x10 | int(self.external_references)])
+        if compact:
+            width = self.sections[0].width if self.sections else 1
+            out[0] |= WIDTHS.index(width) << 2
+            _leb(out, sum(len(s.links) for s in self.sections))
+        else:
+            out[0] |= 2
+            _leb(out, len(self.sections))
+            for section in self.sections:
+                _leb(
+                    out,
+                    section.arity.min << 4
+                    | WIDTHS.index(section.width)
+                    | (4 if section.gap else 0)
+                    | (0 if section.arity.fixed else 8),
+                )
+                if section.gap:
+                    _leb(out, section.gap)
+                if not section.arity.fixed:
+                    _leb(out, 0 if section.arity.max is None else section.arity.max - section.arity.min)
+                _leb(out, len(section.links))
+        for section in self.sections:
+            for link in section.links:
+                if not section.arity.fixed:
+                    _leb(out, len(link) - section.arity.min)
+                for reference in link:
+                    if isinstance(reference, External):
+                        value = (
+                            (1 << (section.width * 8 - 1))
+                            if reference.value == 0
+                            else (1 << (section.width * 8)) - reference.value
+                        )
+                    else:
+                        value = reference
+                    out.extend(value.to_bytes(section.width, "little"))
+        return bytes(out)
+
+    def write_to(self, stream, limits=DecodeLimits()):
+        data = self.to_bytes(limits)
+        offset = 0
+        while offset < len(data):
+            written = stream.write(data[offset:])
+            _require(written is not None and written > 0, "stream did not write data")
+            offset += written
+
+    @classmethod
+    def read_from(cls, stream, limits=DecodeLimits()):
+        """Read exactly one packet; None only at a clean end of stream."""
+        first = stream.read(1)
+        if not first:
+            return None
+        header = first[0]
+        _require(header & 0xF0 == 0x10, "unsupported binary version")
+        reader, sections, counts = _Reader(stream), [], []
+        if header & 2:
+            _require(header & 12 == 0, "explicit header width bits set")
+            count = reader.leb()
+            _require(count <= limits.max_links, "too many sections")
+            address, total = 1, 0
+            for _ in range(count):
+                shape = reader.leb()
+                gap = reader.leb() if shape & 4 else 0
+                minimum = shape >> 4
+                extra = reader.leb() if shape & 8 else None
+                maximum = minimum if extra is None else (None if extra == 0 else _u64(minimum + extra))
+                arity = ArityRange(minimum, maximum)
+                count = reader.leb()
+                address = _u64(address + gap + count)
+                total += count
+                _require(total <= limits.max_links, "too many links")
+                sections.append(Section(gap, arity, WIDTHS[shape & 3], []))
+                counts.append(count)
+        else:
+            count = reader.leb()
+            _u64(6 + count)
+            _require(count <= limits.max_links, "too many links")
+            if count:
+                sections.append(Section(5, ArityRange(), WIDTHS[(header >> 2) & 3], []))
+                counts.append(count)
+        references = 0
+        for section, count in zip(sections, counts):
+            for _ in range(count):
+                length = section.arity.min + (0 if section.arity.fixed else reader.leb())
+                _require(section.arity.contains(_u64(length)), "link outside arity range")
+                references += length
+                _require(references <= limits.max_references, "too many references")
+                link = []
+                for _ in range(length):
+                    raw = reader.raw(section.width)
+                    top = 1 << (section.width * 8 - 1)
+                    link.append(
+                        External(0 if raw == top else (1 << (section.width * 8)) - raw)
+                        if header & 1 and raw >= top
+                        else raw
+                    )
+                section.links.append(link)
+        return cls(bool(header & 1), sections)
+
+    @classmethod
+    def from_bytes(cls, data, limits=DecodeLimits()):
+        stream = io.BytesIO(data)
+        packet = cls.read_from(stream, limits)
+        _require(packet is not None, "empty input")
+        _require(not stream.read(1), "trailing bytes")
+        return packet
+
+    @classmethod
+    def parse_links(cls, text):
+        links = []
+        for entry in text.split(";"):
+            if not entry.strip():
+                continue
+            address, refs = entry.split(":")
+            links.append(
+                (
+                    _u64(int(address)),
+                    [External(int(r[1:])) if r.startswith("#") else _u64(int(r)) for r in refs.split()],
+                )
+            )
+        return links
+
+    @classmethod
+    def pack(cls, external_references, links, packed_widths=False):
+        links = list(links)
+        needs, previous = [], 0
+        for address, refs in links:
+            _require(previous < _u64(address) < U64, "addresses must ascend and leave space for end")
+            _require(refs, "empty raw link")
+            previous = address
+            needs.append(max(_width(r, external_references) for r in refs))
+
+        def layout(packed):
+            if not links:
+                return []
+            widest, costs, opens, bests = max(needs), [float("inf")] * 8, [], []
+            for index, (address, refs) in enumerate(links):
+                best = min(range(8), key=lambda s: costs[s])
+                bests.append(best)
+                before = costs[best] if index else 0
+                next_costs, mask = [float("inf")] * 8, 0
+                for state in range(8):
+                    width, variable = WIDTHS[state // 2], state % 2
+                    if (not packed and width != widest) or width < needs[index]:
+                        continue
+                    opening = before + 2 + variable
+                    continuing = (
+                        costs[state]
+                        if index
+                        and links[index - 1][0] + 1 == address
+                        and (variable or len(links[index - 1][1]) == len(refs))
+                        else float("inf")
+                    )
+                    if continuing <= opening:
+                        next_costs[state] = continuing + len(refs) * width + variable
+                    else:
+                        next_costs[state] = opening + len(refs) * width + variable
+                        mask |= 1 << state
+                opens.append(mask)
+                costs = next_costs
+            result, end, state = [], len(links), min(range(8), key=lambda s: costs[s])
+            for index in range(len(links) - 1, -1, -1):
+                if opens[index] & (1 << state):
+                    result.append((end - index, WIDTHS[state // 2]))
+                    end, state = index, bests[index]
+            return list(reversed(result))
+
+        def packet(plan):
+            sections, index, address = [], 0, 1
+            for count, width in plan:
+                members = links[index : index + count]
+                lengths = [len(refs) for _, refs in members]
+                start = members[0][0]
+                sections.append(
+                    Section(
+                        start - address,
+                        ArityRange(min(lengths), max(lengths)),
+                        width,
+                        [list(refs) for _, refs in members],
+                    )
+                )
+                index, address = index + count, start + count
+            return cls(external_references, sections)
+
+        uniform = packet(layout(False))
+        if not packed_widths:
+            return uniform
+        packed = packet(layout(True))
+        return (
+            packed
+            if len(packed.to_bytes(DecodeLimits.unlimited())) < len(uniform.to_bytes(DecodeLimits.unlimited()))
+            else uniform
+        )
+
+
+class _Encoder:
+    def __init__(self, options):
+        self.options = options
+        self.doublets, self.tuples, self.created = [], [], {}
+        self.powers = [("i", 1)]
+
+    def link(self, items):
+        key = tuple(items)
+        if key not in self.created:
+            doublet = len(items) == 2 and all(kind != "t" for kind, _ in items)
+            target = self.doublets if doublet else self.tuples
+            node = ("d" if doublet else "t", len(target))
+            target.append(list(items))
+            self.created[key] = node
+        return self.created[key]
+
+    def chain(self, items):
+        tail = ("i", 0)
+        for head in reversed(items):
+            tail = self.link([head, tail])
+        return tail
+
+    def typed(self, marker, elements):
+        items = [("i", marker)] + elements
+        return (
+            self.link(items)
+            if len(items) != 2 and self.options.arity.contains(len(items))
+            else self.link([("i", marker), self.chain(elements)])
+        )
+
+    def list(self, elements):
+        if not elements:
+            return ("i", 0)
+        if len(elements) == 2 or self.options.arity.contains(len(elements)):
+            return self.link(elements)
+        return self.typed(4, elements)
+
+    def unary(self, value):
+        powers = []
+        for bit in range(63, -1, -1):
+            if value & (1 << bit):
+                while len(self.powers) <= bit:
+                    self.powers.append(self.link([self.powers[-1], self.powers[-1]]))
+                powers.append(self.powers[bit])
+        if not powers:
+            return ("i", 0)
+        result = powers[-1]
+        for power in reversed(powers[:-1]):
+            result = self.link([power, result])
+        return result
+
+    def scalar(self, value):
+        return ("e", value) if self.options.external_references and value < 1 << 63 else self.unary(value)
+
+    def reference(self, text):
+        if re.fullmatch(r"0|[1-9][0-9]*", text) and len(text) <= 20 and int(text) <= U64:
+            value = int(text)
+            if self.options.external_references and value < 1 << 63:
+                return ("e", value)
+            return self.link([("i", 2), self.unary(value)])
+        return self.typed(3, [self.scalar(ord(c)) for c in text])
+
+    def encode_identified(self, node):
+        if node.id is not None and node.values:
+            identifier = self.reference(node.id)
+            return self.typed(5, [identifier] + [self.encode_identified(v) for v in node.values])
+        if node.id is not None:
+            return self.reference(node.id)
+        return self.list([self.encode_identified(v) for v in node.values])
+
+    def finish(self):
+        def resolve(node):
+            kind, value = node
+            return (
+                External(value)
+                if kind == "e"
+                else (value if kind == "i" else 6 + value + (len(self.doublets) if kind == "t" else 0))
+            )
+
+        links = [(6 + i, [resolve(n) for n in nodes]) for i, nodes in enumerate(self.doublets + self.tuples)]
+        return LinksPacket.pack(self.options.external_references, links, self.options.packed_widths)
+
+
+class _Decoder:
+    def __init__(self, packet, limits):
+        packet.validate(limits)
+        self.limits = limits
+        self.links, self.unary = [], []
+        self.nodes, self.strings = limits.max_nodes, limits.max_string_bytes
+        for address, refs in packet.links():
+            _require(address == 6 + len(self.links), "document links must be contiguous from 6")
+            _require(all(isinstance(r, External) or r < address for r in refs), "document contains forward reference")
+
+            def value(r):
+                if isinstance(r, External):
+                    return None
+                return r if r <= 1 else (self.unary[r - 6] if r >= 6 else None)
+
+            a, b = (value(refs[0]), value(refs[1])) if len(refs) == 2 else (None, None)
+            self.unary.append(a + b if a is not None and b is not None and a + b <= U64 else None)
+            self.links.append(refs)
+
+    def chain(self, tail):
+        result = []
+        while tail != 0:
+            _require(isinstance(tail, int) and tail >= 6, "broken element chain")
+            items = self.links[tail - 6]
+            _require(len(items) == 2, "broken element chain")
+            _require(len(result) < self.limits.max_nodes, "chain too long")
+            result.append(items[0])
+            tail = items[1]
+        return result
+
+    def number(self, reference):
+        if isinstance(reference, External):
+            return reference.value
+        if reference <= 1:
+            return reference
+        value = self.unary[reference - 6] if reference >= 6 else None
+        _require(value is not None, "expected unary number")
+        return value
+
+    def text(self, text):
+        self.strings -= len(text.encode("utf-8"))
+        _require(self.strings >= 0, "too many expanded string bytes")
+        return Link(text)
+
+    def decode(self, reference, depth):
+        _require(depth < self.limits.max_depth, "nesting limit exceeded")
+        self.nodes -= 1
+        _require(self.nodes >= 0, "too many expanded nodes")
+        if isinstance(reference, External):
+            return self.text(str(reference.value))
+        if reference == 0:
+            return Link()
+        _require(reference >= 6, "standalone marker")
+        items = self.links[reference - 6]
+        if isinstance(items[0], int) and 1 <= items[0] <= 5:
+            marker = items[0]
+            elements = items[1:] if len(items) != 2 or marker == 2 else self.chain(items[1])
+            if marker == 2:
+                _require(len(elements) == 1, "number needs one value")
+                return self.text(str(self.number(elements[0])))
+            if marker == 3:
+                chars, byte_count = [], 0
+                for e in elements:
+                    p = self.number(e)
+                    _require(p <= 0x10FFFF and not 0xD800 <= p <= 0xDFFF, "invalid Unicode scalar")
+                    size = 1 if p <= 0x7F else 2 if p <= 0x7FF else 3 if p <= 0xFFFF else 4
+                    _require(size <= self.strings - byte_count, "too many expanded string bytes")
+                    byte_count += size
+                    chars.append(chr(p))
+                return self.text("".join(chars))
+            if marker == 5:
+                _require(elements, "identified link needs id")
+                identifier = self.decode(elements[0], depth)
+                _require(identifier.id is not None and not identifier.values, "id must be reference")
+                return Link(identifier.id, [self.decode(e, depth + 1) for e in elements[1:]])
+            _require(marker == 4, "invalid typed marker")
+        else:
+            elements = items
+        return Link(None, [self.decode(e, depth + 1) for e in elements])
+
+    def document(self):
+        if not self.links:
+            return []
+        root = self.links[-1]
+        if len(root) == 2 and root[0] == 4:
+            elements = self.chain(root[1])
+        else:
+            _require(not (isinstance(root[0], int) and 1 <= root[0] <= 5), "root must be list")
+            elements = root
+        return [self.decode(e, 0) for e in elements]
+
+
+def _canonical(node):
+    if node.id is None and len(node.values) == 1 and node.values[0].id is not None and not node.values[0].values:
+        return node.values[0]
+    return Link(node.id, [_canonical(v) for v in node.values])
+
+
+def format_document(document):
+    def nested(node, top=False):
+        if node.id is not None and not node.values:
+            return format_reference(node.id)
+        values = " ".join(nested(v) for v in node.values)
+        if node.id is not None:
+            return "(" + format_reference(node.id) + ": " + values + ")"
+        if len(node.values) == 1 and node.values[0].id is not None and not node.values[0].values:
+            return "((" + values + "))"
+        return values if top and len(node.values) >= 2 else "(" + values + ")"
+
+    return "\n".join(nested(n, True) for n in document)
+
+
+class BinaryLinoCodec:
+    def __init__(self, options=None, limits=None):
+        self.options = options or BinaryLinoOptions()
+        self.limits = limits or DecodeLimits()
+
+    def encode_packet(self, document):
+        _require(self.options.arity.contains(2), "arity must contain 2")
+        pending = [(node, 0) for node in document]
+        nodes, strings = 0, 0
+        while pending:
+            node, depth = pending.pop()
+            _require(isinstance(node, Link), "expected Link model")
+            _require(depth < self.limits.max_depth, "nesting limit exceeded")
+            nodes += 1 + int(node.id is not None and bool(node.values))
+            if node.id is not None:
+                _require(isinstance(node.id, str), "reference must be string")
+                strings += len(node.id.encode("utf-8"))
+            _require(nodes <= self.limits.max_nodes, "too many nodes")
+            _require(strings <= self.limits.max_string_bytes, "too many string bytes")
+            pending.extend((v, depth + 1) for v in node.values)
+        encoder = _Encoder(self.options)
+        if document:
+            encoder.list([encoder.encode_identified(node) for node in document])
+        return encoder.finish().validate(self.limits)
+
+    def encode(self, document):
+        return self.encode_packet(document).to_bytes(self.limits)
+
+    def decode_packet(self, packet):
+        return _Decoder(packet, self.limits).document()
+
+    def decode(self, data):
+        return self.decode_packet(LinksPacket.from_bytes(data, self.limits))
+
+    @staticmethod
+    def parse_document(text, parser=None):
+        return [_canonical(n) for n in (parser or Parser()).parse(text)]
+
+    @staticmethod
+    def format_document(document):
+        return format_document(document)
+
+    def encode_text(self, text, parser=None):
+        return self.encode(self.parse_document(text, parser))
+
+    def decode_text(self, data):
+        return format_document(self.decode(data))

{links_notation-0.22.0 → links_notation-0.24.0}/links_notation/link.py

@@ -4,6 +4,8 @@ Link class representing a Lino link with optional ID and values.
 
 from typing import TYPE_CHECKING, List, Optional, Union
 
+from .reference_literal import format_reference
+
 if TYPE_CHECKING:
     from .format_config import FormatConfig
 
@@ -92,24 +94,7 @@ class Link:
         if reference is None:
             return ""
 
-        # The empty reference is written as a bare delimiter pair, so that it
-        # reads back as itself instead of disappearing from the document.
-        if reference == "":
-            return '""'
-
-        # Check if single quotes are needed. A reference that begins with a "#"
-        # is quoted too, or it would read back as a comment; a "#" anywhere else
-        # is content ("issue#1047"), so only the first character matters.
-        needs_single_quotes = reference.startswith("#") or any(
-            c in reference for c in [":", "(", ")", " ", "\t", "\n", "\r", '"']
-        )
-
-        if needs_single_quotes:
-            return f"'{reference}'"
-        elif "'" in reference:
-            return f'"{reference}"'
-        else:
-            return reference
+        return format_reference(reference)
 
     def to_link_or_id_string(self) -> str:
         """Convert to string, using just ID if no values, otherwise full format."""

{links_notation-0.22.0 → links_notation-0.24.0}/links_notation/parser.py

@@ -11,6 +11,7 @@ from typing import Any, Dict, List, Optional, Tuple
 from .comments import strip_comments
 from .link import Link
 from .quotes import _parse_quoted_string_at
+from .reference_literal import decode_reference_literal
 
 #: How deep links may nest unless a parser is told otherwise: every
 #: parenthesized group and every indentation level is one level, and the lines
@@ -612,6 +613,9 @@ class Parser:
         """Extract reference, handling quoted strings with escaping support."""
         text = text.strip()
 
+        if re.match(r"~[0-9]+\{", text):
+            return decode_reference_literal(text)
+
         # Try delimited references (any N quotes, or a bare delimiter pair)
         quoted = _parse_quoted_string_at(text, 0)
         if quoted is not None:
@@ -645,7 +649,7 @@ class Parser:
         children = item.get("children", [])
 
         # Special case: indented ID syntax (id: followed by children)
-        if item.get("is_indented_id") and item.get("id") and not item.get("values") and children:
+        if item.get("is_indented_id") and item.get("id") is not None and not item.get("values") and children:
             child_values = [self._transform_indented_value(child) for child in children]
 
             link_with_children = {"id": item["id"], "values": child_values}

links_notation-0.24.0/links_notation/reference_literal.py

@@ -0,0 +1,38 @@
+"""Version 1 lossless Unicode references, with readable legacy quoting."""
+
+import re
+
+
+def encode_reference_literal(text: str) -> str:
+    """Encode exact text as a versioned UTF-8 hexadecimal literal."""
+    return "~1{" + text.encode("utf-8").hex() + "}"
+
+
+def decode_reference_literal(literal: str) -> str:
+    """Decode a complete literal; reject unsupported versions and invalid UTF-8."""
+    match = re.fullmatch(r"~([0-9]+)\{([0-9a-fA-F]*)\}", literal)
+    if not match or match[1] != "1" or len(match[2]) % 2:
+        raise ValueError("Invalid or unsupported reference literal (expected ~1{UTF-8 hex})")
+    return bytes.fromhex(match[2]).decode("utf-8")
+
+
+def format_reference(text: str) -> str:
+    """Quote readable text, using a literal for empty strings and C0 controls."""
+    text.encode("utf-8")  # Reject unpaired surrogates before formatting.
+    if not text or any(ord(c) < 32 or ord(c) == 127 for c in text):
+        return encode_reference_literal(text)
+    if (
+        not text.startswith("#")
+        and not re.match(r"~[0-9]+\{", text)
+        and not any(c.isspace() or c in "\ufeff():\"'`" for c in text)
+    ):
+        return text
+    choices = []
+    for quote in "'\"`":
+        if text.startswith(quote):
+            continue
+        longest = max([len(m.group()) for m in re.finditer(re.escape(quote) + "+", text)] or [0])
+        choices.append(((longest + 1) | 1, quote))
+    count, quote = min(choices, key=lambda item: item[0])
+    delimiter = quote * count
+    return delimiter + text + delimiter

{links_notation-0.22.0 → links_notation-0.24.0/links_notation.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: links-notation
-Version: 0.22.0
+Version: 0.24.0
 Summary: Python implementation of the Links Notation parser
 Author-email: "Link.Foundation" <drakonard@gmail.com>
 License-Expression: Unlicense
@@ -22,18 +22,18 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Provides-Extra: test
-Requires-Dist: pytest>=9.1; extra == "test"
-Requires-Dist: pytest-timeout>=2.4; extra == "test"
+Requires-Dist: pytest>=9.1.1; extra == "test"
+Requires-Dist: pytest-timeout>=2.4.0; extra == "test"
 Provides-Extra: lint
-Requires-Dist: black>=26.5; extra == "lint"
-Requires-Dist: isort>=9.0; extra == "lint"
-Requires-Dist: flake8>=7.3; extra == "lint"
+Requires-Dist: black>=26.10.0; extra == "lint"
+Requires-Dist: isort>=9.0.2; extra == "lint"
+Requires-Dist: flake8>=7.4.1; extra == "lint"
 Provides-Extra: dev
-Requires-Dist: pytest>=9.1; extra == "dev"
-Requires-Dist: pytest-timeout>=2.4; extra == "dev"
-Requires-Dist: black>=26.5; extra == "dev"
-Requires-Dist: isort>=9.0; extra == "dev"
-Requires-Dist: flake8>=7.3; extra == "dev"
+Requires-Dist: pytest>=9.1.1; extra == "dev"
+Requires-Dist: pytest-timeout>=2.4.0; extra == "dev"
+Requires-Dist: black>=26.10.0; extra == "dev"
+Requires-Dist: isort>=9.0.2; extra == "dev"
+Requires-Dist: flake8>=7.4.1; extra == "dev"
 Dynamic: license-file
 
 # Links Notation Parser for Python
@@ -306,7 +306,7 @@ python -m build
 
 This project uses [Black](https://github.com/psf/black) for code formatting,
 [isort](https://pycqa.github.io/isort/) for import sorting, and
-[flake8](https://flake8.pycqa.org/) for linting.
+[flake8](https://flake8.pycqa.org/en/latest/) for linting.
 
 Install linting tools:
 
@@ -354,3 +354,6 @@ This project is released into the public domain under the [Unlicense](../LICENSE
 - [Main Repository](https://github.com/link-foundation/links-notation)
 - [PyPI Package](https://pypi.org/project/links-notation/)
 - [Documentation](https://link-foundation.github.io/links-notation/)
+
+Exact Unicode identifiers and values round-trip with the [reference literal contract](../docs/protocol/reference-literals.md)
+introduced in 0.24.0, including mixed quotes, empty strings, controls and multiline text.

{links_notation-0.22.0 → links_notation-0.24.0}/links_notation.egg-info/SOURCES.txt

@@ -5,12 +5,14 @@ README.ru.md
 pyproject.toml
 setup.cfg
 links_notation/__init__.py
+links_notation/binary.py
 links_notation/comments.py
 links_notation/format_config.py
 links_notation/formatter.py
 links_notation/link.py
 links_notation/parser.py
 links_notation/quotes.py
+links_notation/reference_literal.py
 links_notation/stream_parser.py
 links_notation.egg-info/PKG-INFO
 links_notation.egg-info/SOURCES.txt
@@ -18,6 +20,7 @@ links_notation.egg-info/dependency_links.txt
 links_notation.egg-info/requires.txt
 links_notation.egg-info/top_level.txt
 tests/test_api.py
+tests/test_binary.py
 tests/test_comments.py
 tests/test_edge_case_parser.py
 tests/test_empty_reference.py
@@ -35,5 +38,6 @@ tests/test_nested_indentation.py
 tests/test_nested_parser.py
 tests/test_nested_self_reference.py
 tests/test_nesting_limit.py
+tests/test_reference_fidelity.py
 tests/test_single_line_parser.py
 tests/test_stream_parser.py

links_notation-0.24.0/links_notation.egg-info/requires.txt

@@ -0,0 +1,16 @@
+
+[dev]
+pytest>=9.1.1
+pytest-timeout>=2.4.0
+black>=26.10.0
+isort>=9.0.2
+flake8>=7.4.1
+
+[lint]
+black>=26.10.0
+isort>=9.0.2
+flake8>=7.4.1
+
+[test]
+pytest>=9.1.1
+pytest-timeout>=2.4.0

{links_notation-0.22.0 → links_notation-0.24.0}/pyproject.toml

@@ -1,10 +1,13 @@
 [build-system]
-requires = ["setuptools>=77.0"]
+requires = [
+    "setuptools>=84.0.0; python_version >= '3.10'",
+    "setuptools>=82.0.1; python_version < '3.10'",
+]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "links-notation"
-version = "0.22.0"
+version = "0.24.0"
 description = "Python implementation of the Links Notation parser"
 readme = "README.md"
 license = "Unlicense"
@@ -29,9 +32,9 @@ requires-python = ">=3.9"
 dependencies = []
 
 [project.optional-dependencies]
-test = ["pytest>=9.1", "pytest-timeout>=2.4"]
-lint = ["black>=26.5", "isort>=9.0", "flake8>=7.3"]
-dev = ["pytest>=9.1", "pytest-timeout>=2.4", "black>=26.5", "isort>=9.0", "flake8>=7.3"]
+test = ["pytest>=9.1.1", "pytest-timeout>=2.4.0"]
+lint = ["black>=26.10.0", "isort>=9.0.2", "flake8>=7.4.1"]
+dev = ["pytest>=9.1.1", "pytest-timeout>=2.4.0", "black>=26.10.0", "isort>=9.0.2", "flake8>=7.4.1"]
 
 [project.urls]
 Homepage = "https://github.com/link-foundation/links-notation"

links_notation-0.24.0/tests/test_binary.py

@@ -0,0 +1,107 @@
+"""Shared binary vectors and native text-model parity (issue 325)."""
+
+from pathlib import Path
+
+import pytest
+
+from links_notation import Link, Parser
+from links_notation.binary import ArityRange, BinaryLinoCodec, BinaryLinoOptions, DecodeLimits, LinksPacket
+
+
+def test_string_budget_stops_decoding_before_later_scalars():
+    data = bytes.fromhex("13024605011001030000009fffffff9effffff0028ffff06")
+    with pytest.raises(ValueError, match="string bytes"):
+        BinaryLinoCodec(limits=DecodeLimits(max_string_bytes=1)).decode(data)
+
+
+def test_shared_vectors():
+    count = 0
+    for line in (
+        (Path(__file__).parents[2] / "docs/protocol/binary-links-notation-vectors.txt").read_text().splitlines()
+    ):
+        if not line or line.startswith("#"):
+            continue
+        fields = line.split("\t")
+        expected = bytes.fromhex(fields[-1])
+        if fields[0] == "document":
+            codec = BinaryLinoCodec(
+                BinaryLinoOptions(fields[2] == "external", ArityRange.parse(fields[3]), fields[4] == "packed")
+            )
+            document = codec.parse_document(fields[1].replace("\\n", "\n"))
+            assert codec.encode(document) == expected, line
+            assert codec.decode(expected) == document, line
+        else:
+            links = LinksPacket.parse_links(fields[1])
+            packet = LinksPacket.pack(fields[2] == "external", links, fields[3] == "packed")
+            assert packet.to_bytes() == expected, line
+            assert LinksPacket.from_bytes(expected).links() == links, line
+        count += 1
+    assert count == 96
+
+
+@pytest.mark.parametrize(
+    "source",
+    [
+        "",
+        "()",
+        '""',
+        "a",
+        "(a)",
+        "((a))",
+        "(a b c)",
+        '("": () "" (b))',
+        "a:\n  b\n  c",
+        '"#x" "λ😀" "007" "18446744073709551615"',
+        "# comment\n(a # tail\n b)",
+    ],
+)
+def test_native_text_model_round_trip(source):
+    document = Parser().parse(source)
+    for external in (False, True):
+        for arity in ("2", "2..3", "1.."):
+            for packed in (False, True):
+                codec = BinaryLinoCodec(BinaryLinoOptions(external, ArityRange.parse(arity), packed))
+                assert codec.decode(codec.encode(document)) == document
+
+
+def test_limits_and_malformed_packets():
+    document = [Link(None, [Link("abcdef")])]
+    bytes_ = BinaryLinoCodec().encode(document)
+    for limits in (
+        DecodeLimits(max_nodes=1),
+        DecodeLimits(max_string_bytes=2),
+        DecodeLimits(max_depth=1),
+        DecodeLimits(max_links=1),
+        DecodeLimits(max_references=1),
+    ):
+        with pytest.raises(ValueError):
+            BinaryLinoCodec(limits=limits).decode(bytes_)
+        with pytest.raises(ValueError):
+            BinaryLinoCodec(limits=limits).encode(document)
+    for bytes_ in (b"", b"\x20", b"\x1e\x00", b"\x10\x01", b"\x10\x00\x00", b"\x10" + b"\xff" * 10):
+        with pytest.raises(ValueError):
+            BinaryLinoCodec().decode(bytes_)
+
+
+def test_packet_stream_boundaries_uint64_and_every_truncated_prefix():
+    from io import BytesIO
+
+    for external, references in [(False, "0 18446744073709551615"), (True, "#0 #9223372036854775807")]:
+        links = LinksPacket.parse_links("6:" + references)
+        packet = LinksPacket.pack(external, links, True)
+        data = packet.to_bytes()
+        stream = BytesIO(data + data)
+        assert LinksPacket.read_from(stream).links() == links
+        assert stream.tell() == len(data)
+        assert LinksPacket.read_from(stream).links() == links
+        assert LinksPacket.read_from(stream) is None
+        for end in range(len(data)):
+            with pytest.raises(ValueError):
+                LinksPacket.from_bytes(data[:end])
+    document = [Link("leaf")]
+    for _ in range(70):
+        document = [Link(None, document)]
+    with pytest.raises(ValueError):
+        BinaryLinoCodec().encode(document)
+    codec = BinaryLinoCodec(limits=DecodeLimits(max_depth=80))
+    assert codec.decode(codec.encode(document)) == document

{links_notation-0.22.0 → links_notation-0.24.0}/tests/test_empty_reference.py

@@ -76,8 +76,8 @@ class TestNQuoteMeaningsSurvive:
 class TestEmptyReferenceFormatting:
     """The empty reference is written so that it reads back as itself."""
 
-    def test_empty_reference_is_written_as_a_delimiter_pair(self):
-        assert format_links(Parser().parse('(a "" b)')) == '(a "" b)'
+    def test_empty_reference_is_written_as_a_versioned_literal(self):
+        assert format_links(Parser().parse('(a "" b)')) == "(a ~1{} b)"
 
     def test_empty_reference_survives_a_round_trip(self):
         for source in [

links_notation-0.24.0/tests/test_reference_fidelity.py

@@ -0,0 +1,73 @@
+from pathlib import Path
+
+import pytest
+
+from links_notation import (
+    FormatConfig,
+    Link,
+    Parser,
+    StreamParser,
+    decode_reference_literal,
+    encode_reference_literal,
+    format_binary_reference,
+    format_links,
+)
+from links_notation.binary import BinaryLinoCodec
+
+REFERENCES = [
+    "" if line == "-" else bytes.fromhex(line).decode("utf-8")
+    for line in (Path(__file__).resolve().parents[2] / "docs/protocol/reference-literals.txt").read_text().splitlines()
+    if not line.startswith("#")
+]
+
+
+def test_native_reference_fidelity():
+    for reference in REFERENCES:
+        original = [Link(reference, [Link(reference), Link("nested", [Link(reference)])])]
+        for less_parentheses in (False, True, FormatConfig(max_inline_refs=1, prefer_inline=False)):
+            assert Parser().parse(format_links(original, less_parentheses)) == original
+
+
+def test_reference_literals_at_every_chunk_split():
+    for reference in REFERENCES:
+        assert decode_reference_literal(encode_reference_literal(reference)) == reference
+        for literal in (
+            Link.escape_reference(reference),
+            format_binary_reference(reference),
+            encode_reference_literal(reference),
+        ):
+            text = f"(root: {literal})\n({literal}: fixture)"
+            expected = [Link("root", [Link(reference)]), Link(reference, [Link("fixture")])]
+            assert Parser().parse(text) == expected
+            for split in range(len(text) + 1):
+                stream = StreamParser()
+                stream.write(text[:split])
+                stream.write(text[split:])
+                assert stream.finish() == expected
+
+
+def test_binary_reference_fidelity():
+    codec = BinaryLinoCodec()
+    for reference in REFERENCES:
+        original = [Link(reference, [Link(reference)])]
+        decoded = codec.decode(codec.encode(original))
+        assert decoded == original
+        assert codec.parse_document(codec.format_document(decoded)) == original
+
+
+def test_malformed_reference_literals_are_rejected():
+    invalid = (
+        (Path(__file__).resolve().parents[2] / "docs/protocol/invalid-reference-literals.txt").read_text().splitlines()
+    )
+    for literal in invalid:
+        if literal.startswith("#"):
+            continue
+        with pytest.raises(ValueError):
+            decode_reference_literal(literal)
+        with pytest.raises(Exception):
+            Parser().parse(literal)
+    assert decode_reference_literal("~1{C3A9}") == "é"
+    with pytest.raises(UnicodeEncodeError):
+        encode_reference_literal("\ud800")
+    with pytest.raises(UnicodeEncodeError):
+        Link.escape_reference("\ud800")

links_notation-0.22.0/links_notation.egg-info/requires.txt

@@ -1,16 +0,0 @@
-
-[dev]
-pytest>=9.1
-pytest-timeout>=2.4
-black>=26.5
-isort>=9.0
-flake8>=7.3
-
-[lint]
-black>=26.5
-isort>=9.0
-flake8>=7.3
-
-[test]
-pytest>=9.1
-pytest-timeout>=2.4