korml 0.1.0__py3-none-any.whl

korml/__init__.py

@@ -0,0 +1,45 @@
+# ------------------------------------------------------------------------------
+# SPDX-License-Identifier: Apache-2.0
+# ------------------------------------------------------------------------------
+
+import korml.core as core
+import korml.error as error
+import korml.schema as schema
+from korml.api import dump, dumps, edit, load, loads
+from korml.error import KormlError, LexError, ParseError
+from korml.model import (
+    CST,
+    ConcreteToken,
+    GenericToken,
+    Mapping,
+    Scalar,
+    Sequence,
+    Stream,
+    Style,
+    Value,
+    Version,
+)
+
+__all__ = [
+    "core",
+    "error",
+    "schema",
+    "Version",
+    "Style",
+    "GenericToken",
+    "ConcreteToken",
+    "CST",
+    "Value",
+    "Scalar",
+    "Sequence",
+    "Mapping",
+    "Stream",
+    "KormlError",
+    "LexError",
+    "ParseError",
+    "dump",
+    "dumps",
+    "edit",
+    "load",
+    "loads",
+]

korml/_atomic.py

@@ -0,0 +1,84 @@
+# ------------------------------------------------------------------------------
+# SPDX-License-Identifier: Apache-2.0
+# ------------------------------------------------------------------------------
+
+"""Atomic output replacement, without following final-component symlinks."""
+
+import os
+import pathlib
+import secrets
+import stat
+import tempfile
+
+
+def write_bytes(path: str | pathlib.Path, data: bytes) -> None:
+    """Write a complete new file and atomically replace *path*.
+
+    On POSIX, directory-relative operations pin the parent directory during
+    the write and rename. A symlinked parent directory is supported, but a
+    final-component output symlink is replaced rather than followed. On other
+    platforms, atomic replace is used without directory-relative operations.
+    """
+    destination = pathlib.Path(path)
+    if os.name == "posix":
+        _write_posix(destination, data)
+    else:
+        _write_portable(destination, data)
+
+
+def _write_posix(destination: pathlib.Path, data: bytes) -> None:
+    parent_fd = os.open(destination.parent, os.O_RDONLY | os.O_DIRECTORY)
+    temporary_name = f".{destination.name}.{secrets.token_hex(12)}.tmp"
+    try:
+        fd = os.open(
+            temporary_name, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600, dir_fd=parent_fd
+        )
+        try:
+            with os.fdopen(fd, "wb") as stream:
+                try:
+                    previous = os.stat(
+                        destination.name, dir_fd=parent_fd, follow_symlinks=False
+                    )
+                except FileNotFoundError:
+                    previous = None
+                if previous is not None and stat.S_ISREG(previous.st_mode):
+                    os.fchmod(stream.fileno(), stat.S_IMODE(previous.st_mode))
+                stream.write(data)
+                stream.flush()
+                os.fsync(stream.fileno())
+            os.replace(
+                temporary_name,
+                destination.name,
+                src_dir_fd=parent_fd,
+                dst_dir_fd=parent_fd,
+            )
+            os.fsync(parent_fd)
+        finally:
+            try:
+                os.unlink(temporary_name, dir_fd=parent_fd)
+            except FileNotFoundError:
+                pass
+    finally:
+        os.close(parent_fd)
+
+
+def _write_portable(destination: pathlib.Path, data: bytes) -> None:
+    # Non-POSIX fallback: secure temporary creation and atomic replacement.
+    fd, temporary = tempfile.mkstemp(
+        prefix=f".{destination.name}.", suffix=".tmp", dir=destination.parent
+    )
+    try:
+        with os.fdopen(fd, "wb") as stream:
+            try:
+                previous = destination.lstat()
+            except FileNotFoundError:
+                previous = None
+            if previous is not None and stat.S_ISREG(previous.st_mode) and hasattr(os, "fchmod"):
+                os.fchmod(stream.fileno(), stat.S_IMODE(previous.st_mode))
+            stream.write(data)
+            stream.flush()
+            os.fsync(stream.fileno())
+        os.replace(temporary, destination)
+    finally:
+        if os.path.exists(temporary):
+            os.unlink(temporary)

korml/_limits.py

@@ -0,0 +1,23 @@
+# ------------------------------------------------------------------------------
+# SPDX-License-Identifier: Apache-2.0
+# ------------------------------------------------------------------------------
+
+"""Default resource limits for processing untrusted documents."""
+
+MAX_INPUT_BYTES = 8 * 1024 * 1024
+MAX_TOKENS = 100_000
+MAX_PARSE_WORK = 1_000_000
+
+
+def check_input_size(text: str, *, max_input_bytes: int | None, filename: str) -> None:
+    import korml.error
+
+    if max_input_bytes is not None and (
+        len(text) > max_input_bytes
+        or len(text.encode("utf-8", errors="surrogatepass")) > max_input_bytes
+    ):
+        raise korml.error.CompilerError(
+            message=f"input exceeds {max_input_bytes} bytes",
+            source_name=filename,
+            code="INPUT_TOO_LARGE",
+        )

korml/api.py

@@ -0,0 +1,292 @@
+# ------------------------------------------------------------------------------
+# SPDX-License-Identifier: Apache-2.0
+# ------------------------------------------------------------------------------
+
+import collections.abc
+import copy
+import pathlib
+import typing
+
+import korml._atomic
+import korml._limits
+import korml.emitter.v0
+import korml.emitter.v1
+import korml.error
+import korml.lexer.file
+import korml.lexer.v1
+import korml.model
+import korml.parser.v0
+import korml.parser.v1
+
+__all__ = ["dump", "dumps", "edit", "load", "loads"]
+
+
+def loads(
+    input_text: str,
+    *,
+    filename: str = "<string>",
+    debug: bool = False,
+    max_input_bytes: int | None = korml._limits.MAX_INPUT_BYTES,
+    max_tokens: int | None = korml._limits.MAX_TOKENS,
+    max_parse_work: int | None = korml._limits.MAX_PARSE_WORK,
+) -> korml.model.Stream[korml.lexer.v1.Token]:
+    korml._limits.check_input_size(input_text, max_input_bytes=max_input_bytes, filename=filename)
+    # A byte-order mark is an artefact of the encoding, not Korml text; it is
+    # dropped as YAML does, so a file saved with one reads like one without.
+    file = korml.lexer.file.lex(
+        filename=filename,
+        input_text=input_text.removeprefix("\ufeff"),
+        debug=debug,
+    )
+
+    if file.version is korml.model.Version.v0:
+        return korml.parser.v0.parse(file, max_tokens=max_tokens, max_work=max_parse_work)
+
+    if file.version is korml.model.Version.v1:
+        return korml.parser.v1.parse(file, max_tokens=max_tokens, max_work=max_parse_work)
+
+    raise ValueError(f"unsupported korml version: {file.version.value!r}")
+
+
+def load(
+    filename: str | pathlib.Path,
+    *,
+    debug: bool = False,
+    max_input_bytes: int | None = korml._limits.MAX_INPUT_BYTES,
+    max_tokens: int | None = korml._limits.MAX_TOKENS,
+    max_parse_work: int | None = korml._limits.MAX_PARSE_WORK,
+) -> korml.model.Stream[korml.lexer.v1.Token]:
+    path = pathlib.Path(filename)
+    with path.open("rb") as stream:
+        data = stream.read(max_input_bytes + 1 if max_input_bytes is not None else -1)
+    if max_input_bytes is not None and len(data) > max_input_bytes:
+        raise korml.error.CompilerError(
+            message=f"input exceeds {max_input_bytes} bytes",
+            source_name=str(path),
+            code="INPUT_TOO_LARGE",
+        )
+    return loads(
+        data.decode("utf-8"),
+        filename=str(path),
+        debug=debug,
+        max_input_bytes=max_input_bytes,
+        max_tokens=max_tokens,
+        max_parse_work=max_parse_work,
+    )
+
+
+def dumps(
+    value: object,
+    *,
+    version: korml.model.Version = korml.model.Version.v1,
+) -> str:
+    if version is korml.model.Version.v0:
+        return korml.emitter.v0.emit(_make_stream(value, version=version))
+
+    if version is korml.model.Version.v1:
+        return korml.emitter.v1.emit(_make_stream(value, version=version))
+
+    raise ValueError(f"unsupported korml version: {version.value!r}")
+
+
+def dump(
+    value: object,
+    filename: str | pathlib.Path,
+    *,
+    version: korml.model.Version = korml.model.Version.v1,
+) -> None:
+    korml._atomic.write_bytes(filename, dumps(value, version=version).encode("utf-8"))
+
+
+def _make_stream(
+    value: object,
+    *,
+    version: korml.model.Version,
+) -> korml.model.Stream[korml.lexer.v1.Token]:
+    if isinstance(value, korml.model.Stream):
+        return value
+
+    if isinstance(value, korml.model.Value):
+        return korml.model.Stream[korml.lexer.v1.Token](
+            document_array=[value],
+            cst=korml.model.CST(version=version),
+        )
+
+    return korml.model.Stream[korml.lexer.v1.Token](
+        document_array=[_make_value(value, version=version)],
+        cst=korml.model.CST(version=version),
+    )
+
+
+def _make_value(
+    value: object,
+    *,
+    version: korml.model.Version,
+) -> korml.model.Value[korml.lexer.v1.Token]:
+    if isinstance(value, str):
+        return korml.model.Scalar[korml.lexer.v1.Token](
+            value,
+            cst=korml.model.CST(version=version),
+        )
+
+    if isinstance(value, list):
+        return korml.model.Sequence[korml.lexer.v1.Token](
+            value_array=[_make_value(child, version=version) for child in value],
+            cst=korml.model.CST(version=version),
+        )
+
+    if isinstance(value, dict):
+        mapping = korml.model.Mapping[korml.lexer.v1.Token](
+            cst=korml.model.CST(version=version),
+        )
+
+        for key, child in value.items():
+            if not isinstance(key, str):
+                raise TypeError("mapping keys must be strings")
+
+            mapping.add_item(key, _make_value(child, version=version))
+
+        return mapping
+
+    raise TypeError(f"cannot dump {type(value).__name__!r}")
+
+
+_TRIVIA_SET = {
+    korml.lexer.v1.TokenType.WHITESPACE,
+    korml.lexer.v1.TokenType.COMMENT,
+    korml.lexer.v1.TokenType.NEWLINE,
+}
+
+_BLOCK_SCALAR_SET = {
+    korml.lexer.v1.TokenType.LITERAL_BLOCK_SCALAR,
+    korml.lexer.v1.TokenType.FOLDED_BLOCK_SCALAR,
+}
+
+
+def edit(
+    *,
+    stream: korml.model.Stream[korml.lexer.v1.Token],
+    replacements: collections.abc.Sequence[tuple[korml.model.Value[korml.lexer.v1.Token], str]],
+) -> str:
+    """
+    Return the source of a parsed stream with some values replaced by strings.
+
+    Each replacement pairs a value of the stream, as returned by loads or
+    load, with the string it becomes. Only the tokens of the replaced values
+    change; every comment, blank line, indentation and quoting style
+    elsewhere is kept, so a tool can change a value without reformatting the
+    file. A replaced value can be a scalar or a whole collection; the string
+    is always written as a double-quoted scalar, and a replaced block
+    collection becomes a scalar on the line of its key or dash.
+
+    The result is parsed again and must equal the original with the
+    replacements applied; anything else is a BugError, so an edit can never
+    change more than it says.
+    """
+    token_array = stream.token_array
+    index_map = {id(token): index for index, token in enumerate(token_array)}
+    path_map = _value_path_map(stream)
+    splice_array: list[tuple[int, int, str]] = []
+    path_array: list[tuple[tuple[int | str, ...], str]] = []
+
+    for value, text in replacements:
+        if not isinstance(text, str):
+            raise TypeError("replacement text must be a string")
+
+        path = path_map.get(id(value))
+        if path is None:
+            raise ValueError("replacement value is not a value of this stream")
+
+        splice_array.append(_splice_for(value, text, index_map=index_map))
+        path_array.append((path, text))
+
+    splice_array.sort()
+    for (_, previous_end, _), (start, _, _) in zip(splice_array, splice_array[1:], strict=False):
+        if start < previous_end:
+            raise ValueError("replacement values overlap")
+
+    expected = copy.deepcopy(stream.to_builtin())
+    for path, text in path_array:
+        _set_path(expected, path, text)
+
+    piece_array: list[str] = []
+    cursor = 0
+    for start, end, text in splice_array:
+        piece_array.extend(token.value for token in token_array[cursor:start])
+        piece_array.append(text)
+        cursor = end
+    piece_array.extend(token.value for token in token_array[cursor:])
+    source = "".join(piece_array)
+
+    if loads(source, filename=stream.filename).to_builtin() != expected:
+        raise korml.error.BugError(
+            message="edited source does not read back as the edited values",
+            location="korml.api.edit",
+            source_name=stream.filename,
+        )
+
+    return source
+
+
+def _splice_for(
+    value: korml.model.Value[korml.lexer.v1.Token],
+    text: str,
+    *,
+    index_map: dict[int, int],
+) -> tuple[int, int, str]:
+    """Return the token range a replaced value covers and the text it becomes."""
+    content_array = [token for token in value.token_array if token.type not in _TRIVIA_SET]
+    if not content_array:
+        raise ValueError("replacement value has no tokens")
+
+    start = index_map[id(content_array[0])]
+    end = index_map[id(content_array[-1])] + 1
+    scalar = korml.emitter.v1._format_double_quoted_scalar(text)
+
+    # A block scalar token includes the line break that ends its last line;
+    # the quoted scalar replacing it keeps that line break.
+    if content_array[-1].type in _BLOCK_SCALAR_SET:
+        scalar += _final_line_break(content_array[-1].value)
+
+    # A block collection starts on the line after its key or dash; the scalar
+    # replacing it goes on that line, after one space.
+    block_collection = (
+        isinstance(value, (korml.model.Mapping, korml.model.Sequence))
+        and value.style is korml.model.Style.block
+    )
+    if block_collection and value.token_array:
+        return index_map[id(value.token_array[0])], end, " " + scalar
+
+    return start, end, scalar
+
+
+def _final_line_break(value: str) -> str:
+    for line_break in ("\r\n", "\n", "\r"):
+        if value.endswith(line_break):
+            return line_break
+    return ""
+
+
+def _value_path_map(
+    stream: korml.model.Stream[korml.lexer.v1.Token],
+) -> dict[int, tuple[int | str, ...]]:
+    """Map the identity of every value of a stream to its path."""
+    path_map: dict[int, tuple[int | str, ...]] = {}
+    pending: list[tuple[korml.model.Value[korml.lexer.v1.Token], tuple[int | str, ...]]] = [
+        (document, (index,)) for index, document in enumerate(stream.document_array)
+    ]
+    while pending:
+        value, path = pending.pop()
+        path_map[id(value)] = path
+        if isinstance(value, korml.model.Mapping):
+            pending.extend((child, (*path, key)) for key, child in value.value_map.items())
+        elif isinstance(value, korml.model.Sequence):
+            pending.extend((child, (*path, index)) for index, child in enumerate(value))
+    return path_map
+
+
+def _set_path(builtin: list[object], path: tuple[int | str, ...], text: str) -> None:
+    container: typing.Any = builtin
+    for part in path[:-1]:
+        container = container[part]
+    container[path[-1]] = text

korml/cli/__init__.py

@@ -0,0 +1,80 @@
+# --------------------------------------------------------------------------------------------------
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2026 Jayesh Badwaik <j.badwaik@fz-juelich.de>
+# --------------------------------------------------------------------------------------------------
+
+import argparse
+import pathlib
+import sys
+
+import korml.cli.bugreport
+import korml.cli.lex
+import korml.error
+
+
+def first_level_lexer_parser(sp: argparse.ArgumentParser) -> None:
+    sp.add_argument(
+        "--input",
+        required=True,
+        type=pathlib.Path,
+        help="Input korml document",
+    )
+    sp.add_argument(
+        "--output",
+        required=False,
+        type=pathlib.Path,
+        default=None,
+        help="Output file; standard output if omitted",
+    )
+    sp.add_argument(
+        "--output-format",
+        required=False,
+        choices=["json", "text"],
+        default="json",
+    )
+    sp.set_defaults(func=korml.cli.lex.run)
+
+
+def top_level_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="korml",
+        description="korml command line tools",
+        epilog="Use 'korml -h' for more information",
+    )
+
+    subparser = parser.add_subparsers(title="subcommand", dest="subcommand", required=True)
+
+    lex_sp = subparser.add_parser("lex", help="Lex a korml document and print the tokens")
+    first_level_lexer_parser(lex_sp)
+
+    return parser
+
+
+def main() -> int:
+    parser = top_level_parser()
+    args = parser.parse_args(None if len(sys.argv) > 1 else ["-h"])
+
+    try:
+        args.func(args)
+        return 0
+    except KeyboardInterrupt:
+        print("error: interrupted by user", file=sys.stderr)
+        return 130
+    except korml.error.BugError as exc:
+        return handle_uncaught_exception(exc)
+    except korml.error.CompilerError as exc:
+        print(str(exc), file=sys.stderr)
+        return 1
+    except OSError as exc:
+        filename = korml.error.escape_controls(str(exc.filename))
+        reason = korml.error.escape_controls(str(exc.strerror))
+        print(f"error: {filename}: {reason}", file=sys.stderr)
+        return 1
+    except Exception as exc:
+        return handle_uncaught_exception(exc)
+
+
+def handle_uncaught_exception(exc: Exception) -> int:
+    report = korml.cli.bugreport.render_internal_failure_report(exc=exc, stream=sys.stderr)
+    print(report, file=sys.stderr)
+    return korml.cli.bugreport.DiagnosticExitCode.INTERNAL_ERROR

korml/cli/bugreport.py

@@ -0,0 +1,147 @@
+# --------------------------------------------------------------------------------------------------
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2026 Jayesh Badwaik <j.badwaik@fz-juelich.de>
+# --------------------------------------------------------------------------------------------------
+
+import dataclasses
+import enum
+import importlib.metadata
+import traceback
+import typing
+
+
+class AnsiCode(enum.Enum):
+    RESET = "\033[0m"
+    BOLD = "\033[1m"
+    RED = "\033[31m"
+    YELLOW = "\033[33m"
+    CYAN = "\033[36m"
+
+
+class DiagnosticExitCode(enum.IntEnum):
+    # sysexits.h: an internal software error. argparse uses 2 for invalid
+    # usage, so an internal error needs another status.
+    INTERNAL_ERROR = 70
+
+
+@dataclasses.dataclass(frozen=True)
+class ColorPolicy:
+    enabled: bool
+
+    @classmethod
+    def from_stream(cls, stream: typing.Any) -> "ColorPolicy":
+        enabled = hasattr(stream, "isatty") and stream.isatty()
+        return cls(enabled=enabled)
+
+
+@dataclasses.dataclass(frozen=True)
+class TextStyler:
+    color_policy: ColorPolicy
+
+    def style(self, text: str, *styles: AnsiCode) -> str:
+        if not self.color_policy.enabled:
+            return text
+
+        prefix = "".join(style.value for style in styles)
+        return prefix + text + AnsiCode.RESET.value
+
+
+# The issue tracker when korml runs without installed package metadata.
+_FALLBACK_ISSUES_URL = "https://github.com/seanergys/pykorml/issues"
+
+
+def issues_url() -> str:
+    """
+    Return the issue tracker named by the project URLs in pyproject.toml.
+
+    That is the Issues URL if there is one, otherwise the issues page of the
+    Source URL, so the bug report and the package metadata never disagree.
+    Without installed metadata, it is github.com/seanergys/pykorml/issues.
+    """
+    try:
+        metadata = importlib.metadata.metadata("korml")
+    except importlib.metadata.PackageNotFoundError:
+        return _FALLBACK_ISSUES_URL
+
+    url_map = {}
+    for entry in metadata.get_all("Project-URL") or []:
+        label, _, url = entry.partition(",")
+        url_map[label.strip().lower()] = url.strip().rstrip("/")
+
+    if "issues" in url_map:
+        return url_map["issues"]
+
+    source = url_map.get("source")
+    if source is None:
+        return _FALLBACK_ISSUES_URL
+
+    if "gitlab" in source:
+        return f"{source}/-/issues"
+
+    return f"{source}/issues"
+
+
+@dataclasses.dataclass(frozen=True)
+class BugReportConfig:
+    url: str = dataclasses.field(default_factory=issues_url)
+
+
+@dataclasses.dataclass(frozen=True)
+class InternalFailureReport:
+    exc: Exception
+    styler: TextStyler
+    config: BugReportConfig = dataclasses.field(default_factory=BugReportConfig)
+
+    def render(self) -> str:
+        return "\n".join(
+            [
+                self.styler.style(
+                    "========================================", AnsiCode.RED, AnsiCode.BOLD
+                ),
+                self.styler.style("ERROR: KORML INTERNAL FAILURE", AnsiCode.RED, AnsiCode.BOLD),
+                self.styler.style(
+                    "========================================", AnsiCode.RED, AnsiCode.BOLD
+                ),
+                "",
+                self.styler.style("KORML encountered an unexpected internal error.", AnsiCode.BOLD),
+                "This usually indicates a bug in KORML rather than an error in your input.",
+                "",
+                self.styler.style(
+                    f"If you would like to report this, please open an issue at: {self.config.url}",
+                    AnsiCode.YELLOW,
+                ),
+                "",
+                "A bug report is most helpful when it includes:",
+                "",
+                "  • a short issue title, for example:",
+                "      [BUG] <short description>",
+                "",
+                "  • a minimal working example (MWE), including:",
+                "      - the smallest input that reproduces the problem",
+                "      - the exact command used to trigger it",
+                "",
+                "  • the diagnostic report below",
+                "",
+                "You may copy and paste the following section into the issue if useful.",
+                "",
+                self.styler.style(
+                    "----- BEGIN KORML BUG REPORT -----", AnsiCode.CYAN, AnsiCode.BOLD
+                ),
+                self.styler.style("## Error Summary", AnsiCode.CYAN, AnsiCode.BOLD),
+                f"- Exception type: {type(self.exc).__name__}",
+                f"- Exception message: {self.exc}",
+                "",
+                self.styler.style("## Traceback", AnsiCode.CYAN, AnsiCode.BOLD),
+                "```text",
+                traceback.format_exc().rstrip(),
+                "```",
+                self.styler.style("----- END KORML BUG REPORT -----", AnsiCode.CYAN, AnsiCode.BOLD),
+            ]
+        )
+
+
+def render_internal_failure_report(exc: Exception, stream: typing.Any) -> str:
+    color_policy = ColorPolicy.from_stream(stream)
+    styler = TextStyler(color_policy=color_policy)
+    report = InternalFailureReport(exc=exc, styler=styler)
+    return report.render()