onec-interactive-runtime-core 0.1.19__py3-none-any.whl

onec_runtime/value_materialization.py

@@ -0,0 +1,432 @@
+from __future__ import annotations
+
+from base64 import b64decode
+import binascii
+from collections.abc import Iterator, Mapping
+from dataclasses import dataclass
+from datetime import datetime
+from decimal import Decimal, InvalidOperation
+import json
+from types import MappingProxyType
+from typing import Any
+from uuid import UUID
+
+from onec_runtime.errors import (
+    MaterializationCycleError,
+    MaterializationKeyError,
+    MaterializationLimitError,
+    UnsupportedOnecType,
+    ValuePayloadError,
+)
+
+
+class _OnecSentinel:
+    __slots__ = ("_name",)
+
+    def __init__(self, name: str) -> None:
+        self._name = name
+
+    def __repr__(self) -> str:
+        return self._name
+
+    def __copy__(self) -> _OnecSentinel:
+        return self
+
+    def __deepcopy__(self, memo: dict[int, object]) -> _OnecSentinel:
+        del memo
+        return self
+
+
+ONEC_UNDEFINED = _OnecSentinel("ONEC_UNDEFINED")
+ONEC_NULL = _OnecSentinel("ONEC_NULL")
+
+
+@dataclass(frozen=True, slots=True)
+class MaterializationOptions:
+    refs: str = "presentation"
+    max_depth: int = 32
+    max_items: int = 100_000
+    max_bytes: int = 64 * 1024 * 1024
+
+    def __post_init__(self) -> None:
+        if self.refs not in {"presentation", "uuid", "both"}:
+            raise ValueError("reference mode must be presentation, uuid, or both")
+        for name in ("max_depth", "max_items", "max_bytes"):
+            value = getattr(self, name)
+            if type(value) is not int or value <= 0:
+                raise ValueError(f"{name} must be a positive integer")
+
+
+@dataclass(frozen=True, slots=True)
+class OnecReference:
+    type_name: str
+    uuid: UUID | None
+    presentation: str
+    empty: bool
+
+
+@dataclass(frozen=True, slots=True)
+class OnecEnumValue:
+    type_name: str
+    name: str
+    presentation: str
+
+
+@dataclass(frozen=True, slots=True)
+class OnecObjectSnapshot(Mapping[str, object]):
+    type_name: str
+    attributes: Mapping[str, object]
+    tabular_sections: tuple[str, ...] = ()
+
+    def __post_init__(self) -> None:
+        if not self.type_name:
+            raise ValueError("1C object type name must not be empty")
+        object.__setattr__(self, "attributes", MappingProxyType(dict(self.attributes)))
+        object.__setattr__(self, "tabular_sections", tuple(self.tabular_sections))
+
+    def __getitem__(self, key: str) -> object:
+        return self.attributes[key]
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self.attributes)
+
+    def __len__(self) -> int:
+        return len(self.attributes)
+
+
+@dataclass(frozen=True, slots=True)
+class OnecTreeRow:
+    values: tuple[object, ...]
+    children: tuple[OnecTreeRow, ...] = ()
+
+
+@dataclass(frozen=True, slots=True)
+class OnecTreeSnapshot:
+    columns: tuple[str, ...]
+    rows: tuple[OnecTreeRow, ...]
+
+
+def decode_value_payload(
+    payload: bytes,
+    options: MaterializationOptions | None = None,
+) -> object:
+    selected = options or MaterializationOptions()
+    if not isinstance(payload, bytes):
+        raise TypeError("typed value payload must be bytes")
+    if len(payload) > selected.max_bytes:
+        raise MaterializationLimitError(
+            f"materialization bytes limit {selected.max_bytes} exceeded at $"
+        )
+    try:
+        document = json.loads(payload.decode("utf-8"))
+    except (UnicodeDecodeError, json.JSONDecodeError) as error:
+        raise ValuePayloadError("typed value payload is not valid UTF-8 JSON") from error
+    if not isinstance(document, dict) or document.get("version") != 1:
+        raise ValuePayloadError("typed value payload version is invalid")
+    keys = set(document)
+    if keys == {"version", "error"}:
+        _raise_wire_error(document["error"])
+    if keys != {"version", "root"}:
+        raise ValuePayloadError("typed value payload envelope is invalid")
+    decoder = _ValueDecoder(selected)
+    return decoder.decode(document["root"], path="$", depth=0)
+
+
+class _ValueDecoder:
+    __slots__ = ("options", "items")
+
+    def __init__(self, options: MaterializationOptions) -> None:
+        self.options = options
+        self.items = 0
+
+    def decode(self, node: object, *, path: str, depth: int) -> object:
+        if depth > self.options.max_depth:
+            raise MaterializationLimitError(
+                f"materialization depth limit {self.options.max_depth} exceeded at {path}"
+            )
+        if not isinstance(node, dict) or not isinstance(node.get("t"), str):
+            raise ValuePayloadError(f"typed value node is invalid at {path}")
+        tag = node["t"]
+        if tag == "undefined":
+            self._exact(node, {"t"}, path)
+            return ONEC_UNDEFINED
+        if tag == "null":
+            self._exact(node, {"t"}, path)
+            return ONEC_NULL
+        if tag == "string":
+            return self._scalar(node, str, path)
+        if tag == "boolean":
+            return self._scalar(node, bool, path)
+        if tag == "number":
+            raw = self._scalar(node, str, path)
+            try:
+                return Decimal(raw)
+            except InvalidOperation as error:
+                raise ValuePayloadError(f"number is invalid at {path}") from error
+        if tag == "datetime":
+            raw = self._scalar(node, str, path)
+            try:
+                return datetime.fromisoformat(raw.replace("Z", "+00:00"))
+            except ValueError as error:
+                raise ValuePayloadError(f"datetime is invalid at {path}") from error
+        if tag == "uuid":
+            raw = self._scalar(node, str, path)
+            try:
+                return UUID(raw)
+            except ValueError as error:
+                raise ValuePayloadError(f"UUID is invalid at {path}") from error
+        if tag == "binary":
+            raw = self._scalar(node, str, path)
+            try:
+                return b64decode(raw, validate=True)
+            except (ValueError, binascii.Error) as error:
+                raise ValuePayloadError(f"binary Base64 is invalid at {path}") from error
+        if tag in {"array", "fixed_array"}:
+            values = self._sequence(node, path)
+            decoded = [
+                self.decode(value, path=f"{path}[{index}]", depth=depth + 1)
+                for index, value in enumerate(values)
+            ]
+            return tuple(decoded) if tag == "fixed_array" else decoded
+        if tag == "structure":
+            return self._decode_structure(node, path=path, depth=depth)
+        if tag == "map":
+            return self._decode_map(node, path=path, depth=depth)
+        if tag == "reference":
+            return self._decode_reference(node, path)
+        if tag == "enum":
+            return self._decode_enum(node, path)
+        if tag == "object":
+            return self._decode_object(node, path=path, depth=depth)
+        if tag == "tree":
+            return self._decode_tree(node, path=path, depth=depth)
+        raise ValuePayloadError(f"unknown typed value tag {tag!r} at {path}")
+
+    def _scalar(self, node: dict[str, object], expected: type, path: str) -> Any:
+        self._exact(node, {"t", "v"}, path)
+        value = node["v"]
+        if type(value) is not expected:
+            raise ValuePayloadError(f"typed scalar value is invalid at {path}")
+        return value
+
+    def _sequence(self, node: dict[str, object], path: str) -> list[object]:
+        self._exact(node, {"t", "v"}, path)
+        values = node["v"]
+        if not isinstance(values, list):
+            raise ValuePayloadError(f"typed collection is invalid at {path}")
+        self._add_items(len(values), path)
+        return values
+
+    def _decode_structure(
+        self, node: dict[str, object], *, path: str, depth: int
+    ) -> dict[str, object]:
+        pairs = self._sequence(node, path)
+        result: dict[str, object] = {}
+        normalized: set[str] = set()
+        for index, pair in enumerate(pairs):
+            if not isinstance(pair, list) or len(pair) != 2 or not isinstance(pair[0], str):
+                raise ValuePayloadError(f"structure entry is invalid at {path}[{index}]")
+            key = pair[0]
+            if key.casefold() in normalized:
+                raise ValuePayloadError(f"duplicate structure key at {path}.{key}")
+            normalized.add(key.casefold())
+            result[key] = self.decode(pair[1], path=f"{path}.{key}", depth=depth + 1)
+        return result
+
+    def _decode_map(
+        self, node: dict[str, object], *, path: str, depth: int
+    ) -> dict[object, object]:
+        pairs = self._sequence(node, path)
+        result: dict[object, object] = {}
+        for index, pair in enumerate(pairs):
+            entry_path = f"{path}[{index}]"
+            if not isinstance(pair, list) or len(pair) != 2:
+                raise ValuePayloadError(f"Map entry is invalid at {entry_path}")
+            key = self.decode(pair[0], path=f"{entry_path}.key", depth=depth + 1)
+            try:
+                hash(key)
+            except TypeError as error:
+                raise MaterializationKeyError(
+                    f"materialized Map key is not hashable at {entry_path}.key"
+                ) from error
+            if key in result:
+                raise MaterializationKeyError(
+                    f"materialized Map key collides at {entry_path}.key"
+                )
+            result[key] = self.decode(
+                pair[1], path=f"{entry_path}.value", depth=depth + 1
+            )
+        return result
+
+    def _decode_reference(self, node: dict[str, object], path: str) -> OnecReference:
+        self._exact(node, {"t", "type", "uuid", "presentation", "empty"}, path)
+        type_name = node["type"]
+        raw_uuid = node["uuid"]
+        presentation = node["presentation"]
+        empty = node["empty"]
+        if (
+            not isinstance(type_name, str)
+            or not type_name
+            or raw_uuid is not None
+            and not isinstance(raw_uuid, str)
+            or not isinstance(presentation, str)
+            or type(empty) is not bool
+        ):
+            raise ValuePayloadError(f"reference node is invalid at {path}")
+        try:
+            value_uuid = None if raw_uuid is None else UUID(raw_uuid)
+        except ValueError as error:
+            raise ValuePayloadError(f"reference UUID is invalid at {path}") from error
+        if empty and value_uuid is not None:
+            raise ValuePayloadError(f"empty reference has UUID at {path}")
+        return OnecReference(type_name, value_uuid, presentation, empty)
+
+    def _decode_enum(self, node: dict[str, object], path: str) -> OnecEnumValue:
+        self._exact(node, {"t", "type", "name", "presentation"}, path)
+        values = (node["type"], node["name"], node["presentation"])
+        if (
+            not isinstance(values[0], str)
+            or not values[0]
+            or not isinstance(values[1], str)
+            or not isinstance(values[2], str)
+        ):
+            raise ValuePayloadError(f"enumeration node is invalid at {path}")
+        return OnecEnumValue(values[0], values[1], values[2])
+
+    def _decode_object(
+        self, node: dict[str, object], *, path: str, depth: int
+    ) -> OnecObjectSnapshot:
+        self._exact(node, {"t", "type", "attributes", "sections"}, path)
+        type_name = node["type"]
+        attributes = node["attributes"]
+        sections = node["sections"]
+        if not isinstance(type_name, str) or not type_name:
+            raise ValuePayloadError(f"object type is invalid at {path}")
+        if not isinstance(attributes, list) or not isinstance(sections, list):
+            raise ValuePayloadError(f"object node is invalid at {path}")
+        self._add_items(len(attributes), path)
+        decoded: dict[str, object] = {}
+        normalized: set[str] = set()
+        for index, pair in enumerate(attributes):
+            if not isinstance(pair, list) or len(pair) != 2 or not isinstance(pair[0], str):
+                raise ValuePayloadError(f"object attribute is invalid at {path}[{index}]")
+            name = pair[0]
+            if name.casefold() in normalized:
+                raise ValuePayloadError(f"duplicate object attribute at {path}.{name}")
+            normalized.add(name.casefold())
+            decoded[name] = self.decode(
+                pair[1], path=f"{path}.{name}", depth=depth + 1
+            )
+        if any(not isinstance(section, str) or not section for section in sections):
+            raise ValuePayloadError(f"object tabular sections are invalid at {path}")
+        if len({section.casefold() for section in sections}) != len(sections):
+            raise ValuePayloadError(f"duplicate object tabular section at {path}")
+        return OnecObjectSnapshot(type_name, decoded, tuple(sections))
+
+    def _decode_tree(
+        self, node: dict[str, object], *, path: str, depth: int
+    ) -> OnecTreeSnapshot:
+        self._exact(node, {"t", "columns", "rows"}, path)
+        columns = node["columns"]
+        rows = node["rows"]
+        if (
+            not isinstance(columns, list)
+            or any(not isinstance(column, str) or not column for column in columns)
+            or len(set(columns)) != len(columns)
+            or not isinstance(rows, list)
+        ):
+            raise ValuePayloadError(f"tree node is invalid at {path}")
+        return OnecTreeSnapshot(
+            tuple(columns),
+            tuple(
+                self._decode_tree_row(
+                    row,
+                    columns=len(columns),
+                    path=f"{path}.rows[{index}]",
+                    depth=depth + 1,
+                )
+                for index, row in enumerate(rows)
+            ),
+        )
+
+    def _decode_tree_row(
+        self,
+        row: object,
+        *,
+        columns: int,
+        path: str,
+        depth: int,
+    ) -> OnecTreeRow:
+        if depth > self.options.max_depth:
+            raise MaterializationLimitError(
+                f"materialization depth limit {self.options.max_depth} exceeded at {path}"
+            )
+        if not isinstance(row, dict) or set(row) != {"values", "children"}:
+            raise ValuePayloadError(f"tree row is invalid at {path}")
+        values = row["values"]
+        children = row["children"]
+        if not isinstance(values, list) or len(values) != columns or not isinstance(
+            children, list
+        ):
+            raise ValuePayloadError(f"tree row shape is invalid at {path}")
+        self._add_items(1, path)
+        return OnecTreeRow(
+            tuple(
+                self.decode(value, path=f"{path}.values[{index}]", depth=depth)
+                for index, value in enumerate(values)
+            ),
+            tuple(
+                self._decode_tree_row(
+                    child,
+                    columns=columns,
+                    path=f"{path}.children[{index}]",
+                    depth=depth + 1,
+                )
+                for index, child in enumerate(children)
+            ),
+        )
+
+    def _add_items(self, count: int, path: str) -> None:
+        self.items += count
+        if self.items > self.options.max_items:
+            raise MaterializationLimitError(
+                f"materialization items limit {self.options.max_items} exceeded at {path}"
+            )
+
+    @staticmethod
+    def _exact(node: dict[str, object], expected: set[str], path: str) -> None:
+        if set(node) != expected:
+            raise ValuePayloadError(f"typed value node fields are invalid at {path}")
+
+
+def _raise_wire_error(value: object) -> None:
+    if not isinstance(value, dict) or not isinstance(value.get("kind"), str):
+        raise ValuePayloadError("typed value error envelope is invalid")
+    kind = value["kind"]
+    path = value.get("path")
+    if not isinstance(path, str) or not path:
+        raise ValuePayloadError("typed value error path is invalid")
+    if kind == "cycle":
+        if set(value) != {"kind", "path", "first_path"} or not isinstance(
+            value.get("first_path"), str
+        ):
+            raise ValuePayloadError("typed value cycle envelope is invalid")
+        raise MaterializationCycleError(
+            f"materialization cycle at {path}; first seen at {value['first_path']}"
+        )
+    if kind == "unsupported":
+        if set(value) != {"kind", "path", "type"} or not isinstance(
+            value.get("type"), str
+        ):
+            raise ValuePayloadError("typed value unsupported envelope is invalid")
+        raise UnsupportedOnecType(
+            f"unsupported 1C type {value['type']} at {path}"
+        )
+    if kind in {"depth_limit", "item_limit", "byte_limit"}:
+        if set(value) != {"kind", "path", "limit"} or type(value.get("limit")) is not int:
+            raise ValuePayloadError("typed value limit envelope is invalid")
+        label = kind.removesuffix("_limit").replace("item", "items")
+        raise MaterializationLimitError(
+            f"materialization {label} limit {value['limit']} exceeded at {path}"
+        )
+    raise ValuePayloadError(f"unknown typed value error kind {kind!r}")

onec_runtime/value_transfer_backend.py

@@ -0,0 +1,199 @@
+from __future__ import annotations
+
+from base64 import b64decode
+import binascii
+from collections.abc import Callable
+from hashlib import sha256
+import re
+from uuid import uuid4
+
+from onec_runtime.errors import ProtocolError
+from onec_runtime.experiment import bsl_string_literal
+from onec_runtime.performance_profile import PhaseRecorder
+from onec_runtime.value_materialization import (
+    MaterializationOptions,
+    decode_value_payload,
+)
+
+
+VALUE_CONTEXT_KEY_PREFIX = "__onec_value_"
+_HANDLE = re.compile(
+    r"Контекст\.[^\W\d]\w*(?:\.[^\W\d]\w*)*\Z",
+    re.UNICODE,
+)
+_CONTEXT_KEY = re.compile(r"__onec_value_[0-9a-f]{32}\Z")
+_HASH = re.compile(r"[0-9a-f]{64}\Z")
+_ERROR_ENVELOPE_MAX_BYTES = 4096
+
+
+def build_value_transfer_instruction(
+    handle: str,
+    options: MaterializationOptions,
+    context_key: str,
+    *,
+    runtime_generation: int,
+    context_generation: int,
+) -> str:
+    validate_value_handle(handle)
+    if not _CONTEXT_KEY.fullmatch(context_key):
+        raise ProtocolError("value materialization context key is invalid")
+    if runtime_generation <= 0 or context_generation <= 0:
+        raise ProtocolError("value transfer generations must be positive")
+    return "\n".join(
+        (
+            "МатериализацияЗначения = "
+            "RuntimeValueTransferServer.СериализоватьЗначение("
+            f"{handle}, {bsl_string_literal(options.refs)}, "
+            f"{options.max_depth}, {options.max_items}, {options.max_bytes});",
+            f"Контекст.Вставить({bsl_string_literal(context_key)}, "
+            "МатериализацияЗначения.Base64);",
+            "Результат = "
+            f'Формат({runtime_generation}, "ЧГ=0; ЧДЦ=0") + "|" + '
+            f'Формат({context_generation}, "ЧГ=0; ЧДЦ=0") + "|" + '
+            'Формат(МатериализацияЗначения.Размер, "ЧГ=0; ЧДЦ=0") + "|" + '
+            'МатериализацияЗначения.Хеш + "|" + '
+            "Формат(СтрДлина(МатериализацияЗначения.Base64), "
+            '"ЧГ=0; ЧДЦ=0");',
+        )
+    )
+
+
+def validate_value_handle(handle: str) -> str:
+    if not isinstance(handle, str) or not _HANDLE.fullmatch(handle):
+        raise ProtocolError(
+            "value handle must be one direct or dotted persistent Context path"
+        )
+    return handle
+
+
+class RuntimeValueTransfer:
+    def __init__(
+        self,
+        instruction_executor: Callable[[str], object],
+        context_reader: Callable[[str, int], str],
+        *,
+        context_cleaner: Callable[[str], None],
+        runtime_generation: Callable[[], int],
+        context_generation: int,
+        key_factory: Callable[[], str] | None = None,
+        profiler: PhaseRecorder | None = None,
+    ) -> None:
+        self._execute = instruction_executor
+        self._read = context_reader
+        self._clean = context_cleaner
+        self._runtime_generation = runtime_generation
+        self._context_generation = context_generation
+        self._expected_runtime_generation = runtime_generation()
+        self._key_factory = key_factory or (lambda: VALUE_CONTEXT_KEY_PREFIX + uuid4().hex)
+        self._profiler = profiler
+
+    def materialize(self, handle: str, options: MaterializationOptions) -> object:
+        payload = self.payload(handle, options)
+        return self._profile(
+            "value.decode_snapshot",
+            lambda: decode_value_payload(payload, options),
+            input_bytes=len(payload),
+        )
+
+    def payload(self, handle: str, options: MaterializationOptions) -> bytes:
+        generation = self._runtime_generation()
+        if generation != self._expected_runtime_generation:
+            raise ProtocolError("value materializer runtime generation is stale")
+        key = self._key_factory()
+        source = build_value_transfer_instruction(
+            handle,
+            options,
+            key,
+            runtime_generation=generation,
+            context_generation=self._context_generation,
+        )
+        consumed = False
+        primary_error: BaseException | None = None
+        try:
+            metadata = self._profile(
+                "value.prepare",
+                lambda: self._execute(source),
+                input_bytes=len(source.encode("utf-8")),
+            )
+            maximum_transfer_bytes = max(
+                options.max_bytes, _ERROR_ENVELOPE_MAX_BYTES
+            )
+            maximum_text_size = _base64_length(maximum_transfer_bytes)
+            content = self._profile(
+                "value.transfer_base64",
+                lambda: self._read(key, maximum_text_size),
+                output_bytes=lambda value: len(value.encode("ascii")),
+            )
+            consumed = True
+            observed = _parse_metadata(metadata)
+            if (
+                observed[0] != generation
+                or observed[1] != self._context_generation
+                or observed[2] <= 0
+                or observed[2] > maximum_transfer_bytes
+                or observed[4] <= 0
+                or observed[4] > maximum_text_size
+                or len(content) != observed[4]
+            ):
+                raise ProtocolError("value materialization metadata is invalid")
+            try:
+                payload = self._profile(
+                    "value.decode_base64",
+                    lambda: b64decode("".join(content.split()), validate=True),
+                    input_bytes=len(content.encode("ascii")),
+                    output_bytes=len,
+                )
+            except (ValueError, binascii.Error) as error:
+                raise ProtocolError("value materialization Base64 payload is invalid") from error
+            if len(payload) != observed[2] or sha256(payload).hexdigest() != observed[3]:
+                raise ProtocolError("value materialization payload integrity check failed")
+            return payload
+        except BaseException as error:
+            primary_error = error
+            raise
+        finally:
+            if not consumed:
+                try:
+                    self._clean(key)
+                except BaseException as cleanup_error:
+                    if primary_error is not None:
+                        primary_error.add_note(
+                            "value materialization cleanup failed: "
+                            f"{type(cleanup_error).__name__}: {cleanup_error}"
+                        )
+                    else:
+                        raise
+
+    def _profile(self, phase: str, operation, **metadata):  # type: ignore[no-untyped-def]
+        if self._profiler is None:
+            return operation()
+        return self._profiler.measure(phase, operation, **metadata)
+
+
+def _parse_metadata(value: object) -> tuple[int, int, int, str, int]:
+    if not isinstance(value, str):
+        raise ProtocolError("value materialization metadata is not a string")
+    fields = value.split("|")
+    if len(fields) != 5:
+        raise ProtocolError("value materialization metadata field count is invalid")
+    try:
+        runtime_generation = int(fields[0])
+        context_generation = int(fields[1])
+        byte_count = int(fields[2])
+        base64_count = int(fields[4])
+    except ValueError as error:
+        raise ProtocolError("value materialization metadata number is invalid") from error
+    payload_hash = fields[3]
+    if not _HASH.fullmatch(payload_hash):
+        raise ProtocolError("value materialization metadata hash is invalid")
+    return (
+        runtime_generation,
+        context_generation,
+        byte_count,
+        payload_hash,
+        base64_count,
+    )
+
+
+def _base64_length(byte_count: int) -> int:
+    return ((byte_count + 2) // 3) * 4