yaml-compose 0.1.0__py3-none-any.whl

yaml_compose/__init__.py

@@ -0,0 +1,78 @@
+"""Safe YAML loading with file includes and mapping overlays."""
+
+from ._search import IncludeSearch
+from ._unset import UNSET, UnsetType
+from .limits import (
+    DEFAULT_INCLUDE_LIMITS,
+    IncludeLimits,
+    get_default_include_limits,
+    set_default_include_limits,
+)
+from .loader import (
+    IncludeLoader,
+    YamlConfigLoader,
+    YamlIncludeError,
+    load_yaml_compose,
+    load_yaml_dispatch,
+    load_yaml_file,
+)
+from .resolver import (
+    FileRoot,
+    FileSource,
+    FileSystemIncludeResolver,
+    IncludeRequest,
+    IncludeResolver,
+    IncludeRoot,
+    IncludeSource,
+    MemoryIncludeResolver,
+    MemoryRoot,
+    MemorySource,
+    ResolvedInclude,
+)
+from .tags import (
+    COMPOSE_LOADER_TAG,
+    DEEP_MERGE_KEY,
+    DELETE_TAG,
+    INCLUDE_TAG,
+    LOADER_DIRECTIVE,
+    MERGE_KEY,
+    MERGE_TAG,
+    REPLACE_TAG,
+    SPREAD_TAG,
+)
+
+__all__ = (
+    "COMPOSE_LOADER_TAG",
+    "DEEP_MERGE_KEY",
+    "DEFAULT_INCLUDE_LIMITS",
+    "DELETE_TAG",
+    "INCLUDE_TAG",
+    "LOADER_DIRECTIVE",
+    "MERGE_KEY",
+    "MERGE_TAG",
+    "REPLACE_TAG",
+    "SPREAD_TAG",
+    "UNSET",
+    "FileRoot",
+    "FileSource",
+    "FileSystemIncludeResolver",
+    "IncludeLimits",
+    "IncludeLoader",
+    "IncludeRequest",
+    "IncludeResolver",
+    "IncludeRoot",
+    "IncludeSearch",
+    "IncludeSource",
+    "MemoryIncludeResolver",
+    "MemoryRoot",
+    "MemorySource",
+    "ResolvedInclude",
+    "UnsetType",
+    "YamlConfigLoader",
+    "YamlIncludeError",
+    "get_default_include_limits",
+    "load_yaml_compose",
+    "load_yaml_dispatch",
+    "load_yaml_file",
+    "set_default_include_limits",
+)

yaml_compose/_overlays.py

@@ -0,0 +1,114 @@
+"""Mapping composition and validation shared by YAML merge keys and overlays.
+
+Merge keys combine completed mappings by source priority and apply explicit
+local values over the inherited mapping. Mapping operations copy their inputs
+instead of modifying the original sources.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Hashable, Mapping, MutableSet
+from dataclasses import dataclass
+from typing import cast
+
+import yaml
+from yaml.constructor import ConstructorError
+from yaml.error import Mark
+
+from .tags import DELETE_TAG, REPLACE_TAG, SPREAD_TAG
+
+YamlDict = dict[Hashable, object]
+YamlType = YamlDict | list[object] | str | int | float | bool | None
+DELETE = object()
+
+
+class YamlIncludeError(yaml.YAMLError, ValueError):
+    """YAML error raised for invalid loader selection, includes, or overlays."""
+
+
+@dataclass(frozen=True, slots=True)
+class SpreadMarker:
+    """Sequence items waiting for insertion into their containing list."""
+
+    source: list[object]
+
+
+@dataclass(frozen=True, slots=True)
+class ReplaceMarker:
+    """A complete mapping that replaces one value during a deep overlay."""
+
+    source: YamlDict
+    start_mark: Mark
+
+
+def assert_no_markers(value: object, seen: MutableSet[int] | None = None) -> None:
+    """Reject unapplied operations in a completed value, including cyclic data."""
+    if value is DELETE:
+        raise YamlIncludeError(f"{DELETE_TAG} is only valid inside a mapping overlay")
+    if isinstance(value, SpreadMarker):
+        raise YamlIncludeError(f"{SPREAD_TAG} is only valid as a list item")
+    if isinstance(value, ReplaceMarker):
+        raise ConstructorError(
+            f"while applying {REPLACE_TAG}",
+            None,
+            f"{REPLACE_TAG} is only valid in a deep mapping overlay",
+            value.start_mark,
+        )
+    if seen is None:
+        seen = set()
+    if isinstance(value, dict):
+        mapping = cast(YamlDict, value)
+        if id(mapping) in seen:
+            return
+        seen.add(id(mapping))
+        for key, item in mapping.items():
+            assert_no_markers(key, seen)
+            assert_no_markers(item, seen)
+    elif isinstance(value, (list, tuple)):
+        sequence = cast(list[object] | tuple[object, ...], value)
+        if id(sequence) in seen:
+            return
+        seen.add(id(sequence))
+        for item in sequence:
+            assert_no_markers(item, seen)
+
+
+# opaque-boundary: PyYAML values may contain arbitrary nested mappings.
+def merge_mappings(
+    base: Mapping[Hashable, object],
+    overlay: Mapping[Hashable, object],
+    *,
+    deep: bool,
+) -> YamlDict:
+    """Apply local overrides, optionally recursing into inherited mappings."""
+    active: set[tuple[int, int]] = set()
+
+    # opaque-boundary: PyYAML values may contain arbitrary nested mappings.
+    def merge(left: Mapping[Hashable, object], right: Mapping[Hashable, object]) -> YamlDict:
+        pair = (id(left), id(right))
+        if pair in active:
+            raise YamlIncludeError("cyclic YAML aliases are not supported during mapping overlays")
+        active.add(pair)
+        try:
+            result = dict(left)
+            for key, value in right.items():
+                if value is DELETE:
+                    result.pop(key, None)
+                elif isinstance(value, ReplaceMarker):
+                    if not deep:
+                        raise ConstructorError(
+                            f"while applying {REPLACE_TAG}",
+                            None,
+                            f"{REPLACE_TAG} is only valid in a deep mapping overlay",
+                            value.start_mark,
+                        )
+                    result[key] = value.source
+                elif deep and isinstance(result.get(key), dict) and isinstance(value, dict):
+                    result[key] = merge(cast(YamlDict, result[key]), cast(YamlDict, value))
+                else:
+                    result[key] = value
+            return result
+        finally:
+            active.remove(pair)
+
+    return merge(base, overlay)

yaml_compose/_paths.py

@@ -0,0 +1,58 @@
+"""Canonical filesystem lookup for includes within explicitly allowed roots."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from pathlib import Path
+
+from ._search import IncludeSearch, is_explicit_relative, search_locations
+
+
+def resolve_include_file(
+    name: str,
+    current_file: Path | None,
+    search_paths: Iterable[Path],
+    allowed_roots: Iterable[Path],
+    *,
+    include_search: IncludeSearch = "local-first",
+) -> Path:
+    """Resolve a local include against its parent, search paths, or absolute path.
+
+    Roots arrive canonicalized from the loader. Each candidate is canonicalized
+    before the root check, so a symlink outside an allowed root is rejected.
+    Bare names follow ``include_search``. Explicit ``./`` and ``../`` names
+    require a containing file and never fall back to a search directory.
+    Absolute paths name one exact file independently of the search policy.
+    """
+    requested = Path(name)
+
+    roots = tuple(allowed_roots)
+    if not roots:
+        raise ValueError("include loading requires at least one allowed root")
+
+    candidates: list[Path] = []
+    if requested.is_absolute():
+        candidates.append(requested)
+    elif is_explicit_relative(name):
+        if current_file is None:
+            raise ValueError("explicit relative includes require an including source")
+        candidates.append(current_file.parent / requested)
+    else:
+        local = current_file.parent if current_file is not None else None
+        candidates.extend(root / requested for root in search_locations(local, search_paths, include_search))
+
+    for candidate in candidates:
+        try:
+            resolved = candidate.resolve(strict=True)
+        except FileNotFoundError:
+            continue
+        except (OSError, RuntimeError) as exc:
+            raise ValueError(f"cannot safely resolve include path {name!r}: {exc}") from exc
+
+        if not resolved.is_file():
+            continue
+        if not any(resolved.is_relative_to(root) for root in roots):
+            raise ValueError(f"include path {name!r} resolves outside the allowed roots")
+        return resolved
+
+    raise FileNotFoundError(f"include file {name!r} was not found")

yaml_compose/_search.py

@@ -0,0 +1,45 @@
+"""Shared include lookup policies for filesystem and virtual sources."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from pathlib import PurePath
+from typing import Literal, TypeVar, cast, get_args
+
+IncludeSearch = Literal["local-first", "search-first", "local-only", "search-only"]
+"""Lookup policy for include names without an explicit relative prefix.
+
+``local`` means the directory of the document containing that include.
+``search`` means the configured search locations in their supplied order.
+Absolute paths and explicit ``./`` or ``../`` paths bypass this policy.
+"""
+
+_INCLUDE_SEARCH_CHOICES: tuple[str, ...] = get_args(IncludeSearch)
+
+_TPath = TypeVar("_TPath", bound=PurePath)
+
+
+def validate_include_search(value: object) -> IncludeSearch:
+    """Reject unknown lookup policies before any document is loaded."""
+    if not isinstance(value, str) or value not in _INCLUDE_SEARCH_CHOICES:
+        raise ValueError("include_search must be " + ", ".join(repr(x) for x in _INCLUDE_SEARCH_CHOICES))
+    return cast(IncludeSearch, value)
+
+
+def is_explicit_relative(name: str) -> bool:
+    """Recognize a containing-file reference before path normalization removes dots."""
+    return name in (".", "..") or name.startswith(("./", "../", ".\\", "..\\"))
+
+
+def search_locations(local: _TPath | None, search_paths: Iterable[_TPath], policy: IncludeSearch) -> tuple[_TPath, ...]:
+    """Order candidate directories without changing which roots are authorized."""
+    validate_include_search(policy)
+    nearby = () if local is None else (local,)
+    configured = tuple(search_paths)
+    if policy == "local-first":
+        return *nearby, *configured
+    if policy == "search-first":
+        return *configured, *nearby
+    if policy == "local-only":
+        return nearby
+    return configured

yaml_compose/_streams.py

@@ -0,0 +1,37 @@
+"""Read resolver-owned binary streams with optional byte bounds."""
+
+from typing import BinaryIO, cast
+
+_DEFAULT_MAX_CHUNK_SIZE = 65_536
+
+
+def _read_binary_chunk(stream: BinaryIO, size: int) -> bytes | None:
+    """Keep a resolver's possibly non-blocking read result at the boundary."""
+    value: object = stream.read(size)
+    return cast(bytes | None, value)
+
+
+def read_bounded(stream: BinaryIO, limit: int | None, max_chunk_size: int | None = None) -> bytes:
+    """Read a binary stream in chunks, stopping at ``limit`` or EOF.
+
+    ``None`` reads to EOF. Bounded callers include one probe byte in ``limit``
+    so they can distinguish a complete file from one that exceeds the budget.
+    """
+    content = bytearray()
+    remaining = limit
+    chunk_size = _DEFAULT_MAX_CHUNK_SIZE if (max_chunk_size is None or max_chunk_size < 1) else max_chunk_size
+    while remaining is None or remaining > 0:
+        requested = min(remaining, chunk_size) if remaining is not None else chunk_size
+        chunk = _read_binary_chunk(stream, requested)
+        if not isinstance(chunk, bytes):
+            if chunk is None:
+                raise OSError("include resolver stream returned no blocking read result")
+            raise TypeError("include resolver streams must return bytes")
+        if len(chunk) > requested:
+            raise ValueError("include resolver stream returned more bytes than requested")
+        if not chunk:
+            break
+        content.extend(chunk)
+        if remaining is not None:
+            remaining -= len(chunk)
+    return bytes(content)

yaml_compose/_unset.py

@@ -0,0 +1,23 @@
+"""Typed singleton distinguishing omitted arguments from explicit values."""
+
+from __future__ import annotations
+
+from typing import cast
+
+
+class UnsetType:
+    """Sentinel type distinguishing an omitted argument from explicit ``None``."""
+
+    _instance: UnsetType | None = None
+
+    def __new__(cls) -> UnsetType:
+        """Return the one module-level instance for stable identity checks."""
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+        return cast(UnsetType, cls._instance)
+
+    def __repr__(self) -> str:
+        return "UNSET"
+
+
+UNSET = UnsetType()

yaml_compose/_yaml_loader.py

@@ -0,0 +1,123 @@
+"""Keep caller-selected PyYAML loaders and duplicate-key policy together."""
+
+from __future__ import annotations
+
+from collections.abc import Hashable
+from typing import Protocol, TypeVar, cast
+
+from yaml import FullLoader, Loader, MappingNode, Node, SafeLoader, ScalarNode, UnsafeLoader
+from yaml.constructor import ConstructorError
+
+from .tags import (
+    DELETE_TAG,
+    INCLUDE_TAG,
+    MERGE_KEY,
+    MERGE_TAG,
+    REPLACE_TAG,
+    SPREAD_TAG,
+)
+
+
+class _ObjectConstructor(Protocol):
+    def construct_object(self, node: Node, deep: bool = False) -> object: ...
+
+
+class _LoaderRegistries(Protocol):
+    yaml_constructors: dict[Hashable, object]
+    yaml_multi_constructors: dict[Hashable, object]
+    yaml_implicit_resolvers: dict[Hashable, object]
+    yaml_path_resolvers: dict[Hashable, object]
+
+
+class UniqueKeySafeLoader(SafeLoader):
+    """Reject duplicate explicit keys while retaining SafeLoader constructors."""
+
+    @staticmethod
+    def _is_merge_key(node: Node) -> bool:
+        return isinstance(node, ScalarNode) and node.tag == MERGE_TAG and node.value == MERGE_KEY
+
+    # opaque-boundary: PyYAML mappings can contain arbitrary hashable keys and values.
+    def construct_mapping(self, node: MappingNode, deep: bool = False) -> dict[Hashable, object]:
+        """Construct a mapping after checking its explicit keys."""
+        seen: set[Hashable] = set()
+        for key_node, _ in node.value:
+            if self._is_merge_key(key_node):
+                continue
+            construct = cast(_ObjectConstructor, self).construct_object
+            key = construct(key_node, False)
+            try:
+                hash(key)
+            except TypeError as exc:
+                raise ConstructorError(
+                    "while constructing a mapping",
+                    node.start_mark,
+                    f"found unhashable key {key!r}",
+                    key_node.start_mark,
+                ) from exc
+            if key in seen:
+                raise ConstructorError(
+                    "while constructing a mapping",
+                    node.start_mark,
+                    f"found duplicate key {key!r}",
+                    key_node.start_mark,
+                )
+            seen.add(key)
+        return super().construct_mapping(node, deep=deep)
+
+
+def validate_yaml_loader(yaml_loader: object) -> None:
+    """Require a supported PyYAML loader class or one of its subclasses."""
+    supported_loaders = (SafeLoader, FullLoader, Loader, UnsafeLoader)
+    if not isinstance(yaml_loader, type) or not issubclass(yaml_loader, supported_loaders):
+        raise TypeError(
+            "yaml_loader must be a class derived from a supported PyYAML loader "
+            "(yaml.SafeLoader, yaml.FullLoader, yaml.Loader, or yaml.UnsafeLoader)"
+        )
+
+
+def ordinary_loader(yaml_loader: type[object]) -> type[object]:
+    """Layer duplicate-key checking over a caller's PyYAML loader class."""
+    validate_yaml_loader(yaml_loader)
+    if yaml_loader is SafeLoader:
+        return UniqueKeySafeLoader
+    loader_class = cast(_LoaderRegistries, cast(object, yaml_loader))
+    return cast(
+        type[object],
+        type(
+            "UniqueKeyConfiguredLoader",
+            (UniqueKeySafeLoader, yaml_loader),
+            {
+                "yaml_constructors": dict(loader_class.yaml_constructors),
+                "yaml_multi_constructors": dict(loader_class.yaml_multi_constructors),
+                "yaml_implicit_resolvers": dict(loader_class.yaml_implicit_resolvers),
+                "yaml_path_resolvers": dict(loader_class.yaml_path_resolvers),
+            },
+        ),
+    )
+
+
+_TLoader = TypeVar("_TLoader", bound=SafeLoader)
+
+
+def configured_include_loader(yaml_loader: type[object], adapter: type[_TLoader]) -> type[_TLoader]:
+    """Give the include adapter a caller's constructors without changing global loaders."""
+    validate_yaml_loader(yaml_loader)
+    if yaml_loader is SafeLoader:
+        return adapter
+    loader_class = cast(_LoaderRegistries, cast(object, yaml_loader))
+    constructors = dict(loader_class.yaml_constructors)
+    for tag in (INCLUDE_TAG, DELETE_TAG, REPLACE_TAG, SPREAD_TAG):
+        constructors[tag] = adapter.yaml_constructors[tag]
+    return cast(
+        type[_TLoader],
+        type(
+            "ConfiguredIncludeLoader",
+            (adapter, yaml_loader),
+            {
+                "yaml_constructors": constructors,
+                "yaml_multi_constructors": dict(loader_class.yaml_multi_constructors),
+                "yaml_implicit_resolvers": dict(loader_class.yaml_implicit_resolvers),
+                "yaml_path_resolvers": dict(loader_class.yaml_path_resolvers),
+            },
+        ),
+    )

yaml_compose/limits.py

@@ -0,0 +1,129 @@
+"""Bounds for included files and the defaults used by new loaders.
+
+``DEFAULT_INCLUDE_LIMITS`` is the documented baseline. The active process-wide
+value starts as a separate copy, which applications can inspect with
+``get_default_include_limits`` and replace with ``set_default_include_limits``.
+A top-level load captures one limits snapshot; changing the default does not
+change an existing loader.
+"""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+
+from ._unset import UNSET, UnsetType
+
+
+def _resolve_limit(value: int | UnsetType | None, default: int | None) -> int | None:
+    """Resolve an omitted override; the resulting profile validates its fields."""
+    return default if isinstance(value, UnsetType) else value
+
+
+_DEFAULT_MAX_INCLUDE_DEPTH = 32
+_DEFAULT_MAX_INCLUDE_FILES = 1024
+_DEFAULT_MAX_FILE_BYTES = 8_388_608
+_DEFAULT_MAX_TOTAL_INCLUDE_BYTES = 67_108_864
+
+
+@dataclass(slots=True)
+class IncludeLimits:
+    """Optional non-negative limits applied to files reached through ``!include``.
+
+    The top-level YAML input is not byte-limited. Each new top-level load starts
+    a fresh count. A non-negative integer enforces a bound; zero forbids any
+    use of that resource; ``None`` leaves that resource unbounded. Booleans,
+    negative numbers, and other values raise ``ValueError``.
+    ``UNSET`` is reserved for omitted function arguments, not profile fields.
+
+    These are package policy choices for configuration composition, not YAML
+    specification limits. File and byte counts include repeated reads of the
+    same source, including reads for spread and replacement values.
+    They bound input content, not the size of the constructed Python object graph.
+
+    Attributes:
+        max_include_depth: Maximum nested ``!include`` levels, or ``None``.
+        max_include_files: Maximum ``!include`` operations per top-level load,
+            counting repeated loads of the same source, or ``None``.
+        max_file_bytes: Maximum encoded bytes read from one included file, or ``None``.
+        max_total_include_bytes: Maximum encoded bytes read across all includes,
+            or ``None``.
+
+    """
+
+    max_include_depth: int | None = _DEFAULT_MAX_INCLUDE_DEPTH
+    max_include_files: int | None = _DEFAULT_MAX_INCLUDE_FILES
+    max_file_bytes: int | None = _DEFAULT_MAX_FILE_BYTES
+    max_total_include_bytes: int | None = _DEFAULT_MAX_TOTAL_INCLUDE_BYTES
+
+    def __post_init__(self) -> None:
+        """Reject invalid bounds before a limits profile can be used."""
+        values = (
+            self.max_include_depth,
+            self.max_include_files,
+            self.max_file_bytes,
+            self.max_total_include_bytes,
+        )
+        if any(value is not None and (type(value) is not int or value < 0) for value in values):
+            raise ValueError("include limits must be non-negative integers or None")
+
+
+def _copy_limits(limits: IncludeLimits) -> IncludeLimits:
+    """Copy a profile into a validated snapshot."""
+    return IncludeLimits(**asdict(limits))
+
+
+DEFAULT_INCLUDE_LIMITS = IncludeLimits()
+_configured_default_limits = _copy_limits(DEFAULT_INCLUDE_LIMITS)
+
+
+def get_default_include_limits() -> IncludeLimits:
+    """Return the active defaults for new top-level loads and loader instances.
+
+    The returned ``IncludeLimits`` is an independent copy. Use
+    ``dataclasses.replace`` and ``set_default_include_limits`` to change active
+    defaults.
+    """
+    return _copy_limits(_configured_default_limits)
+
+
+def set_default_include_limits(limits: IncludeLimits) -> None:
+    """Replace the process-wide defaults for future loads and loader instances.
+
+    Pass an ``IncludeLimits`` instance, usually at application startup. Existing
+    ``YamlConfigLoader`` instances and in-progress loads keep their captured
+    limits. Per-call ``max_*`` arguments still override individual fields;
+    passing ``None`` explicitly disables one bound.
+    Pass ``DEFAULT_INCLUDE_LIMITS`` to restore the package baseline. A value
+    of another type raises ``TypeError``.
+    """
+    if type(limits) is not IncludeLimits:
+        raise TypeError("limits must be an IncludeLimits instance")
+    global _configured_default_limits
+    _configured_default_limits = _copy_limits(limits)
+
+
+def resolve_include_limits(
+    max_include_depth: int | UnsetType | None,
+    max_include_files: int | UnsetType | None,
+    max_file_bytes: int | UnsetType | None,
+    max_total_include_bytes: int | UnsetType | None,
+    *,
+    include_limits: IncludeLimits | UnsetType = UNSET,
+) -> IncludeLimits:
+    """Combine a reusable profile and individual overrides into one snapshot.
+
+    An omitted profile uses the current process-wide default. Omitted field
+    overrides keep the profile value; explicit ``None`` removes that bound.
+    The returned limits profile is one snapshot, so later default changes
+    cannot alter an in-progress load. Invalid bounds raise ``ValueError`` and
+    an invalid profile raises ``TypeError``.
+    """
+    defaults = get_default_include_limits() if include_limits is UNSET else include_limits
+    if not isinstance(defaults, IncludeLimits):
+        raise TypeError("include_limits must be an IncludeLimits instance")
+    return IncludeLimits(
+        max_include_depth=_resolve_limit(max_include_depth, defaults.max_include_depth),
+        max_include_files=_resolve_limit(max_include_files, defaults.max_include_files),
+        max_file_bytes=_resolve_limit(max_file_bytes, defaults.max_file_bytes),
+        max_total_include_bytes=_resolve_limit(max_total_include_bytes, defaults.max_total_include_bytes),
+    )