hocon-parser 1.8.0__py3-none-any.whl

hocon/_internal/properties/properties.py

@@ -0,0 +1,84 @@
+"""Java-properties parsing for ``.properties`` includes.
+
+Mirrors ts.hocon ``src/internal/properties/properties.ts``. Multi-line + Unicode
+escapes are a documented simplification (S23.5, S23.6). Object always wins over
+scalar on a key conflict (S23.4, HOCON.md L1485), enforced via key-sorted
+insertion so conflict direction is input-order independent.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ...value import HoconObject, HoconScalar, HoconValue
+
+__all__ = ["parse_properties", "properties_to_hocon_value"]
+
+_DANGEROUS_KEYS = frozenset(("__proto__", "constructor", "prototype"))
+
+
+def parse_properties(input_text: str) -> dict[str, Any]:
+    root: dict[str, Any] = {}
+
+    pairs: list[tuple[str, str]] = []
+    for line in input_text.split("\n"):
+        trimmed = line.strip()
+        if trimmed == "" or trimmed.startswith("#") or trimmed.startswith("!"):
+            continue
+        sep_idx = _find_separator(trimmed)
+        if sep_idx == -1:
+            continue
+        key = trimmed[:sep_idx].strip()
+        value = trimmed[sep_idx + 1 :].strip()
+        if key == "":
+            continue
+        pairs.append((key, value))
+
+    # Sort by key so conflict-direction is input-order independent (S23.4).
+    pairs.sort(key=lambda kv: kv[0])
+
+    for key, value in pairs:
+        _set_nested(root, key.split("."), value)
+
+    return root
+
+
+def _find_separator(line: str) -> int:
+    for i, ch in enumerate(line):
+        if ch == "=" or ch == ":":
+            return i
+    return -1
+
+
+def _set_nested(obj: dict[str, Any], segments: list[str], value: str) -> None:
+    current = obj
+    for seg in segments[:-1]:
+        if seg in _DANGEROUS_KEYS:
+            return
+        existing = current.get(seg)
+        if not isinstance(existing, dict):
+            current[seg] = {}
+        current = current[seg]
+    last = segments[-1]
+    if last in _DANGEROUS_KEYS:
+        return
+    # S23.4 — object always wins over scalar: do not overwrite an object.
+    if isinstance(current.get(last), dict):
+        return
+    current[last] = value
+
+
+def properties_to_hocon_value(input_text: str) -> HoconValue:
+    """Convert a ``.properties`` string to a HoconValue (object with string
+    scalars). All values remain strings — no type coercion."""
+    return _record_to_hocon_value(parse_properties(input_text))
+
+
+def _record_to_hocon_value(obj: dict[str, Any]) -> HoconObject:
+    fields: dict[str, HoconValue] = {}
+    for key, val in obj.items():
+        if isinstance(val, str):
+            fields[key] = HoconScalar(val, "string")
+        elif isinstance(val, dict):
+            fields[key] = _record_to_hocon_value(val)
+    return HoconObject(fields)

hocon/_internal/resolver/__init__.py

@@ -0,0 +1 @@
+"""Stage 3 — Resolver. Mirrors ts.hocon ``src/internal/resolver/``."""

hocon/_internal/resolver/fold_self_ref.py

@@ -0,0 +1,282 @@
+"""Helpers for chained / value-interior self-referential-substitution support.
+
+Mirrors ts.hocon ``src/internal/resolver/fold-self-ref.ts`` (itself a port of
+go.hocon ``foldselfref.go`` and rs.hocon ``fold_self_ref.rs``). Folds
+occurrences of ``${key}`` inside a value about to be saved as ``prior_values[key]``
+against the OLD prior, so by induction every saved prior is self-ref-free — the
+fix for the N≥3 self-append chain overflow (#118 / #120).
+
+For the ts-known-violation item S13a.3, the reference behaviour is rs.hocon's;
+this port follows ts.hocon structurally, matching rs where they agree.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import cast
+
+from ...value import HoconArray, HoconObject, HoconValue
+from ..lexer.token import Segment
+from .types import (
+    ConcatPlaceholder,
+    ResObj,
+    ResolverValue,
+    SubstPlaceholder,
+    is_concat,
+    is_res_obj,
+    is_subst,
+)
+
+__all__ = [
+    "clone_resolver_value",
+    "contains_self_ref",
+    "contains_subst_by_path",
+    "fold_known_absent_self_ref",
+    "fold_nested_self_refs",
+    "fold_or_skip_prior",
+    "fold_self_ref",
+    "string_segments_to_key",
+    "subst_full_key",
+]
+
+_NON_BARE_KEY = re.compile(r"[^a-zA-Z0-9\-_]")
+
+
+def _hv_list(items: list[ResolverValue]) -> list[HoconValue]:
+    # Resolver containers transiently hold placeholders during folding; ts.hocon
+    # casts these ``as HoconValue``. The public array/object contract remains
+    # HoconValue-only, so the cast is contained to these builder sites.
+    return cast("list[HoconValue]", items)
+
+
+def _hv_dict(fields: dict[str, ResolverValue]) -> dict[str, HoconValue]:
+    return cast("dict[str, HoconValue]", fields)
+
+
+def string_segments_to_key(segments: list[str]) -> str:
+    parts = []
+    for t in segments:
+        if t == "" or _NON_BARE_KEY.search(t):
+            parts.append('"' + t.replace("\\", "\\\\").replace('"', '\\"') + '"')
+        else:
+            parts.append(t)
+    return ".".join(parts)
+
+
+def subst_full_key(s: SubstPlaceholder) -> str:
+    return string_segments_to_key([seg.text for seg in s.segments])
+
+
+def contains_self_ref(v: ResolverValue, full_key: str) -> bool:
+    if is_subst(v):
+        return not v.known_absent and subst_full_key(v) == full_key
+    if is_concat(v):
+        return any(contains_self_ref(n, full_key) for n in v.nodes)
+    if is_res_obj(v):
+        return any(contains_self_ref(f, full_key) for f in v.fields.values())
+    if isinstance(v, HoconArray):
+        return any(contains_self_ref(item, full_key) for item in v.items)
+    if isinstance(v, HoconObject):
+        return any(contains_self_ref(f, full_key) for f in v.fields.values())
+    return False
+
+
+def fold_self_ref(
+    v: ResolverValue, full_key: str, replacement: ResolverValue
+) -> ResolverValue:
+    if is_subst(v):
+        return replacement if subst_full_key(v) == full_key else v
+    if is_concat(v):
+        return ConcatPlaceholder(
+            [fold_self_ref(n, full_key, replacement) for n in v.nodes], v.line, v.col
+        )
+    if is_res_obj(v):
+        out = ResObj()
+        for k, val in v.fields.items():
+            out.fields[k] = fold_self_ref(val, full_key, replacement)
+        out.prior_values = dict(v.prior_values)
+        out.reset_keys = set(v.reset_keys)
+        return out
+    if isinstance(v, HoconArray):
+        return HoconArray(
+            _hv_list([fold_self_ref(item, full_key, replacement) for item in v.items])
+        )
+    if isinstance(v, HoconObject):
+        return HoconObject(
+            _hv_dict({k: fold_self_ref(val, full_key, replacement) for k, val in v.fields.items()})
+        )
+    return v
+
+
+def fold_known_absent_self_ref(
+    v: ResolverValue, full_key: str, replacement: ResolverValue
+) -> ResolverValue:
+    if is_subst(v):
+        return replacement if (v.known_absent and subst_full_key(v) == full_key) else v
+    if is_concat(v):
+        return ConcatPlaceholder(
+            [fold_known_absent_self_ref(n, full_key, replacement) for n in v.nodes],
+            v.line,
+            v.col,
+        )
+    if is_res_obj(v):
+        out = ResObj()
+        for k, val in v.fields.items():
+            out.fields[k] = fold_known_absent_self_ref(val, full_key, replacement)
+        out.prior_values = dict(v.prior_values)
+        out.reset_keys = set(v.reset_keys)
+        return out
+    if isinstance(v, HoconArray):
+        return HoconArray(
+            _hv_list(
+                [fold_known_absent_self_ref(item, full_key, replacement) for item in v.items]
+            )
+        )
+    if isinstance(v, HoconObject):
+        return HoconObject(
+            _hv_dict(
+                {
+                    k: fold_known_absent_self_ref(val, full_key, replacement)
+                    for k, val in v.fields.items()
+                }
+            )
+        )
+    return v
+
+
+def fold_or_skip_prior(
+    prior: ResolverValue, full_key: str, old: ResolverValue | None
+) -> ResolverValue | None:
+    """Three-way decision at a prior-save site (see ts.hocon fold-self-ref.ts)."""
+    if not contains_self_ref(prior, full_key):
+        return clone_resolver_value(prior)
+    if old is None:
+        return _fold_optional_self_ref_absent(prior, full_key)
+    return fold_self_ref(prior, full_key, old)
+
+
+def _fold_optional_self_ref_absent(
+    v: ResolverValue, full_key: str
+) -> ResolverValue | None:
+    if is_subst(v) and subst_full_key(v) == full_key:
+        if not v.optional:
+            return None
+        return SubstPlaceholder(
+            [Segment(seg.text, seg.line, seg.col) for seg in v.segments],
+            v.optional,
+            True,
+            v.list_suffix,
+            v.line,
+            v.col,
+            v.prefix_len,
+        )
+    if is_concat(v):
+        nodes: list[ResolverValue] = []
+        for node in v.nodes:
+            folded = _fold_optional_self_ref_absent(node, full_key)
+            if folded is None:
+                return None
+            nodes.append(folded)
+        return ConcatPlaceholder(nodes, v.line, v.col)
+    if is_res_obj(v):
+        out = ResObj()
+        for key, value in v.fields.items():
+            folded = _fold_optional_self_ref_absent(value, full_key)
+            if folded is None:
+                return None
+            out.fields[key] = folded
+        out.prior_values = dict(v.prior_values)
+        out.reset_keys = set(v.reset_keys)
+        return out
+    if isinstance(v, HoconArray):
+        items: list[ResolverValue] = []
+        for item in v.items:
+            folded = _fold_optional_self_ref_absent(item, full_key)
+            if folded is None:
+                return None
+            items.append(folded)
+        return HoconArray(_hv_list(items))
+    if isinstance(v, HoconObject):
+        obj_fields: dict[str, ResolverValue] = {}
+        for key, value in v.fields.items():
+            folded = _fold_optional_self_ref_absent(value, full_key)
+            if folded is None:
+                return None
+            obj_fields[key] = folded
+        return HoconObject(_hv_dict(obj_fields))
+    return clone_resolver_value(v)
+
+
+def clone_resolver_value(v: ResolverValue) -> ResolverValue:
+    """Deep-clone a ResolverValue. Scalars share their reference (immutable and
+    identity-observable via ``separator_values``)."""
+    if is_subst(v):
+        return SubstPlaceholder(
+            [Segment(seg.text, seg.line, seg.col) for seg in v.segments],
+            v.optional,
+            v.known_absent,
+            v.list_suffix,
+            v.line,
+            v.col,
+            v.prefix_len,
+        )
+    if is_concat(v):
+        return ConcatPlaceholder([clone_resolver_value(n) for n in v.nodes], v.line, v.col)
+    if is_res_obj(v):
+        out = ResObj()
+        for k, val in v.fields.items():
+            out.fields[k] = clone_resolver_value(val)
+        for k, val in v.prior_values.items():
+            out.prior_values[k] = clone_resolver_value(val)
+        out.reset_keys = set(v.reset_keys)
+        return out
+    if isinstance(v, HoconArray):
+        return HoconArray(_hv_list([clone_resolver_value(item) for item in v.items]))
+    if isinstance(v, HoconObject):
+        return HoconObject(
+            _hv_dict({k: clone_resolver_value(val) for k, val in v.fields.items()})
+        )
+    return v
+
+
+def fold_nested_self_refs(v: ResolverValue, path_prefix: list[str]) -> ResolverValue:
+    """Recursively fold nested self-refs inside a ResObj tree using each enclosing
+    ResObj's ``prior_values`` as the substitution target (multi-segment
+    object-merge case, #120-class)."""
+    if not is_res_obj(v):
+        return v
+    out = ResObj()
+    for k, field_val in v.fields.items():
+        child_path = [*path_prefix, k]
+        full_key = string_segments_to_key(child_path)
+        folded = fold_nested_self_refs(field_val, child_path)
+        final_val = folded
+        if contains_self_ref(folded, full_key):
+            leaf_prior = v.prior_values.get(k)
+            if leaf_prior is not None:
+                leaf_prior_folded = fold_nested_self_refs(leaf_prior, child_path)
+                final_val = fold_self_ref(folded, full_key, leaf_prior_folded)
+        out.fields[k] = final_val
+    out.prior_values = dict(v.prior_values)
+    out.reset_keys = set(v.reset_keys)
+    return out
+
+
+def contains_subst_by_path(v: ResolverValue, target: list[Segment]) -> bool:
+    if is_subst(v):
+        return not v.known_absent and _segments_text_equal(v.segments, target)
+    if is_concat(v):
+        return any(contains_subst_by_path(n, target) for n in v.nodes)
+    if is_res_obj(v):
+        return any(contains_subst_by_path(f, target) for f in v.fields.values())
+    if isinstance(v, HoconArray):
+        return any(contains_subst_by_path(item, target) for item in v.items)
+    if isinstance(v, HoconObject):
+        return any(contains_subst_by_path(f, target) for f in v.fields.values())
+    return False
+
+
+def _segments_text_equal(a: list[Segment], b: list[Segment]) -> bool:
+    if len(a) != len(b):
+        return False
+    return all(a[i].text == b[i].text for i in range(len(a)))

hocon/_internal/resolver/include_loader.py

@@ -0,0 +1,252 @@
+"""Include loading (file / package). Mirrors ts.hocon ``src/internal/resolver/include-loader.ts``.
+
+URL and classpath includes are unsupported by design across all siblings
+(S14a.2 etc.). Node's ``require.resolve`` has no Python analogue, so
+``include package("id", "file")`` resolves via a filesystem convention (search
+``resolve_from`` / ``base_dir`` / CWD for ``<base>/id/file``) or a caller-supplied
+``package_resolver`` hook. Sync path only (py.hocon has no async parse variant).
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import os
+import re
+from collections.abc import Callable
+
+from ...errors import PackageLookupError, ParseError, ResolveError
+from ..lexer.lexer import tokenize
+from ..parser.ast import AstNode
+from ..parser.empty_check import assert_non_empty_document, has_content_tokens
+from ..parser.parser import parse_tokens
+from ..properties.properties import properties_to_hocon_value
+from .types import PackageResolver, ResObj, ResolveOptions, make_res_obj
+from .utils import (
+    deep_merge_res_obj_into,
+    hocon_value_to_res_obj,
+    is_file_not_found_error,
+)
+
+__all__ = ["IncludeLoader"]
+
+
+def _validate_package_file(file: str, identifier: str) -> None:
+    """Validate the file argument of ``package("id", "file")`` per E11 decision 6."""
+    if len(file) == 0:
+        raise ParseError(
+            f'include package("{identifier}", ...): file argument must be non-empty', 0, 0
+        )
+    if "\\" in file:
+        raise ParseError(
+            f'include package("{identifier}", "{file}"): backslash not allowed in file '
+            f"argument (use forward slash)",
+            0,
+            0,
+        )
+    if file.startswith("/"):
+        raise ParseError(
+            f'include package("{identifier}", "{file}"): absolute path not allowed '
+            f"in file argument",
+            0,
+            0,
+        )
+    if re.search(r"(?:^|/)\.\.?(?:/|$)", file):
+        raise ParseError(
+            f'include package("{identifier}", "{file}"): path traversal (. or ..) not '
+            f"allowed in file argument",
+            0,
+            0,
+        )
+    if "//" in file:
+        raise ParseError(
+            f'include package("{identifier}", "{file}"): consecutive slashes not allowed '
+            f"in file argument",
+            0,
+            0,
+        )
+
+
+def _validate_package_identifier(identifier: str) -> None:
+    if len(identifier) == 0:
+        raise ParseError("include package: identifier argument must be non-empty", 0, 0)
+
+
+def _default_package_resolver(
+    identifier: str,
+    file: str,
+    including_file: str | None,
+    base_dir: str | None,
+    resolve_from: object,
+) -> str:
+    """Filesystem-convention resolver: search candidate bases for ``id/file``.
+
+    Node's module resolution is not available in Python, so this looks for
+    ``<base>/<identifier>/<file>`` under, in priority order: ``resolve_from`` >
+    ``base_dir`` > dir of ``including_file`` > CWD. Raises PackageLookupError on
+    miss. Callers needing package-manager semantics pass a custom resolver.
+    """
+    if resolve_from:
+        bases = (
+            list(resolve_from)
+            if isinstance(resolve_from, (list, tuple))
+            else [str(resolve_from)]
+        )
+    elif base_dir:
+        bases = [base_dir]
+    elif including_file:
+        bases = [os.path.dirname(including_file)]
+    else:
+        bases = [os.getcwd()]
+
+    for base in bases:
+        candidate = os.path.join(base, identifier, file)
+        if os.path.isfile(candidate):
+            # E11 decision 5: case-sensitive basename check.
+            expected_basename = file.split("/")[-1]
+            if os.path.basename(candidate) != expected_basename:
+                raise PackageLookupError(
+                    f'include package("{identifier}", "{file}"): case mismatch',
+                    identifier,
+                    file,
+                    0,
+                    0,
+                )
+            return candidate
+
+    raise PackageLookupError(
+        f'include package("{identifier}", "{file}"): module not found '
+        f"(searched: {', '.join(bases)})",
+        identifier,
+        file,
+        0,
+        0,
+    )
+
+
+def _unset_build(_ast: AstNode, _opts: ResolveOptions) -> ResObj:  # pragma: no cover
+    raise RuntimeError("IncludeLoader.on_build_res_obj was not set by StructureBuilder")
+
+
+class IncludeLoader:
+    def __init__(self, opts: ResolveOptions) -> None:
+        self.opts = opts
+        # Set by StructureBuilder to delegate back to build() (avoids a cycle).
+        self.on_build_res_obj: Callable[[AstNode, ResolveOptions], ResObj] = _unset_build
+
+    def _resolve_include_path(
+        self, include_path: str, base_dir: str | None, is_file: bool
+    ) -> str:
+        if is_file:
+            return os.path.abspath(include_path)
+        if base_dir:
+            return os.path.abspath(os.path.join(base_dir, include_path))
+        return os.path.abspath(include_path)
+
+    def load(self, include_path: str, required: bool, is_file: bool = False) -> ResObj:
+        base_dir = self.opts.base_dir
+        include_stack = self.opts.include_stack
+        abs_path = self._resolve_include_path(include_path, base_dir, is_file)
+
+        if abs_path in include_stack:
+            raise ResolveError(f"circular include: {abs_path}", abs_path, 0, 0)
+        if len(include_stack) >= 50:
+            raise ResolveError("include depth limit exceeded (max 50)", include_path, 0, 0)
+
+        has_explicit_ext = abs_path.endswith((".conf", ".json", ".properties"))
+
+        if has_explicit_ext:
+            result = self._load_single(abs_path)
+            if result is not None:
+                return result
+            if required:
+                raise ResolveError(
+                    f"required include file not found: {include_path}", include_path, 0, 0
+                )
+            return make_res_obj()
+
+        # No extension: merge all found extensions (.properties, .json, .conf; last wins).
+        merged = make_res_obj()
+        found_any = False
+        for ext in (".properties", ".json", ".conf"):
+            obj = self._load_single(f"{abs_path}{ext}")
+            if obj is not None:
+                deep_merge_res_obj_into(merged, obj, [])
+                found_any = True
+
+        if not found_any and required:
+            raise ResolveError(
+                f"required include file not found: {include_path}", include_path, 0, 0
+            )
+        return merged
+
+    def load_package(self, identifier: str, file: str, _required: bool) -> ResObj:
+        _validate_package_identifier(identifier)
+        _validate_package_file(file, identifier)
+
+        include_stack = self.opts.include_stack
+        cycle_key = f'package\x00{identifier}\x00{file}'
+
+        if cycle_key in include_stack:
+            raise ResolveError(
+                f'circular include: package("{identifier}", "{file}")',
+                f"{identifier}/{file}",
+                0,
+                0,
+            )
+        if len(include_stack) >= 50:
+            raise ResolveError(
+                "include depth limit exceeded (max 50)", f"{identifier}/{file}", 0, 0
+            )
+
+        resolver: PackageResolver = self.opts.package_resolver or (
+            lambda id_, f, including_file, b_dir: _default_package_resolver(
+                id_, f, including_file, b_dir, self.opts.resolve_from
+            )
+        )
+        resolved_path = resolver(identifier, file, None, self.opts.base_dir)
+
+        content = self.opts.read_file_sync(resolved_path)
+        # Empty content (zero bytes) is valid for package includes (ipk08) → {}.
+        if len(content) == 0:
+            return make_res_obj()
+
+        tokens = tokenize(content)
+        assert_non_empty_document(tokens, resolved_path)
+        ast = parse_tokens(tokens)
+        return self.on_build_res_obj(
+            ast,
+            dataclasses.replace(
+                self.opts,
+                base_dir=os.path.dirname(resolved_path),
+                include_stack=[*include_stack, cycle_key],
+            ),
+        )
+
+    def _load_single(self, candidate: str) -> ResObj | None:
+        include_stack = self.opts.include_stack
+        if candidate in include_stack:
+            raise ResolveError(f"circular include: {candidate}", candidate, 0, 0)
+
+        try:
+            content = self.opts.read_file_sync(candidate)
+        except Exception as e:
+            if is_file_not_found_error(e):
+                return None
+            raise
+
+        if candidate.endswith(".properties"):
+            return hocon_value_to_res_obj(properties_to_hocon_value(content))
+
+        tokens = tokenize(content)
+        # Lightbend-compat carve-out (#105): empty/comment-only included file → {}.
+        if not has_content_tokens(tokens):
+            return make_res_obj()
+        ast = parse_tokens(tokens)
+        return self.on_build_res_obj(
+            ast,
+            dataclasses.replace(
+                self.opts,
+                base_dir=os.path.dirname(candidate),
+                include_stack=[*include_stack, candidate],
+            ),
+        )