framework-py 0.1.0.dev0__py3-none-any.whl

framework/spec/merge.py

@@ -0,0 +1,195 @@
+"""The layer merge shared by every Framework engine.
+
+This is the part of ``spec_resolve()`` that R, Python, and ``fw`` must agree
+on exactly. The shared test cases are the conformance fixtures in the R
+repository (``inst/conformance/merge``). Package defaults are not part of the
+merge: each engine passes its own as the first layer.
+
+Rules:
+
+- Objects merge by key, and a later layer wins.
+- A key set to null removes it.
+- Entries of ``files``, ``skills``, ``extra_directories``,
+  ``packages.default_packages``, ``connections.databases``, and
+  ``connections.storage_buckets`` are replaced whole.
+- An empty object or array changes nothing, but makes sure the section exists.
+- ``git.gitignore_content`` is collected per layer instead of merged.
+- A key set to different values by more than one layer (package defaults
+  excepted) is a conflict.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+from typing import Any
+
+from framework.errors import FrameworkError
+
+PACKAGE_DEFAULTS = "package defaults"
+
+# Sections whose entries are replaced whole.
+ATOMIC_MAPS = frozenset(
+    {
+        "files",
+        "skills",
+        "extra_directories",
+        "packages/default_packages",
+        "connections/databases",
+        "connections/storage_buckets",
+    }
+)
+
+# Sections that must be objects keyed by name.
+MAP_PATHS: tuple[tuple[str, ...], ...] = (
+    ("directories",),
+    ("extra_directories",),
+    ("render_dirs",),
+    ("env",),
+    ("data_hooks",),
+    ("files",),
+    ("skills",),
+    ("packages", "default_packages"),
+    ("git", "hooks"),
+    ("ai", "assistants"),
+    ("connections", "databases"),
+    ("connections", "storage_buckets"),
+)
+
+_ENVELOPE_KEYS = frozenset({"kind", "key", "version", "layer"})
+_REMOVED = "<removed>"
+
+
+class SpecError(FrameworkError):
+    """A layer or spec is malformed."""
+
+
+@dataclass(frozen=True)
+class Layer:
+    label: str
+    layer: dict[str, Any]
+
+
+@dataclass(frozen=True)
+class GitignoreBlock:
+    label: str
+    text: str
+
+
+@dataclass(frozen=True)
+class Conflict:
+    path: str
+    layers: str
+
+
+@dataclass
+class MergeResult:
+    spec: dict[str, Any] = field(default_factory=dict)
+    gitignore_blocks: list[GitignoreBlock] = field(default_factory=list)
+    setters: dict[str, list[tuple[str, Any]]] = field(default_factory=dict)
+
+    @property
+    def conflicts(self) -> list[Conflict]:
+        return conflicts(self.setters)
+
+
+def normalize_layers(layers: Sequence[Any]) -> list[Layer]:
+    """Label each layer. An envelope ``{kind, key, version, layer}`` is unwrapped."""
+    normalized = []
+    for index, entry in enumerate(layers, start=1):
+        if isinstance(entry, Layer):
+            normalized.append(entry)
+            continue
+        is_envelope = (
+            isinstance(entry, Mapping) and "layer" in entry and set(entry) <= _ENVELOPE_KEYS
+        )
+        if is_envelope:
+            parts = [str(entry.get("kind") or f"layer {index}")]
+            if entry.get("key") is not None:
+                parts.append(str(entry["key"]))
+            if entry.get("version") is not None:
+                parts.append(f"v{entry['version']}")
+            normalized.append(Layer(" ".join(parts), dict(entry.get("layer") or {})))
+        else:
+            normalized.append(Layer(f"layer {index}", dict(entry or {})))
+    return normalized
+
+
+def _get(data: Mapping[str, Any], path: tuple[str, ...]) -> Any:
+    value: Any = data
+    for part in path:
+        if not isinstance(value, Mapping):
+            return None
+        value = value.get(part)
+    return value
+
+
+def _merge(
+    base: dict[str, Any],
+    layer: Mapping[str, Any],
+    path: tuple[str, ...],
+    label: str,
+    setters: dict[str, list[tuple[str, Any]]],
+) -> dict[str, Any]:
+    for key, value in layer.items():
+        key_path = (*path, key)
+        path_str = "/".join(key_path)
+
+        if value is None:
+            base.pop(key, None)
+            setters.setdefault(path_str, []).append((label, _REMOVED))
+            continue
+
+        atomic = "/".join(path) in ATOMIC_MAPS
+        current = base.get(key)
+        if (
+            not atomic
+            and isinstance(value, Mapping)
+            and (current is None or isinstance(current, dict))
+        ):
+            base[key] = _merge(dict(current or {}), value, key_path, label, setters)
+        elif isinstance(value, list) and not value:
+            base.setdefault(key, {})
+        else:
+            base[key] = value
+            setters.setdefault(path_str, []).append((label, value))
+    return base
+
+
+def merge_layers(layers: Sequence[Any]) -> MergeResult:
+    """Merge layers, bottom to top, with the shared rules."""
+    result = MergeResult()
+    for entry in normalize_layers(layers):
+        layer = dict(entry.layer)
+        for path in MAP_PATHS:
+            value = _get(layer, path)
+            if isinstance(value, list) and value:
+                raise SpecError(
+                    f"Layer '{entry.label}': {'.'.join(path)} must be an object keyed by name, "
+                    "not a list"
+                )
+        git = layer.get("git")
+        if isinstance(git, Mapping):
+            text = git.get("gitignore_content")
+            if isinstance(text, list):
+                text = "\n".join(str(line) for line in text)
+            if text:
+                result.gitignore_blocks.append(GitignoreBlock(entry.label, str(text)))
+            layer["git"] = {k: v for k, v in git.items() if k != "gitignore_content"}
+        result.spec = _merge(result.spec, layer, (), entry.label, result.setters)
+    return result
+
+
+def conflicts(setters: Mapping[str, list[tuple[str, Any]]]) -> list[Conflict]:
+    """Keys set to different values by more than one layer, package defaults excepted."""
+    found = []
+    for path, entries in setters.items():
+        relevant = [(label, value) for label, value in entries if label != PACKAGE_DEFAULTS]
+        labels = list(dict.fromkeys(label for label, _ in relevant))
+        values: list[Any] = []
+        for _, value in relevant:
+            if value not in values:
+                values.append(value)
+        if len(labels) > 1 and len(values) > 1:
+            found.append(Conflict(path, ", ".join(labels)))
+    return found

framework/spec/resolve.py

@@ -0,0 +1,254 @@
+"""Resolving a project spec from layers, with Python's package defaults."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+from importlib import resources
+from typing import Any
+
+from framework import _yaml
+from framework.spec.check import spec_check
+from framework.spec.merge import (
+    MAP_PATHS,
+    PACKAGE_DEFAULTS,
+    Conflict,
+    GitignoreBlock,
+    Layer,
+    SpecError,
+    merge_layers,
+    normalize_layers,
+)
+
+TYPES = ("project", "project_sensitive", "course", "presentation", "bare")
+LANGUAGES = ("r", "python", "stata")
+LANGUAGE = "python"
+
+_ORDER = (
+    "spec_version",
+    "name",
+    "type",
+    "language",
+    "author",
+    "directories",
+    "extra_directories",
+    "render_dirs",
+    "packages",
+    "git",
+    "ai",
+    "scaffold",
+    "quarto",
+    "connections",
+    "security",
+    "env",
+    "data_hooks",
+    "files",
+    "skills",
+    "source",
+)
+_AFFILIATION_FIELDS = ("name", "department", "city", "country")
+
+
+def language_message(language: str) -> str:
+    return {
+        "r": "This is an R project. Create it with the framework R package or `fw new`.",
+        "stata": "This is a Stata project. Create it with `fw new` (the fw command-line tool).",
+    }.get(language, f"The Python package creates Python projects only (got language '{language}').")
+
+
+def _template(name: str) -> str:
+    return resources.files("framework").joinpath(f"templates/{name}").read_text(encoding="utf-8")
+
+
+def _defaults() -> dict[str, Any]:
+    data: dict[str, Any] = _yaml.load(_template("defaults.yml"))
+    return data
+
+
+def package_defaults(type: str) -> dict[str, Any]:
+    """Python's package defaults for a project type: the bottom layer of every spec."""
+    defaults = _defaults()
+    type_config = defaults["types"][type]
+    skills: dict[str, Any] = {}
+    if type != "bare":
+        names = list(defaults["skills"])
+        if type == "project_sensitive":
+            names += defaults["sensitive_skills"]
+        skills = {name: {"package": True} for name in names}
+    quarto = dict(defaults["quarto"])
+    if type_config.get("render_dir") is not None:
+        quarto["render_dir"] = type_config["render_dir"]
+    return {
+        "author": dict(defaults["author"]),
+        "directories": dict(type_config["directories"]),
+        "extra_directories": {},
+        "render_dirs": dict(type_config["render_dirs"]),
+        "packages": defaults["packages"],
+        "git": defaults["git"],
+        "ai": defaults["ai"],
+        "scaffold": defaults["scaffold"],
+        "quarto": quarto,
+        "connections": defaults["connections"],
+        "security": {"sensitive": type == "project_sensitive"},
+        "env": defaults["env"],
+        "data_hooks": {},
+        "files": {},
+        "skills": skills,
+    }
+
+
+# Authors ----------------------------------------------------------------------
+
+
+def _text(value: Any) -> str:
+    return value if isinstance(value, str) else ""
+
+
+def author_normalize(author: Mapping[str, Any] | None) -> dict[str, Any]:
+    """Known author fields with non-empty values, as R's ``.author_normalize()`` keeps them."""
+    author = author or {}
+    result: dict[str, Any] = {
+        "name": _text(author.get("name")),
+        "email": _text(author.get("email")),
+        "affiliation": _text(author.get("affiliation")),
+    }
+    for key in ("orcid", "url"):
+        if _text(author.get(key)):
+            result[key] = author[key]
+    affiliations = []
+    for entry in author.get("affiliations") or []:
+        if isinstance(entry, str):
+            entry = {"name": entry}
+        if not isinstance(entry, Mapping):
+            continue
+        kept = {k: entry[k] for k in _AFFILIATION_FIELDS if _text(entry.get(k))}
+        if kept.get("name"):
+            affiliations.append(kept)
+    if affiliations:
+        result["affiliations"] = affiliations
+        if not result["affiliation"]:
+            result["affiliation"] = affiliations[0]["name"]
+    return result
+
+
+def author_merge(
+    base: Mapping[str, Any] | None, override: Mapping[str, Any] | None
+) -> dict[str, Any]:
+    """Non-empty override fields replace the base, as R's ``.author_merge()`` does."""
+    merged = author_normalize(base)
+    override = override or {}
+    for key in ("name", "email", "affiliation", "orcid", "url"):
+        if _text(override.get(key)):
+            merged[key] = override[key]
+    if override.get("affiliations") is not None:
+        merged["affiliations"] = override["affiliations"]
+        if not _text(override.get("affiliation")):
+            merged["affiliation"] = ""
+    return author_normalize(merged)
+
+
+# Resolution -------------------------------------------------------------------
+
+
+@dataclass
+class ResolvedSpec:
+    """A complete project spec and the keys more than one layer set."""
+
+    spec: dict[str, Any]
+    conflicts: list[Conflict] = field(default_factory=list)
+
+
+def _gitignore(type: str, sensitive: bool, blocks: Sequence[GitignoreBlock]) -> str:
+    template = (
+        "sensitive"
+        if sensitive
+        else {"course": "course", "presentation": "presentation"}.get(
+            type, "sensitive" if type == "project_sensitive" else "project"
+        )
+    )
+    base = _template(f"gitignore/{template}").rstrip("\n")
+    extra = [f"\n# --- from {block.label} ---\n{block.text}" for block in blocks]
+    return "\n".join([base, *extra])
+
+
+def _mark_maps(spec: dict[str, Any]) -> None:
+    """Make sure every keyed section exists, as an empty object if unset."""
+    for path in MAP_PATHS:
+        node = spec
+        for part in path[:-1]:
+            node = node.setdefault(part, {})
+        if not node.get(path[-1]):
+            node[path[-1]] = {}
+
+
+def _finalize(spec: dict[str, Any], blocks: Sequence[GitignoreBlock]) -> dict[str, Any]:
+    spec["spec_version"] = 1
+    security = spec.setdefault("security", {})
+    security["sensitive"] = security.get("sensitive") is True
+    skills = spec.setdefault("skills", {})
+    if (
+        security["sensitive"]
+        and spec["type"] != "bare"
+        and "framework-sensitive-data" not in skills
+    ):
+        skills["framework-sensitive-data"] = {"package": True}
+
+    ai = spec.setdefault("ai", {})
+    ai["assistants"] = {k: True for k, v in (ai.get("assistants") or {}).items() if v is True}
+    ai.setdefault("agents_md", "stub")
+
+    spec.setdefault("git", {})["gitignore_content"] = _gitignore(
+        spec["type"], security["sensitive"], blocks
+    )
+    _mark_maps(spec)
+    ordered = {key: spec[key] for key in _ORDER if key in spec}
+    ordered.update({key: value for key, value in spec.items() if key not in ordered})
+    return ordered
+
+
+def spec_resolve(
+    layers: Sequence[Any] = (),
+    name: str = "",
+    author: Mapping[str, Any] | None = None,
+    check: bool = True,
+) -> ResolvedSpec:
+    """Merge layers over Python's package defaults into one complete project spec.
+
+    The project type comes from the last layer that sets ``type`` (default
+    ``"project"``), and the language from the last layer that sets
+    ``language`` (default ``"r"``, as in R). This package builds Python
+    projects only, so the language must resolve to ``"python"``.
+
+    Args:
+        layers: Layers, bottom to top. Each is a layer in spec shape, or an
+            envelope ``{kind, key, version, layer}`` as in a layers file.
+        name: The project name. It overrides any name in the layers.
+        author: Non-empty fields override the layers' author.
+        check: Raise ``SpecError`` when the result is invalid.
+    """
+    if not name:
+        raise SpecError("A project name is required")
+    entries = normalize_layers(layers)
+    type, language = "project", "r"
+    for entry in entries:
+        type = entry.layer.get("type") or type
+        language = entry.layer.get("language") or language
+    if type not in TYPES:
+        raise SpecError(f"Unknown project type '{type}'. Use one of: {', '.join(TYPES)}")
+    if language != LANGUAGE:
+        raise SpecError(language_message(language))
+
+    merged = merge_layers([Layer(PACKAGE_DEFAULTS, package_defaults(type)), *entries])
+    spec = merged.spec
+    spec["type"], spec["language"], spec["name"] = type, language, name
+    if author is not None:
+        spec["author"] = author_merge(spec.get("author"), author)
+    spec = _finalize(spec, merged.gitignore_blocks)
+
+    if check:
+        problems = spec_check(spec)
+        if problems:
+            raise SpecError(
+                "The resolved project spec is invalid:\n" + "\n".join(f"  - {p}" for p in problems)
+            )
+    return ResolvedSpec(spec, merged.conflicts)