tan-cli 0.7.0__py3-none-any.whl

tan/commands/build/token_substitution.py

@@ -0,0 +1,381 @@
+# SPDX-License-Identifier: Apache-2.0
+"""The IO/glue side of build-plan token substitution (alp-sdk #865): resolves
+the SDK root, project root and planner python, drives the pure
+`tan.core.plan_tokens` pass, and runs the one guard that pass cannot do
+itself -- comparing the plan's `sdkCommit` against the resolved SDK
+checkout's actual git HEAD (a subprocess call, so it belongs here, not in the
+pure module).
+
+`board_yaml_path`, `sdk_root`, `python` and `toolchain_root` are each
+resolved exactly ONCE by the caller and handed in -- never re-resolved here
+per-slice or on retry, the same "one resolution for the whole plan" contract
+`tan_core::plan_tokens::TokenValues` documents. `toolchain_advice` travels
+WITH `toolchain_root` for that reason: it is the second half of one
+resolution (`toolchain.ToolchainResolution`), not an independent knob, and
+re-deriving it here would let the reason drift from the value it explains."""
+import os
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+
+from tan.commands.build.toolchain import NO_TOOLCHAIN_ADVICE
+from tan.commands.sdk_cmd import NO_SDK_NEXT_STEPS
+from tan.core.build_plan import BuildPlan
+from tan.core.plan_tokens import (
+    DeferredPlaceholder,
+    DeferredPolicy,
+    LeftoverToken,
+    PLAN_TOKEN_NAMES,
+    PLAN_PATH_MODE_TOKENED,
+    TokenValues,
+    UnknownPlanPathMode,
+    UnresolvedToolchainRoot,
+    project_root_diverges_from_exec_base,
+    sdk_commit_mismatches,
+    substitute_plan_tokens,
+)
+from tan.envelope import Issue
+from tan.core.subprocess_env import spawn_env
+from tan.core.sysbuild_image import rescope_sysbuild_extra_conf
+from tan.core.tool_lookup import resolve_tool
+
+
+class TokenSubstitutionError(Exception):
+    def __init__(self, code: str, message: str) -> None:
+        super().__init__(message)
+        self.code = code
+        self.message = message
+
+
+@dataclass(frozen=True)
+class SliceDemotion:
+    """A slice demoted by `substitute_plan_tokens` (tan-cli #89): its fields
+    still name a literal `${TOOLCHAIN_ROOT}` because this host has none
+    resolved, but the demotion has an owning slice AND an owning dispatch
+    seam (`executionPolicy.missingTool`) to route to, so it did not fail the
+    whole plan. `reason` is the full sentence the executor surfaces verbatim
+    at dispatch."""
+
+    slice_index: int
+    core_id: str
+    reason: str
+
+
+def _resolve_git_executable() -> str | None:
+    """The absolute path to `git`, resolved through the shared
+    `tool_lookup.resolve_tool` walk -- never `shutil.which`, which on Windows
+    inserts the current directory ahead of `%PATH%`. `None` when `git` is not
+    on PATH at all, which both callers below treat as "no git provenance",
+    never a crash (tan-cli#797, mirroring `doctor_cmd._resolve_git_executable`
+    -- the same fix, duplicated here rather than imported for the same reason
+    `_is_own_git_checkout`'s own docstring gives: this is the second of the
+    two sites the issue names, not a shared library call).
+    """
+    return resolve_tool("git", os.environ).resolved
+
+
+def _is_own_git_checkout(sdk_root: Path, git_exe: str | None) -> bool:
+    """Whether `sdk_root` is itself the TOP of a git checkout, not merely
+    nested somewhere inside an unrelated ENCLOSING one (tan-cli#488 defect 4,
+    mirroring `doctor_cmd._is_own_git_checkout` -- the SAME defect, the SAME
+    fix, duplicated here rather than imported because this is the second of
+    the two sites the issue names, not a shared library call).
+
+    `git -C <root> ...` discovery walks UPWARD looking for a `.git`, so an SDK
+    with no `.git` of its own -- an extracted release archive vendored under a
+    customer's own application repository, a setup this port explicitly
+    supports (`sdk_cmd.check_sdk_readiness`'s own docstring names exactly this
+    shape) -- answers every git query with the ENCLOSING repo's state instead
+    of "not a checkout". `git_short_head` below calls this first and returns
+    no signal (`""`) rather than misattributing a foreign repository's HEAD as
+    the SDK's own -- which, left unguarded, both stamps a build plan's
+    `sdkCommit` with the wrong provenance and lets `sdk_commit_mismatches` fire
+    a false split-brain refusal (or miss a real one) keyed off the enclosing
+    app repo's commit instead of the SDK's.
+
+    Compares the RESOLVED `--show-toplevel` against the RESOLVED `sdk_root`:
+    lexical string comparison alone would false-negative on a symlinked or
+    differently-cased path to the same directory.
+
+    `git_exe` -- the PATH-RESOLVED absolute `git` from `_resolve_git_executable`
+    (tan-cli#797): this runs on the BUILD path with cwd set to the CUSTOMER's
+    project, the Windows-reachable strong form of the cwd-shadowing hazard.
+    `None` refuses outright rather than falling back to a bare spawn.
+    """
+    if git_exe is None:
+        return False
+    try:
+        out = subprocess.run(
+            ["git", "-C", str(sdk_root), "rev-parse", "--show-toplevel"],
+            executable=git_exe,
+            capture_output=True,
+            text=True,
+            timeout=10,
+            env=spawn_env(),
+            check=False,
+        )
+    except (OSError, subprocess.SubprocessError):
+        return False
+    if out.returncode != 0:
+        return False
+    top = out.stdout.strip()
+    if not top:
+        return False
+    try:
+        return Path(top).resolve() == Path(sdk_root).resolve()
+    except OSError:
+        return False
+
+
+def git_short_head(sdk_root: Path) -> str:
+    """`git rev-parse --short HEAD`. Empty string when git is missing, the
+    checkout has no `.git` OF ITS OWN (see `_is_own_git_checkout` --
+    tan-cli#488 defect 4), or the command otherwise fails -- all NO SIGNAL,
+    never a hard failure: an SDK checkout with no `.git` (a release tarball,
+    including one vendored inside a customer's own git repository) is a
+    normal, supported setup.
+
+    `git` is resolved ONCE, through `_resolve_git_executable`, and threaded
+    into both the `_is_own_git_checkout` guard and the spawn below as
+    `executable=` (tan-cli#797) -- neither hands `subprocess` the bare
+    `"git"` identity.
+    """
+    git_exe = _resolve_git_executable()
+    if not _is_own_git_checkout(sdk_root, git_exe):
+        return ""
+    try:
+        out = subprocess.run(
+            ["git", "-C", str(sdk_root), "rev-parse", "--short", "HEAD"],
+            executable=git_exe,
+            capture_output=True,
+            text=True,
+            timeout=10,
+            env=spawn_env(),
+            check=False,
+        )
+    except (OSError, subprocess.SubprocessError):
+        return ""
+    return out.stdout.strip() if out.returncode == 0 else ""
+
+
+def _deferred_extra_note(plan: BuildPlan, e: LeftoverToken, policy: DeferredPolicy) -> str:
+    """What to append to a `build.plan-token-unresolved` message so the
+    refusal also says what a malformed `deferredPlaceholders` cost, or how a
+    config-artefact placeholder could have been accepted (tan-cli#1302)."""
+    notes = ""
+    if plan.deferred_placeholders_problem is not None:
+        notes += (
+            f" Note: {plan.deferred_placeholders_problem} -- the field was ignored, so no "
+            f"`${{NAME}}` placeholder was exempt."
+        )
+    elif e.deferrable:
+        name = e.token[2:-1]
+        if policy.has_plan_list:
+            notes += (
+                f" If `{e.token}` is a placeholder meant for the build host or the device, "
+                f"the plan's `deferredPlaceholders` has to list `{name}` "
+                f"(alplabai/alp-sdk#2696); it does not."
+            )
+        else:
+            notes += (
+                f" If `{e.token}` is a placeholder meant for the build host or the device, "
+                f"it is accepted only when the plan's `deferredPlaceholders` lists `{name}` "
+                f"(alplabai/alp-sdk#2696); this plan carries no `deferredPlaceholders` at "
+                f"all -- re-emit it with a current planner."
+            )
+    return notes
+
+
+def _deferred_text(d: DeferredPlaceholder) -> str:
+    if d.live_kconfig:
+        return (
+            f"placeholder `${{{d.name}}}` in `{d.field}` sits on a live line of a Kconfig "
+            f"fragment; Kconfig does not expand `${{{d.name}}}`, so the firmware will carry "
+            f"the literal text unless something substitutes it before the build"
+        )
+    return (
+        f"placeholder `${{{d.name}}}` in `{d.field}` is left as written for the build "
+        f"host or the device to supply; tan does not substitute it"
+    )
+
+
+def deferred_placeholder_issues(
+    plan: BuildPlan, deferred: list[DeferredPlaceholder]
+) -> list[Issue]:
+    """The envelope issues for tan-cli#1302: one issue per placeholder NAME
+    left standing in a config artefact (`info`, or `warning` when any
+    occurrence is on a live Kconfig line), plus `build.plan-warning`s for a
+    malformed `deferredPlaceholders` and for a plan path token listed in it."""
+    issues: list[Issue] = []
+    if plan.deferred_placeholders_problem is not None:
+        text = f"[deferred-placeholders-malformed] {plan.deferred_placeholders_problem} -- ignoring the field"
+        if plan.plan_path_mode == PLAN_PATH_MODE_TOKENED:
+            text += (
+                ", so no `${NAME}` placeholder is exempt from the unresolved-token refusal"
+            )
+        issues.append(Issue("build.plan-warning", "warning", text))
+    bugs = [n for n in plan.deferred_placeholders or () if n in PLAN_TOKEN_NAMES]
+    if bugs:
+        listed = ", ".join(f"`{n}`" for n in bugs)
+        issues.append(
+            Issue(
+                "build.plan-warning",
+                "warning",
+                f"[deferred-placeholders-plan-token] `deferredPlaceholders` lists the plan "
+                f"path token(s) {listed} -- a planner bug: tan substitutes those and they are "
+                f"never deferred; ignoring the entries",
+            )
+        )
+    by_name: dict[str, list[DeferredPlaceholder]] = {}
+    for d in deferred:
+        by_name.setdefault(d.name, []).append(d)
+    for occurrences in by_name.values():
+        pick = next((d for d in occurrences if d.live_kconfig), occurrences[0])
+        severity = "warning" if pick.live_kconfig else "info"
+        issues.append(Issue("build.deferred-placeholder", severity, _deferred_text(pick)))
+    return issues
+
+
+def apply_plan_token_substitution(
+    plan: BuildPlan,
+    *,
+    board_yaml_path: str | None,
+    exec_base: str,
+    sdk_root: str | None,
+    python: str,
+    toolchain_root: str | None,
+    toolchain_advice: str = NO_TOOLCHAIN_ADVICE,
+    deferred_out: list[DeferredPlaceholder] | None = None,
+    project_root: str | None = None,
+) -> tuple[BuildPlan, list[SliceDemotion]]:
+    """Apply the build-plan token-substitution pass to `plan` before
+    materialise writes anything or a slice command runs. A no-op unless
+    `plan.plan_path_mode` is present; a present-but-unrecognised mode is a
+    hard error, never a half-applied guess.
+
+    `toolchain_advice` is the reason an unresolved `${TOOLCHAIN_ROOT}` gets,
+    both in the plan-fatal refusal and in each demoted slice's `reason`.
+    It defaults to the no-toolchain wording this pass always used, so a
+    caller that has not resolved a toolchain at all -- and every existing
+    test -- keeps its exact previous message. `build_cmd` passes the
+    resolver's own reason instead, which distinguishes "this host has none"
+    from "this host has SEVERAL and nothing picking between them"
+    (tan-cli#547): two different fixes, and only the second one names the
+    installs the caller can choose from.
+
+    `deferred_out` (tan-cli#1302), when given, receives every `${NAME}` that
+    was deliberately left standing in a `configArtefacts[*].contents` value
+    (see `DeferredPolicy` for the exact rule). A caller that passes nothing
+    still gets the exemption -- it just cannot report it."""
+    if plan.plan_path_mode is None:
+        return plan, []
+    if plan.plan_path_mode != PLAN_PATH_MODE_TOKENED:
+        raise TokenSubstitutionError(
+            "build.plan-invalid",
+            f'unknown planPathMode `{plan.plan_path_mode}` (only "tokened" is defined) -- '
+            f"refusing rather than silently treating it as legacy or applying token "
+            f"substitution",
+        )
+
+    # Guard: PROJECT_ROOT (board.yaml's directory) vs the executor's actual
+    # base dir. They coincide only in the default config -- a tokened plan
+    # substituting ${PROJECT_ROOT} from one while slices actually run under
+    # the other would silently build against the wrong tree.
+    # `project_root` is the `board.yaml`-less route's own (tan-cli#1359: `tan
+    # build --board`): there is no board.yaml to derive it from, so the caller
+    # names it (the build root) instead.
+    if board_yaml_path is None and project_root is None:
+        raise TokenSubstitutionError(
+            "build.plan-invalid",
+            "a `planPathMode: tokened` plan needs a resolved board.yaml to derive "
+            "${PROJECT_ROOT} from -- pass `--board-yaml <PATH>` or run from a project.",
+        )
+    if project_root is None:
+        project_root = str(Path(board_yaml_path).parent).replace("\\", "/")
+
+    if project_root_diverges_from_exec_base(project_root, exec_base):
+        raise TokenSubstitutionError(
+            "build.project-root-mismatch",
+            f"plan is `planPathMode: tokened`, but ${{PROJECT_ROOT}} (`{project_root}`, the "
+            f"board.yaml directory) differs from where tan actually runs each slice's "
+            f"command (`{exec_base}`) -- refusing to build against the wrong tree. This only "
+            f"happens with a non-default `west_cwd`/`--project` split; align them, or drop "
+            f"`planPathMode` from the plan.",
+        )
+
+    # ONE resolved sdk_root for the whole substitution. A tokened plan NEEDS
+    # a real ${SDK_ROOT} value: degrading an unresolved sdk_root to "" would
+    # substitute ${SDK_ROOT}/scripts into the bare relative path /scripts,
+    # sailing straight past the leftover-token guard -- refuse instead.
+    if sdk_root is None:
+        # `tan sdk switch` refuses in this build (tan-cli#305) -- kept the
+        # two mechanisms `resolve_sdk_root_ladder` (this value's caller)
+        # actually walks (`--sdk-root`, the lateral discovery tail) and
+        # swapped the third for NO_SDK_NEXT_STEPS's honest "how to get a
+        # checkout at all".
+        raise TokenSubstitutionError(
+            "build.sdk-root-unresolved",
+            "plan is `planPathMode: tokened` (needs ${SDK_ROOT} substituted with a real "
+            "path), but no alp-sdk checkout resolved -- pass `--sdk-root <PATH>`, run from "
+            f"a project near an alp-sdk checkout, or {NO_SDK_NEXT_STEPS}.",
+        )
+
+    # The two-SDK split-brain guard: whenever the plan carries a sdkCommit,
+    # a mismatch against the resolved SDK checkout's actual HEAD means tan
+    # is about to build against a DIFFERENT SDK checkout/SHA than the plan
+    # was emitted from.
+    if plan.sdk_commit:
+        resolved = git_short_head(Path(sdk_root))
+        if sdk_commit_mismatches(plan.sdk_commit, resolved):
+            raise TokenSubstitutionError(
+                "build.sdk-commit-mismatch",
+                f"plan was emitted from alp-sdk commit `{plan.sdk_commit}`, but the resolved "
+                f"SDK checkout is at `{resolved}` -- building against a different SDK "
+                f"checkout than the plan was captured from can silently produce the wrong "
+                f"image. Re-emit the plan from the current SDK checkout, or point "
+                f"`--sdk-root` at the checkout the plan names.",
+            )
+
+    values = TokenValues(
+        sdk_root=sdk_root, project_root=project_root, python=python, toolchain_root=toolchain_root
+    )
+    policy = DeferredPolicy(plan.deferred_placeholders)
+    try:
+        out, demoted = substitute_plan_tokens(plan, values, policy)
+    except LeftoverToken as e:
+        raise TokenSubstitutionError(
+            "build.plan-token-unresolved",
+            f"plan is `planPathMode: tokened` but field `{e.field}` still names the literal "
+            f"token `{e.token}` after substitution -- an SDK-side token this CLI does not "
+            f"resolve (only ${{SDK_ROOT}}, ${{PROJECT_ROOT}}, ${{PYTHON}}, ${{TOOLCHAIN_ROOT}} "
+            f"are known). Upgrade tan, or check the plan for a bug."
+            + _deferred_extra_note(plan, e, policy),
+        ) from e
+    except UnresolvedToolchainRoot as e:
+        raise TokenSubstitutionError(
+            "build.toolchain-root-unresolved",
+            f"plan field `{e.field}` names ${{TOOLCHAIN_ROOT}}, but {toolchain_advice}. "
+            f"tan refuses rather than substituting an empty path, which would silently build "
+            f"against the host root.",
+        ) from e
+    except UnknownPlanPathMode as e:
+        raise TokenSubstitutionError("build.plan-invalid", str(e)) from e
+
+    if deferred_out is not None:
+        deferred_out.extend(policy.exempted)
+
+    # alp-sdk#866: a sysbuild slice's `-D<image>_EXTRA_CONF_FILE` prefix was
+    # derived from the project root the plan was EMITTED in; re-derive it from
+    # the substituted app dir so a relocated plan still configures its image
+    # (see `tan.core.sysbuild_image` for why this rewrites rather than refuses).
+    out = rescope_sysbuild_extra_conf(out)
+
+    slice_demotions = [
+        SliceDemotion(
+            slice_index=d.slice_index,
+            core_id=d.core_id,
+            reason=f"plan field `{d.field}` names ${{TOOLCHAIN_ROOT}}, but {toolchain_advice}",
+        )
+        for d in demoted
+    ]
+    return out, slice_demotions