tangier 0.2.1__py3-none-any.whl

tangier/__init__.py

@@ -0,0 +1,15 @@
+"""tangier — a content-addressed CI/deploy pipeline toolkit.
+
+Five concerns, one config (`pipeline.toml`):
+
+  changemap  which parts of the repo does this diff touch?
+  image      what is this bucket's content hash, and how do I build it?
+  deploy     render and apply k8s manifests for those image tags.
+  tailnet    can this machine reach the cluster, and as what identity?
+  gate       has this content already passed its gate?
+
+Pure stdlib. See `docs/specs/changemap.md` and `docs/specs/gate.md`.
+"""
+
+# Kept equal to `pyproject.toml` by a test. Gate records carry it.
+__version__ = "0.2.1"

tangier/__main__.py

@@ -0,0 +1,5 @@
+"""`python -m tangier` entry point."""
+
+from tangier.cli import cli_main
+
+cli_main()

tangier/changemap.py

@@ -0,0 +1,310 @@
+"""The answer set: which tags does this diff touch, and what are their SHAs?
+
+The single computed view that every `changemap` subcommand formats. Nothing
+here formats output or shells out to a cluster.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass
+
+from tangier import git, globs
+from tangier.config import Config
+
+# `from tangier import git` — NOT `from tangier.git import changed_files`. Tests
+# patch `git.changed_files`, and a direct-symbol import would bind a second
+# reference that the patch never reaches.
+
+
+@dataclass
+class AnswerSet:
+    matched: set[str]
+    expanded: set[str]
+    shas: dict[str, str]
+    items: dict[str, list[str]]
+    touched: dict[str, bool]
+    # File-set group name (e.g. `unittest-files`) -> directly-changed files
+    # selected by that group's globs. Drives each runner's `--files` flag.
+    file_sets: dict[str, list[str]]
+    ignored: list[str]
+
+
+def tag_matches(cfg: Config, tag: str, path: str) -> bool:
+    """Whether `path` belongs to `tag`: matches its `paths`, minus its `exclude`.
+
+    The single definition of tag membership — every match site (diff resolution
+    and SHA bucket hashing) goes through here so the two cannot diverge.
+    """
+    return globs.matches(path, cfg.paths.get(tag, [])) and not globs.matches(path, cfg.exclude.get(tag, []))
+
+
+# ---------------------------------------------------------------------------
+# SHA — content hash for a bucket
+# ---------------------------------------------------------------------------
+
+
+def sha_of_paths(paths: list[str], head: str, keep: Callable[[str], bool] | None = None) -> str:
+    """Hash the `git ls-tree -r` walk of `paths`, optionally filtered by `keep`.
+
+    `keep` receives each walked file path and decides whether it contributes —
+    the only way to express exclusion, since `ls-tree` has no negation.
+    `keep=None` skips the per-path call entirely; it is an optimisation, not a
+    parity mechanism, since a `keep` returning True for every line hashes
+    identically.
+
+    What byte-identical hashing actually rests on is here and in
+    `globs.globs_to_ls_tree_paths`: the exact `ls-tree` line text, joined with
+    `\\n` and hashed with no trailing newline, truncated to 10 hex chars. Change
+    any of that and every image tag in every consuming repo moves.
+
+    Policy-free by design — the caller composes `keep`, so `[sha] exclude` and
+    per-tag `exclude` meet in one place rather than two.
+    """
+    return _sha_of_lines(tree_lines(paths, head, keep))
+
+
+def tree_lines(paths: list[str], head: str, keep: Callable[[str], bool] | None = None) -> list[str]:
+    """The `git ls-tree -r` lines for `paths`, optionally filtered by `keep`."""
+    lines = git.ls_tree(head, paths)
+    return [line for line in lines if keep is None or keep(git.ls_tree_path(line))]
+
+
+def _sha_of_lines(lines: list[str]) -> str:
+    return hashlib.sha1("\n".join(lines).encode()).hexdigest()[:10]
+
+
+def transitive_deps(tag: str, depends: dict[str, list[str]]) -> set[str]:
+    """Transitive closure of `tag`'s forward dependencies (what `tag` depends on)."""
+    seen: set[str] = set()
+    queue = [tag]
+    while queue:
+        cur = queue.pop()
+        for dep in depends.get(cur, ()):
+            if dep not in seen:
+                seen.add(dep)
+                queue.append(dep)
+    return seen
+
+
+def buckets(cfg: Config) -> dict[str, list[str]]:
+    """bucket-name -> member tags (tags whose `sha` field maps here)."""
+    out: dict[str, list[str]] = {}
+    for tag, bucket in cfg.sha_bucket.items():
+        out.setdefault(bucket, []).append(tag)
+    return out
+
+
+def sha_for_bucket(cfg: Config, bucket: str, head: str = "HEAD") -> str:
+    """SHA of the union of a bucket's members' paths + their transitive deps' paths."""
+    return _sha_of_lines(scope_lines(cfg, buckets(cfg).get(bucket, []), head))
+
+
+def scope_lines(cfg: Config, tags: Iterable[str], head: str = "HEAD") -> list[str]:
+    """The `ls-tree` lines for `tags`' paths plus their transitive deps' paths.
+
+    A SHA bucket and a gate scope both hash these lines, so a package means
+    the same thing to an image and to a gate.
+
+    Contributing tags' `exclude` globs are honoured so the lines match
+    the tags' match sets. `git ls-tree` cannot express negation, so the file list
+    it returns is filtered afterwards. Exclusion is a property of the individual
+    tag, not the union: a path one contributing tag excludes still counts if
+    another contributing tag claims it (that tag genuinely ships the file).
+
+    Two filters compose here, and they are deliberately different:
+      - `[sha] exclude` applies unconditionally (docs never rebuild an image).
+      - the per-tag `claimed_by` filter applies only when some contributing tag
+        actually declares `exclude`.
+    Conflating them would change the hash of every bucket and scope that has no exclusions.
+    """
+    bucket_globs, keep = _scope_filter(cfg, tags)
+    return tree_lines(globs.globs_to_ls_tree_paths(bucket_globs), head, keep)
+
+
+def scope_touched(cfg: Config, tags: Iterable[str], files: Iterable[str]) -> bool:
+    """Whether any of `files` is one of `scope_lines`' inputs for `tags`.
+
+    The same globs and the same `keep` filter as `scope_lines`, so a file
+    touches the scope exactly when it can move the scope's lines. Gate scopes
+    hold only literal paths and `dir/**` globs, for which a glob match equals
+    membership in the `ls-tree` walk. A deleted file counts: it left the walk.
+    """
+    scope_globs, keep = _scope_filter(cfg, tags)
+    return any(globs.matches(f, scope_globs) and (keep is None or keep(f)) for f in files)
+
+
+def _scope_filter(cfg: Config, tags: Iterable[str]) -> tuple[list[str], Callable[[str], bool] | None]:
+    """The globs `tags` contribute, with their transitive deps, and the `keep` filter for them.
+
+    `keep` is None when neither filter applies (see `scope_lines`).
+    """
+    bucket_globs: list[str] = []
+    contributing: set[str] = set()
+    for m in tags:
+        # Set iteration, so the order of `bucket_globs` — and hence of the path
+        # arguments to `git ls-tree` — is not stable across runs. That is safe
+        # only because `git ls-tree -r` sorts and dedups its output regardless
+        # of argument order; the SHA depends on the output, not the arguments.
+        for tag in {m, *transitive_deps(m, cfg.depends)}:
+            if tag in contributing:
+                continue
+            contributing.add(tag)
+            bucket_globs.extend(cfg.paths.get(tag, []))
+
+    sha_exclude = cfg.sha.exclude
+    needs_claim_filter = any(cfg.exclude.get(tag) for tag in contributing)
+
+    def _keep(path: str) -> bool:
+        if sha_exclude and globs.matches(path, sha_exclude):
+            return False
+        if needs_claim_filter:
+            return any(tag_matches(cfg, tag, path) for tag in contributing)
+        return True
+
+    # Neither filter applies -> walk everything unfiltered, the byte-stable fast path.
+    return bucket_globs, _keep if (sha_exclude or needs_claim_filter) else None
+
+
+# ---------------------------------------------------------------------------
+# Dependency expansion
+# ---------------------------------------------------------------------------
+
+
+def _invert_depends(depends: dict[str, list[str]]) -> dict[str, set[str]]:
+    """Build dep -> {tags that directly depend on dep}."""
+    out: dict[str, set[str]] = {}
+    for tag, deps in depends.items():
+        for dep in deps:
+            out.setdefault(dep, set()).add(tag)
+    return out
+
+
+def expand_with_dependents(tags: Iterable[str], depends: dict[str, list[str]]) -> set[str]:
+    """Expand a matched tag set via the reverse-transitive closure of depends.
+
+    For each input tag X, union in every tag whose dependency closure
+    transitively contains X.
+    """
+    inverse = _invert_depends(depends)
+    result: set[str] = set(tags)
+    queue: list[str] = list(tags)
+    while queue:
+        cur = queue.pop()
+        for dependent in inverse.get(cur, ()):
+            if dependent not in result:
+                result.add(dependent)
+                queue.append(dependent)
+    return result
+
+
+def provenance(matched_raw: set[str], expanded: set[str], depends: dict[str, list[str]]) -> dict[str, set[str]]:
+    """For each tag in `expanded - matched_raw`, the matched-raw tags whose
+    dependents transitively include it."""
+    added = expanded - matched_raw
+    out: dict[str, set[str]] = {}
+    for tag in added:
+        sources = {src for src in matched_raw if tag in expand_with_dependents({src}, depends)}
+        if sources:
+            out[tag] = sources
+    return out
+
+
+# ---------------------------------------------------------------------------
+# Answer set
+# ---------------------------------------------------------------------------
+
+
+def match_files_to_tags(cfg: Config, files: Iterable[str]) -> tuple[dict[str, list[str]], list[str]]:
+    """Group files by the tag(s) they match. Returns (per_tag, ignored)."""
+    per_tag: dict[str, list[str]] = {}
+    ignored: list[str] = []
+    for f in files:
+        hit = False
+        for tag in cfg.paths:
+            if tag_matches(cfg, tag, f):
+                per_tag.setdefault(tag, []).append(f)
+                hit = True
+        if not hit:
+            ignored.append(f)
+    return per_tag, ignored
+
+
+def project_items(cfg: Config, name: str, expanded: set[str]) -> list[str]:
+    """Stable, deduped projection of one items name through the expanded set."""
+    tag_map = cfg.items.get(name, {})
+    seen: set[str] = set()
+    collected: list[str] = []
+    for tag in sorted(expanded):
+        for path in tag_map.get(tag, []):
+            if path not in seen:
+                seen.add(path)
+                collected.append(path)
+    return collected
+
+
+def compute_answer_set(
+    cfg: Config, base: str, head: str, *, expand: bool = True, compute_shas: bool = True
+) -> AnswerSet:
+    return answer_set_for_files(cfg, git.changed_files(base, head), head, expand=expand, compute_shas=compute_shas)
+
+
+def answer_set_for_files(
+    cfg: Config, files: list[str], head: str, *, expand: bool = True, compute_shas: bool = True
+) -> AnswerSet:
+    """The answer set for a list of changed files. `head` is read only for the bucket SHAs."""
+    per_tag, ignored = match_files_to_tags(cfg, files)
+    matched = set(per_tag.keys())
+    expanded = expand_with_dependents(matched, cfg.depends) if expand else set(matched)
+
+    items = {name: project_items(cfg, name, expanded) for name in cfg.items}
+
+    shas: dict[str, str] = {}
+    if compute_shas:
+        for bucket in sorted(buckets(cfg).keys()):
+            shas[bucket] = sha_for_bucket(cfg, bucket, head)
+
+    touched = {tag: (tag in expanded) for tag in cfg.touched}
+
+    # File-set projection: each `files = true` table's globs select the
+    # directly-changed files for a runner's `--files` flag. Membership is decided
+    # purely by the table's globs over the raw diff — scoped to the code each
+    # runner can actually execute. Files outside every file-set's globs silently
+    # drop out, the same ignore-by-default contract as the rest of changemap.
+    file_sets = {
+        group: [f for f in files if globs.matches(f, group_globs)] for group, group_globs in cfg.file_sets.items()
+    }
+
+    return AnswerSet(
+        matched=matched,
+        expanded=expanded,
+        shas=shas,
+        items=items,
+        touched=touched,
+        file_sets=file_sets,
+        ignored=ignored,
+    )
+
+
+def full_answer_set(cfg: Config, head: str, *, compute_shas: bool = True) -> AnswerSet:
+    """The answer set as if every tag changed: `--full`. Reads no diff, so it needs no merge base.
+
+    Each file-set holds every tracked file at `head` that its globs match. See
+    `[run-full]` in `docs/specs/gate.md`.
+    """
+    # `ls_tree` reads a bad ref as an empty tree, which would empty every file-set without a word.
+    _ = git.rev_parse_tree(head)
+    every_tag = set(cfg.paths)
+    tracked = [git.ls_tree_path(line) for line in git.ls_tree(head, [])]
+    return AnswerSet(
+        matched=every_tag,
+        expanded=every_tag,
+        shas={bucket: sha_for_bucket(cfg, bucket, head) for bucket in sorted(buckets(cfg))} if compute_shas else {},
+        items={name: project_items(cfg, name, every_tag) for name in cfg.items},
+        touched={tag: True for tag in cfg.touched},
+        file_sets={
+            group: [f for f in tracked if globs.matches(f, group_globs)] for group, group_globs in cfg.file_sets.items()
+        },
+        ignored=[],
+    )

tangier/cli.py

@@ -0,0 +1,164 @@
+"""tangier CLI."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+
+from tangier.commands import changemap_cmds, deploy_cmds, gate_cmds, image_cmds, tailnet_cmds
+from tangier.commands.args import add_diff_args, add_full
+from tangier.config import Config, ConfigError, read_config
+from tangier.deploy import DeployError
+from tangier.gate import GateError
+from tangier.git import GitError
+from tangier.image import ImageError
+from tangier.runner import Runner
+from tangier.tailnet import TailnetError
+
+# The config lives in the repo tangier runs from — each project carries its own
+# pipeline.toml at its root. TANGIER_CONFIG lets the parity harness point both
+# implementations at the same file without copying or symlinking it.
+DEFAULT_CONFIG = os.environ.get("TANGIER_CONFIG") or "pipeline.toml"
+
+
+def cli_main() -> None:
+    """Console-script entry point."""
+    raise SystemExit(main())
+
+
+def main(argv: list[str] | None = None, *, runner: Runner | None = None) -> int:
+    """Run one command. `runner` replaces the subprocess seam, for tests."""
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+    # The arguments as given, so a gate job can run the same command again.
+    args.argv = list(sys.argv[1:] if argv is None else argv)
+    if runner is not None:
+        args.runner = runner
+    if not getattr(args, "func", None):
+        parser.print_help()
+        return 1
+    try:
+        config = read_config(args.config)
+    except ConfigError as e:
+        print(f"config error: {e}", file=sys.stderr)
+        return 2
+    except FileNotFoundError as e:
+        # A repo with no images and no deploy environments still wants the
+        # tailnet preflight — k8s-cluster is helm and kubectl, and a config
+        # file there would describe nothing. Every other command needs the
+        # config to mean anything, so only this one degrades.
+        if not getattr(args, "config_optional", False):
+            print(f"config error: {e}", file=sys.stderr)
+            return 2
+        config = Config()
+    try:
+        return args.func(config, args) or 0
+    except ConfigError as e:
+        print(f"config error: {e}", file=sys.stderr)
+        return 2
+    except (ImageError, DeployError, TailnetError, GateError, GitError) as e:
+        print(f"error: {e}", file=sys.stderr)
+        return 2
+
+
+def _add_no_expand(p: argparse.ArgumentParser) -> None:
+    _ = p.add_argument(
+        "--no-expand",
+        action="store_true",
+        help="don't expand matched tags via depends (debug)",
+    )
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="tangier",
+        description="smart CI for monorepos: change-scoped tests, local gate reuse, content-addressed builds",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    _ = p.add_argument(
+        "--config",
+        default=DEFAULT_CONFIG,
+        help=f"path to TOML config (default: {DEFAULT_CONFIG})",
+    )
+    sub = p.add_subparsers(dest="group")
+    _add_changemap_parser(sub)
+    image_cmds.add_parsers(sub)
+    deploy_cmds.add_parsers(sub)
+    tailnet_cmds.add_parsers(sub)
+    gate_cmds.add_parsers(sub)
+    return p
+
+
+def _add_changemap_parser(sub: argparse._SubParsersAction) -> None:
+    cm = sub.add_parser("changemap", help="which parts of the repo does this diff touch?")
+    cmsub = cm.add_subparsers(dest="cmd", required=True)
+
+    lp = cmsub.add_parser("list", help="print all tags and their globs")
+    _ = lp.add_argument(
+        "--graph",
+        action="store_true",
+        help="print each tag with its depends entries indented underneath",
+    )
+    lp.set_defaults(func=changemap_cmds.cmd_list)
+
+    sp = cmsub.add_parser("sha", help="git-tree content hash for a SHA bucket")
+    _ = sp.add_argument("bucket", nargs="?")
+    _ = sp.add_argument("--all", action="store_true")
+    _ = sp.add_argument("--github-notice", action="store_true")
+    _ = sp.add_argument("--head", default="HEAD", help="git ref to hash at (default: HEAD)")
+    sp.set_defaults(func=changemap_cmds.cmd_sha)
+
+    ip = cmsub.add_parser("items", help="project the expanded changed set into a named item list")
+    _ = ip.add_argument("name", help="items name (e.g. `unittest`, `e2e`)")
+    add_diff_args(ip)
+    _add_no_expand(ip)
+    add_full(ip)
+    ip.set_defaults(func=changemap_cmds.cmd_items)
+
+    li = cmsub.add_parser(
+        "list-ignored",
+        help="list changed files in the diff that matched no tag — sanity-check helper",
+    )
+    add_diff_args(li)
+    li.set_defaults(func=changemap_cmds.cmd_list_ignored)
+
+    ep = cmsub.add_parser("explain", help="show modified tags, dependents, and resulting CI invocations")
+    add_diff_args(ep)
+    _add_no_expand(ep)
+    add_full(ep)
+    _ = ep.add_argument(
+        "--files",
+        action="append",
+        metavar="GROUP=CSV",
+        help="csv of changed files for a file-set group, forwarded into that runner's --files arg (repeatable)",
+    )
+    ep.set_defaults(func=_explain_with_files)
+
+    bm = cmsub.add_parser(
+        "build-matrix",
+        help="emit the buildable buckets this diff touches, as a JSON array for a build matrix",
+    )
+    add_diff_args(bm)
+    _add_no_expand(bm)
+    add_full(bm)
+    bm.set_defaults(func=changemap_cmds.cmd_build_matrix)
+
+    gp = cmsub.add_parser(
+        "github-outputs",
+        help="emit the answer set (SHAs, items, touched, file-sets) as $GITHUB_OUTPUT lines",
+    )
+    add_diff_args(gp)
+    _add_no_expand(gp)
+    add_full(gp)
+    gp.set_defaults(func=changemap_cmds.cmd_github_outputs)
+
+
+def _explain_with_files(config, args: argparse.Namespace) -> int:
+    """Validate `--files GROUP=CSV` before running explain, so a malformed pair exits 2."""
+    try:
+        args.files_map = changemap_cmds.parse_files_args(args.files)
+    except ValueError as e:
+        print(f"error: {e}", file=sys.stderr)
+        return 2
+    return changemap_cmds.cmd_explain(config, args)

tangier/commands/__init__.py

@@ -0,0 +1 @@
+"""CLI subcommand implementations — formatters over the computed answer set."""

tangier/commands/args.py

@@ -0,0 +1,18 @@
+"""Arguments that the `changemap` and `gate` parsers share, so each flag is defined once."""
+
+from __future__ import annotations
+
+import argparse
+
+
+def add_diff_args(p: argparse.ArgumentParser, *, head: bool = True) -> None:
+    """`--base`, and `--head` unless the command keys the working tree."""
+    _ = p.add_argument("--base", default="origin/main")
+    if head:
+        _ = p.add_argument("--head", default="HEAD")
+
+
+def add_full(
+    p: argparse.ArgumentParser, help: str = "answer as if every tag changed; reads no diff, so --base is ignored"
+) -> None:
+    _ = p.add_argument("--full", action="store_true", help=help)