formwork-kit 0.1.0__py3-none-any.whl

formwork_cli/kit/check/run

@@ -0,0 +1,340 @@
+#!/usr/bin/env python3
+"""The gate. One command, and the only one a brief may name.
+
+    formwork/check/run                run every check over the project
+    formwork/check/run --demo-fail    watch each check refuse a broken input
+    formwork/check/run --list         what exists
+
+Exit status is the contract. Every adapter and every brief depends on it:
+
+    0   clean
+    1   something is wrong, and it is named
+    2   the gate could not run, so this is NOT a pass
+
+Status 2 exists because the most dangerous outcome is a gate reporting success
+when it never ran.
+
+WHAT A CHECK MUST SHIP, AND WHY IT IS TWO THINGS
+------------------------------------------------
+Every check ships inputs of both kinds:
+
+    fixtures/<check>/must-fail/<case>/   the check must reject these
+    fixtures/<check>/must-pass/<case>/   the check must accept these
+
+A case may also carry a `state/` directory. If it does, the runner points
+FORMWORK_STATE_DIR at it, so a check that reads something kept outside the
+repository can still be exercised.
+
+An earlier version demanded only the first, and an audit broke it in one
+minute: a check that ignores its input and fails on any directory named
+"broken" passed the gate while examining nothing at all. Rejecting something
+proves a check can say no. It does not prove the check looked.
+
+**Requiring both means a check has to discriminate.** A check missing either
+kind does not run — status 2, not a pass.
+
+And every fixture is copied to a directory with a random name before the check
+sees it, so a check cannot cheat by recognising "must-fail" or the case name.
+It has to look at the contents.
+
+TIMEOUTS
+--------
+A check that never returns used to hang the gate forever, which in a hook
+freezes the agent until somebody kills it. Every check now has a deadline, and
+exceeding it is "could not run", never "fine".
+
+Python 3, standard library only, no dependencies.
+"""
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+CHECKS_DIR = os.path.join(HERE, "checks")
+FIXTURES_DIR = os.path.join(HERE, "fixtures")
+PROJECT_ROOT = os.path.dirname(os.path.dirname(HERE))
+
+CLEAN, FINDINGS, CANNOT_RUN = 0, 1, 2
+
+# Seconds a single check may take. Generous for a repository check, and far
+# short of a hook timeout.
+def _seconds(name, default):
+    """A malformed timeout must not become a traceback on the blocking path."""
+    raw = os.environ.get(name, str(default))
+    try:
+        v = int(raw)
+    except (TypeError, ValueError):
+        return None
+    if v <= 0:
+        return None
+    # A number too large to be a timeout raised OverflowError deep inside
+    # subprocess, which surfaced as exit 1. A day is already absurd.
+    return min(v, 86400)
+
+
+DEADLINE = _seconds("FORMWORK_CHECK_TIMEOUT", 60)
+
+MUST_FAIL, MUST_PASS = "must-fail", "must-pass"
+
+
+def cases(check_name, kind):
+    d = os.path.join(FIXTURES_DIR, check_name, kind)
+    if not os.path.isdir(d):
+        return []
+    return [os.path.join(d, c) for c in sorted(os.listdir(d))
+            if os.path.isdir(os.path.join(d, c))]
+
+
+def discover():
+    """Every file in checks/, with the inputs it ships and whether it can run.
+
+    Non-executable files are listed rather than skipped. A check silently
+    absent is the one failure a gate must never report as success, and an
+    audit removed one with a single allowed `chmod` and watched the gate
+    report green over the remaining eight.
+    """
+    if not os.path.isdir(CHECKS_DIR):
+        return None
+    found = []
+    for name in sorted(os.listdir(CHECKS_DIR)):
+        path = os.path.join(CHECKS_DIR, name)
+        if name.startswith(".") or not os.path.isfile(path):
+            continue
+        found.append((name, path, cases(name, MUST_FAIL), cases(name, MUST_PASS),
+                      os.access(path, os.X_OK)))
+    return found
+
+
+def anonymised(fixture):
+    """A copy of a fixture under a name that carries no information.
+
+    Without this, a check can pass the gate by recognising the word
+    "must-fail" or the case name, while examining nothing. An audit did
+    exactly that in about a minute.
+
+    **It hides the case name and nothing else.** Names inside the fixture are
+    left alone, because several checks legitimately read them: a decision
+    record is identified by its numbered filename, a role by its `.md`
+    ending. So a check that looked for a particular file INSIDE a fixture
+    could still pass without examining anything. A later audit wrote one and
+    it worked.
+
+    Scrambling the contents was tried and broke the checks that read names
+    for real reasons. The limit is recorded in formwork/limits.md instead of
+    being papered over.
+    """
+    tmp = tempfile.mkdtemp(prefix="fw-")
+    dest = os.path.join(tmp, "subject")
+    shutil.copytree(fixture, dest)
+    return tmp, dest
+
+
+def invoke(path, target, exclude=()):
+    """Run a check over a directory, with a deadline.
+
+    FORMWORK_EXCLUDE carries paths the check must not descend into. The
+    fixtures are wrong on purpose, so a check run over the project would find
+    them and be right to complain. The runner owns that knowledge; a check
+    must not have to know where the kit stores things.
+
+    Returns (exit_code, output). A check that overruns returns CANNOT_RUN,
+    because a gate that hangs is worse than one that fails.
+    """
+    env = dict(os.environ)
+    env["FORMWORK_EXCLUDE"] = os.pathsep.join(exclude)
+    # A check may depend on state kept outside the repository — a word list, a
+    # set of fingerprints. A fixture supplies its own by carrying a `state/`
+    # directory, and without this such a check could never ship a fixture,
+    # which the gate requires. So it could never ship at all.
+    state = os.path.join(target, "state")
+    if os.path.isdir(state):
+        env["FORMWORK_STATE_DIR"] = state
+    try:
+        p = subprocess.run([path, target], capture_output=True, text=True,
+                           env=env, timeout=DEADLINE)
+    except subprocess.TimeoutExpired:
+        return CANNOT_RUN, ("took longer than %ds and was stopped. A check that "
+                            "does not return is not a pass." % DEADLINE)
+    except OSError as e:
+        return CANNOT_RUN, "could not be started: %s" % e
+    return p.returncode, (p.stdout + p.stderr).strip()
+
+
+KNOWN_FLAGS = {"--list", "--demo-fail"}
+
+
+def main(argv):
+    args = set(argv[1:])
+    unknown = sorted(a for a in args if a.startswith("-")
+                     and a not in KNOWN_FLAGS)
+    if unknown:
+        print("ERROR: unknown option(s): %s" % ", ".join(unknown),
+              file=sys.stderr)
+        print("       Known: %s. A mistyped --demo-fail used to run the whole "
+              "gate for real." % ", ".join(sorted(KNOWN_FLAGS)), file=sys.stderr)
+        return CANNOT_RUN
+    checks = discover()
+
+    if checks is None:
+        print("ERROR: no checks directory at %s" % CHECKS_DIR, file=sys.stderr)
+        print("       The gate did not run. This is not a pass.", file=sys.stderr)
+        return CANNOT_RUN
+    if DEADLINE is None:
+        print("ERROR: FORMWORK_CHECK_TIMEOUT is not a positive whole number of "
+              "seconds: %r" % os.environ.get("FORMWORK_CHECK_TIMEOUT"),
+              file=sys.stderr)
+        print("       The gate did not run. This is not a pass.", file=sys.stderr)
+        return CANNOT_RUN
+
+    if not checks:
+        print("ERROR: no checks found in %s" % CHECKS_DIR, file=sys.stderr)
+        print("       A gate with nothing in it reports success about nothing.",
+              file=sys.stderr)
+        return CANNOT_RUN
+
+    # A check that cannot be executed is not a check that passed. The gate
+    # refuses rather than quietly running the remainder.
+    unrunnable = [n for n, _p, _f, _q, ok in checks if not ok]
+    if unrunnable:
+        print("ERROR: %d check(s) present but not executable:" % len(unrunnable),
+              file=sys.stderr)
+        for n in unrunnable:
+            print("       %s" % n, file=sys.stderr)
+        print("       The gate did not run. A check that cannot run has not "
+              "passed. Restore the execute bit: chmod +x formwork/check/checks/*",
+              file=sys.stderr)
+        return CANNOT_RUN
+
+    # A check that ships only one kind of input cannot be trusted. One kind
+    # proves it can say no; the other proves it was listening.
+    incomplete = []
+    for name, _, fails, passes, _x in checks:
+        if not fails:
+            incomplete.append("%s has no must-fail input" % name)
+        if not passes:
+            incomplete.append("%s has no must-pass input" % name)
+    if incomplete:
+        print("ERROR: %d check(s) are not properly equipped:" % len(incomplete),
+              file=sys.stderr)
+        for i in incomplete:
+            print("       %s" % i, file=sys.stderr)
+        print("       A check needs an input it rejects AND one it accepts, or "
+              "it has not been shown to discriminate.", file=sys.stderr)
+        return CANNOT_RUN
+
+    if "--list" in args:
+        for name, _, fails, passes, _x in checks:
+            print("%-20s %d must-fail, %d must-pass"
+                  % (name, len(fails), len(passes)))
+            for f in fails:
+                print("%-20s   reject  %s" % ("", os.path.basename(f)))
+            for p in passes:
+                print("%-20s   accept  %s" % ("", os.path.basename(p)))
+        return CLEAN
+
+    problems = []
+
+    if "--demo-fail" in args:
+        print("Each check, against the input it is meant to reject.")
+        print()
+        for name, path, fails, _p, _x in checks:
+            for fixture in fails:
+                tmp, subject = anonymised(fixture)
+                code, out = invoke(path, subject)
+                shutil.rmtree(tmp, ignore_errors=True)
+                case = os.path.basename(fixture)
+                if code == 0:
+                    print("  %-14s %-24s ACCEPTED — and it must not"
+                          % (name, case))
+                    problems.append("%s/%s was accepted" % (name, case))
+                elif code == CANNOT_RUN:
+                    print("  %-14s %-24s could not run: %s"
+                          % (name, case, out.split("\n")[0]))
+                    problems.append("%s/%s could not run" % (name, case))
+                else:
+                    first = out.split("\n")[0] if out else "(no output)"
+                    print("  %-14s %-24s rejected, as it should: %s"
+                          % (name, case, first))
+        print()
+        if problems:
+            print("GATE: %d check(s) did not behave." % len(problems))
+            return FINDINGS
+        print("GATE: every check rejected its broken input. That is the point.")
+        return CLEAN
+
+    # Pass 1 — the project itself.
+    for name, path, _f, _p, _x in checks:
+        code, out = invoke(path, PROJECT_ROOT, exclude=[FIXTURES_DIR])
+        if code == CANNOT_RUN:
+            print("ERROR: check %s could not run:\n%s" % (name, out),
+                  file=sys.stderr)
+            return CANNOT_RUN
+        if code != 0:
+            problems.append(name)
+            print("FAIL  %s" % name)
+            for line in out.split("\n"):
+                if line.strip():
+                    print("      %s" % line)
+        else:
+            print("ok    %s" % name)
+
+    # Pass 2 — prove each check rejects what it should.
+    accepted = []
+    for name, path, fails, _p, _x in checks:
+        for fixture in fails:
+            tmp, subject = anonymised(fixture)
+            code, out = invoke(path, subject)
+            shutil.rmtree(tmp, ignore_errors=True)
+            if code == CANNOT_RUN:
+                print("ERROR: %s could not run on %s: %s"
+                      % (name, os.path.basename(fixture), out), file=sys.stderr)
+                return CANNOT_RUN
+            if code == 0:
+                accepted.append("%s/%s" % (name, os.path.basename(fixture)))
+
+    # Pass 3 — prove each check accepts what it should. Without this a check
+    # that simply always fails would look rigorous.
+    rejected = []
+    for name, path, _f, passes, _x in checks:
+        for fixture in passes:
+            tmp, subject = anonymised(fixture)
+            code, out = invoke(path, subject)
+            shutil.rmtree(tmp, ignore_errors=True)
+            if code == CANNOT_RUN:
+                print("ERROR: %s could not run on %s: %s"
+                      % (name, os.path.basename(fixture), out), file=sys.stderr)
+                return CANNOT_RUN
+            if code != 0:
+                rejected.append("%s/%s" % (name, os.path.basename(fixture)))
+
+    if accepted:
+        print()
+        print("FAIL  these inputs are meant to be rejected, and were not:")
+        for a in accepted:
+            print("      %s" % a)
+        print("      A check that cannot fail is not evidence.")
+
+    if rejected:
+        print()
+        print("FAIL  these inputs are meant to be accepted, and were not:")
+        for r in rejected:
+            print("      %s" % r)
+        print("      A check that refuses everything has not been shown to "
+              "look at anything.")
+
+    print()
+    if problems or accepted or rejected:
+        print("GATE: red. %d check(s) failed on the project, %d input(s) "
+              "wrongly accepted, %d wrongly rejected."
+              % (len(problems), len(accepted), len(rejected)))
+        return FINDINGS
+
+    print("GATE: green. %d check(s), each shown to reject the wrong and accept "
+          "the right." % len(checks))
+    return CLEAN
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

formwork_cli/kit/check/test_gate.py

@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+"""Prove the gate can fail.
+
+The gate's whole job is refusing. A gate nobody has watched refuse is not
+evidence of anything, so this exercises every way it is supposed to say no.
+
+    green     a sound project                     -> 0
+    red       a check that finds something        -> 1
+    red       a broken input that does not break  -> 1
+    cannot    no checks at all                    -> 2
+    cannot    a check with no broken input        -> 2
+
+Python 3, standard library only, no dependencies.
+"""
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+RUN = os.path.join(HERE, "run")
+results = []
+
+
+def check(name, got, want, output=""):
+    ok = got == want
+    results.append(ok)
+    print("  [%s] %-46s expected %d, got %d"
+          % ("pass" if ok else "FAIL", name, want, got))
+    if not ok and output:
+        for line in output.strip().split("\n")[:6]:
+            print("        %s" % line)
+
+
+def run_gate(gate_dir, *args):
+    p = subprocess.run([sys.executable, os.path.join(gate_dir, "run")] + list(args),
+                       capture_output=True, text=True)
+    return p.returncode, p.stdout + p.stderr
+
+
+def build_gate(project_files, checks, fixtures, passes=None):
+    """A throwaway project with its own copy of the gate inside it.
+
+    The runner takes the project root to be two levels above itself, so the
+    layout here mirrors a real fork: <project>/formwork/check/.
+    """
+    root = tempfile.mkdtemp(prefix="formwork-gate-")
+    gate = os.path.join(root, "formwork", "check")
+    os.makedirs(os.path.join(gate, "checks"))
+    os.makedirs(os.path.join(gate, "fixtures"))
+    shutil.copy(RUN, os.path.join(gate, "run"))
+    os.chmod(os.path.join(gate, "run"), 0o755)
+
+    for rel, text in project_files.items():
+        full = os.path.join(root, rel)
+        os.makedirs(os.path.dirname(full), exist_ok=True)
+        open(full, "w", encoding="utf-8").write(text)
+
+    for name, body in checks.items():
+        p = os.path.join(gate, "checks", name)
+        open(p, "w", encoding="utf-8").write(body)
+        os.chmod(p, 0o755)
+
+    for kind, table in (("must-fail", fixtures), ("must-pass", passes or {})):
+        for cname, cases in table.items():
+            for case, files in cases.items():
+                d = os.path.join(gate, "fixtures", cname, kind, case)
+                os.makedirs(d)
+                for fn, text in files.items():
+                    open(os.path.join(d, fn), "w", encoding="utf-8").write(text)
+
+    return root, gate
+
+
+# A check that fails when it finds a file called "wrong.md".
+FINDS_WRONG = '''#!/usr/bin/env python3
+import os, sys
+root = sys.argv[1]
+ex = [os.path.abspath(p) for p in os.environ.get("FORMWORK_EXCLUDE","").split(os.pathsep) if p]
+hits = []
+for dp, dns, fns in os.walk(root):
+    here = os.path.abspath(dp)
+    if any(here == e or here.startswith(e + os.sep) for e in ex):
+        dns[:] = []
+        continue
+    hits += [os.path.join(dp, f) for f in fns if f == "wrong.md"]
+print("%d found" % len(hits))
+sys.exit(1 if hits else 0)
+'''
+
+# A check that never fails, whatever it is given. The gate must notice.
+NEVER_FAILS = '''#!/usr/bin/env python3
+import sys
+print("all fine, always")
+sys.exit(0)
+'''
+
+
+def main():
+    print("THE GATE")
+
+    # green - a sound project, one check, one input that does break it
+    root, gate = build_gate({"README.md": "fine\\n"},
+                            {"finds-wrong": FINDS_WRONG},
+                            {"finds-wrong": {"has-one": {"wrong.md": "x\\n"}}},
+                            {"finds-wrong": {"has-none": {"fine.md": "x\\n"}}})
+    code, out = run_gate(gate)
+    check("green: sound project, check proved able to fail", code, 0, out)
+    shutil.rmtree(root)
+
+    # red - the check finds something in the project
+    root, gate = build_gate({"README.md": "fine\\n", "docs/wrong.md": "x\\n"},
+                            {"finds-wrong": FINDS_WRONG},
+                            {"finds-wrong": {"has-one": {"wrong.md": "x\\n"}}},
+                            {"finds-wrong": {"has-none": {"fine.md": "x\\n"}}})
+    code, out = run_gate(gate)
+    check("red: a check finds something", code, 1, out)
+    shutil.rmtree(root)
+
+    # red - the broken input does not break the check
+    root, gate = build_gate({"README.md": "fine\\n"},
+                            {"never-fails": NEVER_FAILS},
+                            {"never-fails": {"supposedly-broken": {"a.md": "x\\n"}}},
+                            {"never-fails": {"fine": {"b.md": "x\\n"}}})
+    code, out = run_gate(gate)
+    check("red: a check that cannot fail", code, 1, out)
+    shutil.rmtree(root)
+
+    # cannot run - a check with no broken input at all
+    root, gate = build_gate({"README.md": "fine\\n"},
+                            {"finds-wrong": FINDS_WRONG},
+                            {})
+    code, out = run_gate(gate)
+    check("cannot run: a check ships no broken input", code, 2, out)
+    shutil.rmtree(root)
+
+    # cannot run - nothing to run
+    root, gate = build_gate({"README.md": "fine\\n"}, {}, {})
+    code, out = run_gate(gate)
+    check("cannot run: no checks at all", code, 2, out)
+    shutil.rmtree(root)
+
+    # the teaching mode must itself refuse when a broken input does not break
+    root, gate = build_gate({"README.md": "fine\\n"},
+                            {"never-fails": NEVER_FAILS},
+                            {"never-fails": {"supposedly-broken": {"a.md": "x\\n"}}},
+                            {"never-fails": {"fine": {"b.md": "x\\n"}}})
+    code, out = run_gate(gate, "--demo-fail")
+    check("red: --demo-fail catches a check that cannot fail", code, 1, out)
+    shutil.rmtree(root)
+
+    # A check that recognises the fixture's name instead of reading it. This
+    # passed the gate before fixtures were anonymised.
+    NAME_CHEAT = """#!/usr/bin/env python3
+import sys, os
+sys.exit(1 if "broken" in sys.argv[1] else 0)
+"""
+    root, gate = build_gate({"README.md": "fine\n"},
+                            {"cheat": NAME_CHEAT},
+                            {"cheat": {"broken": {"a.md": "x\n"}}},
+                            {"cheat": {"ok": {"a.md": "x\n"}}})
+    code, out = run_gate(gate)
+    check("red: a check that reads the name, not the input", code, 1, out)
+    shutil.rmtree(root)
+
+    # A check that never returns used to hang the gate for ever.
+    HANG = """#!/usr/bin/env python3
+import time
+time.sleep(600)
+"""
+    root, gate = build_gate({"README.md": "fine\n"},
+                            {"slow": HANG},
+                            {"slow": {"broken": {"a.md": "x\n"}}},
+                            {"slow": {"ok": {"a.md": "x\n"}}})
+    env_before = os.environ.get("FORMWORK_CHECK_TIMEOUT")
+    os.environ["FORMWORK_CHECK_TIMEOUT"] = "2"
+    code, out = run_gate(gate)
+    if env_before is None:
+        del os.environ["FORMWORK_CHECK_TIMEOUT"]
+    else:
+        os.environ["FORMWORK_CHECK_TIMEOUT"] = env_before
+    check("cannot run: a check that never returns", code, 2, out)
+    shutil.rmtree(root)
+
+    # An audit removed a check with one allowed `chmod` and watched the gate
+    # report green over the remaining ones. A missing check must never look
+    # like a passing check.
+    root, gate = build_gate({"README.md": "fine\n"},
+                            {"finds-wrong": FINDS_WRONG,
+                             "also-finds-wrong": FINDS_WRONG},
+                            {"finds-wrong": {"has-one": {"wrong.md": "x\n"}},
+                             "also-finds-wrong": {"has-one": {"wrong.md": "x\n"}}},
+                            {"finds-wrong": {"has-none": {"fine.md": "x\n"}},
+                             "also-finds-wrong": {"has-none": {"fine.md": "x\n"}}})
+    os.chmod(os.path.join(gate, "checks", "also-finds-wrong"), 0o644)
+    code, out = run_gate(gate)
+    check("cannot run: a check present but not executable", code, 2, out)
+    shutil.rmtree(root)
+
+    # A malformed timeout must be status 2, not a traceback with status 1.
+    root, gate = build_gate({"README.md": "fine\n"},
+                            {"finds-wrong": FINDS_WRONG},
+                            {"finds-wrong": {"has-one": {"wrong.md": "x\n"}}},
+                            {"finds-wrong": {"has-none": {"fine.md": "x\n"}}})
+    before = os.environ.get("FORMWORK_CHECK_TIMEOUT")
+    os.environ["FORMWORK_CHECK_TIMEOUT"] = "not-a-number"
+    code, out = run_gate(gate)
+    if before is None:
+        del os.environ["FORMWORK_CHECK_TIMEOUT"]
+    else:
+        os.environ["FORMWORK_CHECK_TIMEOUT"] = before
+    check("cannot run: a timeout that is not a number", code, 2, out)
+    shutil.rmtree(root)
+
+    print()
+    print("%d of %d behaved as specified." % (sum(results), len(results)))
+    return 0 if all(results) else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())