labtrail 0.1.0__py3-none-any.whl

labtrail/__init__.py

@@ -0,0 +1,22 @@
+"""labtrail -- git-centric lab trails.
+
+This module deliberately re-exports nothing (D-6). The public surface is
+exactly ``labtrail.check``, the ``labtrail`` command line, and the collected
+document schema. Everything else is private and may change in a minor
+release (R5.7).
+"""
+
+try:
+    from ._version import version as __version__
+except ImportError:  # pragma: no cover - only when not installed
+    try:
+        from importlib.metadata import PackageNotFoundError, version as _pkg_version
+
+        try:
+            __version__ = _pkg_version("labtrail")
+        except PackageNotFoundError:
+            __version__ = "0.0.0"
+    except ImportError:
+        __version__ = "0.0.0"
+
+__all__ = ["__version__"]

labtrail/__main__.py

@@ -0,0 +1,6 @@
+import sys
+
+from .cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

labtrail/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.1.0'
+__version_tuple__ = version_tuple = (0, 1, 0)
+
+__commit_id__ = commit_id = None

labtrail/check/__init__.py

@@ -0,0 +1,73 @@
+"""The public helper API for check scripts.
+
+**This module is labtrail's only public Python surface** (R5.5). Everything
+else in the package is private and may change in a minor release. Check
+scripts are committed to a trail's history and call into this module
+forever, so a breaking change here is a major version bump (R5.7) -- which
+is why the surface is deliberately small.
+
+A check script is a plain executable (R5.4); importing this is optional and
+changes nothing about how labtrail runs it. What it buys is the exit-code
+contract of R2.16 for free, and failure messages that tell a learner what
+was expected and what happened (R5.3)::
+
+    #!/usr/bin/env python3
+    from labtrail.check import main, assert_golden
+
+    def check():
+        out = run(["./build.sh"]).stdout
+        assert_golden(out, "golden/build.txt", normalizers=[timestamps()])
+
+    main(check)
+
+``main`` is the important part: a raised :class:`CheckFailure` exits 1
+("the work is wrong") and any other exception exits 2 ("the check could not
+run"), which is the distinction the whole methodology rests on.
+"""
+
+from ._assertions import (
+    CheckFailure,
+    assert_contains,
+    assert_equal,
+    assert_fails,
+    assert_golden,
+    assert_order,
+    assert_succeeds,
+)
+from ._normalize import (
+    addresses,
+    numbers,
+    paths,
+    regex_sub,
+    seeds,
+    timestamps,
+    trailing_ws,
+)
+from ._run import entrypoint, main, run
+
+#: The frozen public surface (R5.5). A test asserts this list literally, so
+#: adding a name here is a deliberate act with a compatibility obligation
+#: attached, not a side effect of writing a new function.
+__all__ = [
+    # Entry points
+    "main",
+    "entrypoint",
+    "run",
+    # Failure
+    "CheckFailure",
+    # Assertions (R5.1)
+    "assert_golden",
+    "assert_order",
+    "assert_fails",
+    "assert_succeeds",
+    "assert_equal",
+    "assert_contains",
+    # Normalizers for volatile fields (R5.1)
+    "timestamps",
+    "paths",
+    "addresses",
+    "seeds",
+    "numbers",
+    "trailing_ws",
+    "regex_sub",
+]

labtrail/check/_assertions.py

@@ -0,0 +1,276 @@
+"""Assertions for check scripts (R5.1, R5.3).
+
+Every failure here produces the same shape of message: what was expected,
+what happened, and -- where it helps -- a diff. That consistency is most of
+the value. A learner reads these messages more often than they read the lab
+text, and the difference between "assertion failed" and a labelled diff is
+the difference between a check that teaches and a check that stonewalls.
+"""
+
+from __future__ import annotations
+
+import difflib
+import os
+from pathlib import Path
+from typing import Callable, Iterable, Sequence
+
+#: A normalizer is just str -> str. Deliberately not a class: a course
+#: author writing an ad-hoc one should not have to learn an interface.
+Normalizer = Callable[[str], str]
+
+MAX_DIFF_LINES = 60
+
+
+class CheckFailure(AssertionError):
+    """The work is wrong.
+
+    Raised by every assertion in this module, and mapped to exit 1 by
+    :func:`labtrail.check.main` (R2.16). Subclasses ``AssertionError`` so a
+    check written with bare ``assert`` and a check written with these
+    helpers behave the same way under ``pytest``.
+    """
+
+    def __init__(self, summary: str, *, expected: object = None,
+                 actual: object = None, detail: str | None = None,
+                 hint: str | None = None):
+        self.summary = summary
+        self.expected = expected
+        self.actual = actual
+        self.detail = detail
+        self.hint = hint
+        super().__init__(self._format())
+
+    def _format(self) -> str:
+        parts = [self.summary]
+        if self.expected is not None or self.actual is not None:
+            parts.append(f"  expected: {_brief(self.expected)}")
+            parts.append(f"  actual:   {_brief(self.actual)}")
+        if self.detail:
+            parts.append(self.detail)
+        if self.hint:
+            parts.append(f"  hint: {self.hint}")
+        return "\n".join(parts)
+
+
+def _brief(value: object, limit: int = 200) -> str:
+    text = value if isinstance(value, str) else repr(value)
+    text = text.replace("\n", "\\n")
+    return text if len(text) <= limit else text[:limit] + " ..."
+
+
+def _apply(text: str, normalizers: Iterable[Normalizer] | None) -> str:
+    for norm in normalizers or ():
+        text = norm(text)
+    return text
+
+
+def _diff(expected: str, actual: str, expected_label: str,
+          actual_label: str) -> str:
+    lines = list(difflib.unified_diff(
+        expected.splitlines(keepends=True), actual.splitlines(keepends=True),
+        fromfile=expected_label, tofile=actual_label, n=3,
+    ))
+    if len(lines) > MAX_DIFF_LINES:
+        dropped = len(lines) - MAX_DIFF_LINES
+        lines = lines[:MAX_DIFF_LINES] + [f"... {dropped} more diff lines ...\n"]
+    return "".join(lines).rstrip()
+
+
+# -- golden comparison -------------------------------------------------
+
+
+def assert_golden(actual: str, golden: str | os.PathLike[str], *,
+                  normalizers: Sequence[Normalizer] = (),
+                  message: str | None = None) -> None:
+    """Compare output against a golden file (R5.1).
+
+    ``normalizers`` are applied to *both* sides, so a golden file may be
+    recorded from a real run without hand-editing the timestamps out of it.
+
+    Set ``LABTRAIL_UPDATE_GOLDEN=1`` to rewrite the golden from the actual
+    output. That is an author's tool: it is off by default and it says what
+    it did, because a golden that silently updates proves nothing.
+    """
+    path = Path(golden)
+    if os.environ.get("LABTRAIL_UPDATE_GOLDEN"):
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(actual)
+        print(f"labtrail: updated golden {path}")
+        return
+    if not path.exists():
+        raise CheckFailure(
+            f"the golden file {path} does not exist",
+            hint="this is a defect in the lab; the check cannot compare "
+                 "against a file that is not there",
+        )
+    expected = path.read_text()
+    if _apply(expected, normalizers) == _apply(actual, normalizers):
+        return
+    raise CheckFailure(
+        message or f"output does not match {path}",
+        detail=_diff(_apply(expected, normalizers), _apply(actual, normalizers),
+                     f"expected ({path})", "actual"),
+        hint="the '-' lines are what was expected; the '+' lines are what "
+             "your program produced",
+    )
+
+
+# -- ordering ----------------------------------------------------------
+
+
+def assert_order(sequence: Sequence[str] | str, *items: str,
+                 message: str | None = None) -> None:
+    """Assert that items appear in this order within a sequence (R5.1).
+
+    ``sequence`` may be a list of lines or a block of text. The items need
+    not be adjacent and need not be the only things present -- this checks
+    order, not equality, because "the reset came before the first write" is
+    the shape of the question.
+    """
+    lines = sequence.splitlines() if isinstance(sequence, str) else list(sequence)
+    positions: list[int] = []
+    for item in items:
+        start = positions[-1] + 1 if positions else 0
+        for i in range(start, len(lines)):
+            if item in lines[i]:
+                positions.append(i)
+                break
+        else:
+            if any(item in line for line in lines):
+                previous = items[len(positions) - 1]
+                raise CheckFailure(
+                    message or f"{item!r} appears, but not after {previous!r}",
+                    detail=_numbered(lines, items),
+                    hint="the expected order is: " + " -> ".join(map(repr, items)),
+                )
+            raise CheckFailure(
+                message or f"{item!r} never appears",
+                detail=_numbered(lines, items),
+                hint="the expected order is: " + " -> ".join(map(repr, items)),
+            )
+
+
+def _numbered(lines: Sequence[str], items: Sequence[str]) -> str:
+    """Show the input with the items of interest marked."""
+    out = []
+    for i, line in enumerate(lines[:40]):
+        mark = ">" if any(item in line for item in items) else " "
+        out.append(f"  {mark} {i + 1:4d}| {line.rstrip()}")
+    if len(lines) > 40:
+        out.append(f"    ... {len(lines) - 40} more lines ...")
+    return "\n".join(out)
+
+
+# -- process outcome ---------------------------------------------------
+
+
+def assert_fails(result_or_argv, *, exit_code: int | None = None,
+                 stderr_contains: str | None = None,
+                 message: str | None = None):
+    """Assert that a command fails as expected (R5.1).
+
+    Accepts either a :class:`labtrail.check.run` result or an argv to run.
+    "Does the tool reject bad input" is a common lab, and writing it by hand
+    invites the bug where the assertion passes because the command was never
+    found.
+    """
+    result = _resolve(result_or_argv)
+    if result.returncode == 0:
+        raise CheckFailure(
+            message or f"expected {_cmd(result)} to fail, but it succeeded",
+            expected="a non-zero exit", actual="exit 0",
+            detail=_streams(result),
+        )
+    if exit_code is not None and result.returncode != exit_code:
+        raise CheckFailure(
+            message or f"{_cmd(result)} failed with the wrong exit code",
+            expected=f"exit {exit_code}", actual=f"exit {result.returncode}",
+            detail=_streams(result),
+        )
+    if stderr_contains is not None and stderr_contains not in result.stderr:
+        raise CheckFailure(
+            message or f"{_cmd(result)} did not report the expected error",
+            expected=f"stderr containing {stderr_contains!r}",
+            actual=_brief(result.stderr),
+            detail=_streams(result),
+        )
+    return result
+
+
+def assert_succeeds(result_or_argv, *, message: str | None = None):
+    """Assert a command exits 0, showing its output when it does not."""
+    result = _resolve(result_or_argv)
+    if result.returncode != 0:
+        raise CheckFailure(
+            message or f"{_cmd(result)} failed",
+            expected="exit 0", actual=f"exit {result.returncode}",
+            detail=_streams(result),
+        )
+    return result
+
+
+def _resolve(result_or_argv):
+    from ._run import Completed, run
+
+    if isinstance(result_or_argv, Completed):
+        return result_or_argv
+    return run(result_or_argv)
+
+
+def _cmd(result) -> str:
+    return " ".join(result.argv)
+
+
+def _streams(result) -> str:
+    out = []
+    if result.stdout.strip():
+        out.append("  stdout:\n" + _indent(result.stdout))
+    if result.stderr.strip():
+        out.append("  stderr:\n" + _indent(result.stderr))
+    return "\n".join(out)
+
+
+def _indent(text: str, prefix: str = "    ") -> str:
+    return "\n".join(prefix + line for line in text.rstrip().splitlines())
+
+
+# -- the two everyone writes by hand anyway ----------------------------
+
+
+def assert_equal(actual, expected, *, message: str | None = None,
+                 normalizers: Sequence[Normalizer] = ()) -> None:
+    """Equality, with a diff for multi-line strings."""
+    a, e = actual, expected
+    if isinstance(a, str) and isinstance(e, str):
+        a, e = _apply(a, normalizers), _apply(e, normalizers)
+    if a == e:
+        return
+    detail = None
+    if isinstance(a, str) and isinstance(e, str) and ("\n" in a or "\n" in e):
+        detail = _diff(e, a, "expected", "actual")
+    raise CheckFailure(message or "values differ", expected=e, actual=a,
+                       detail=detail)
+
+
+def assert_contains(haystack: str, needle: str, *,
+                    message: str | None = None) -> None:
+    """Substring presence, showing the haystack when it is missing."""
+    if needle in haystack:
+        return
+    raise CheckFailure(
+        message or f"expected to find {needle!r}",
+        expected=needle, actual=_brief(haystack),
+        detail="  in:\n" + _indent(haystack[:2000]),
+    )
+
+
+#: Exported for the ``labtrail assert`` shell entry point (R5.2). Keyed by
+#: the name a shell script spells.
+SHELL_ASSERTIONS = {
+    "golden": assert_golden,
+    "order": assert_order,
+    "fails": assert_fails,
+    "succeeds": assert_succeeds,
+    "equal": assert_equal,
+    "contains": assert_contains,
+}

labtrail/check/_normalize.py

@@ -0,0 +1,121 @@
+"""Normalizers for volatile fields (R5.1).
+
+A golden comparison is only useful if it fails when the work is wrong and
+not when the clock moved. Each function here returns a ``str -> str``
+callable, so they compose in the order given::
+
+    assert_golden(out, "golden.txt",
+                  normalizers=[timestamps(), paths(repo_root), trailing_ws()])
+
+Every one of them replaces the volatile text with a stable *label* rather
+than deleting it. A golden that reads ``simulation finished at <TIME>``
+still shows a reader that a time was printed; one with the time silently
+removed does not, and the next author to edit the golden will not know to
+put it back.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from typing import Callable
+
+Normalizer = Callable[[str], str]
+
+# ISO-8601-ish, "Mon Jan  2 15:04:05 2006", and bare clock times.
+_TIMESTAMP_PATTERNS = [
+    r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?",
+    r"\b(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+"
+    r"(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+"
+    r"\d{1,2}\s+\d{2}:\d{2}:\d{2}(?:\s+\d{4})?",
+    r"\b\d{2}:\d{2}:\d{2}(?:\.\d+)?\b",
+]
+
+_DURATION = re.compile(
+    r"\b\d+(?:\.\d+)?\s?(?:ns|us|ms|s|sec|secs|seconds|min|minutes|h|hours)\b",
+    re.IGNORECASE)
+_ADDRESS = re.compile(r"\b0x[0-9a-fA-F]{4,}\b")
+_SEED = re.compile(r"(?i)\b(seed|random_seed|rng[_ ]?seed)\b(\s*[:=]\s*)(0x[0-9a-fA-F]+|\d+)")
+_NUMBER = re.compile(r"-?\b\d+(?:\.\d+)?\b")
+
+
+def timestamps(label: str = "<TIME>", *, durations: bool = True) -> Normalizer:
+    """Replace dates, clock times and (by default) durations.
+
+    Durations are included because "elapsed: 1.34s" is the single most
+    common reason a golden that should be stable is not.
+    """
+    compiled = [re.compile(p) for p in _TIMESTAMP_PATTERNS]
+
+    def normalize(text: str) -> str:
+        for pattern in compiled:
+            text = pattern.sub(label, text)
+        if durations:
+            text = _DURATION.sub("<DURATION>", text)
+        return text
+
+    return normalize
+
+
+def paths(root: str | os.PathLike[str] | None = None,
+          label: str = "<ROOT>") -> Normalizer:
+    """Replace absolute paths under ``root`` with a label.
+
+    Defaults to the current working directory, which is the lab's worktree
+    when labtrail runs the check -- and that path is a temporary directory,
+    so a golden recorded with it in is worthless.
+    """
+    base = str(os.path.abspath(root if root is not None else os.getcwd()))
+
+    def normalize(text: str) -> str:
+        text = text.replace(base, label)
+        # Also catch the resolved form, which differs on macOS (/private/...)
+        real = os.path.realpath(base)
+        if real != base:
+            text = text.replace(real, label)
+        return text
+
+    return normalize
+
+
+def addresses(label: str = "<ADDR>") -> Normalizer:
+    """Replace hexadecimal addresses, e.g. in a backtrace or a dump."""
+    return lambda text: _ADDRESS.sub(label, text)
+
+
+def seeds(label: str = "<SEED>") -> Normalizer:
+    """Replace ``seed = 12345`` and its spellings, keeping the label."""
+    return lambda text: _SEED.sub(lambda m: f"{m.group(1)}{m.group(2)}{label}", text)
+
+
+def numbers(label: str = "<N>") -> Normalizer:
+    """Replace every number.
+
+    A blunt instrument, and usually the wrong one -- a golden with no
+    numbers in it often no longer tests anything. Provided because there
+    are cases (a coverage percentage, a cycle count) where the number is
+    genuinely not the subject, and an author who needs it should not have
+    to write the regex.
+    """
+    return lambda text: _NUMBER.sub(label, text)
+
+
+def trailing_ws() -> Normalizer:
+    """Strip trailing whitespace per line and normalise the final newline."""
+
+    def normalize(text: str) -> str:
+        lines = [line.rstrip() for line in text.splitlines()]
+        return "\n".join(lines) + ("\n" if text else "")
+
+    return normalize
+
+
+def regex_sub(pattern: str, replacement: str = "<X>", *, flags: int = 0) -> Normalizer:
+    """The escape hatch: an arbitrary substitution.
+
+    Every normalizer above is a special case of this one. It exists so that
+    a course with a volatile field labtrail has never heard of does not have
+    to wait for a labtrail release.
+    """
+    compiled = re.compile(pattern, flags)
+    return lambda text: compiled.sub(replacement, text)

labtrail/check/_run.py

@@ -0,0 +1,136 @@
+"""Entry point and process helpers for check scripts."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import traceback
+from dataclasses import dataclass
+from typing import Callable
+
+from ._assertions import CheckFailure
+
+EXIT_CORRECT = 0
+EXIT_WRONG = 1
+EXIT_COULD_NOT_RUN = 2
+
+
+def main(fn: Callable[[], object] | None = None, *, argv: list[str] | None = None) -> int:
+    """Run a check function and exit with the code R2.16 requires.
+
+    =========================  ====  ===================================
+    what happened              exit  what labtrail concludes
+    =========================  ====  ===================================
+    returned normally          0     the work is correct
+    raised CheckFailure        1     the check ran; the work is wrong
+    raised a bare Assertion-   1     likewise
+    Error
+    raised anything else       2     the check could not run
+    =========================  ====  ===================================
+
+    That last row is the one worth being careful about, and it is why this
+    function exists rather than a bare ``assert``. A ``NameError`` in a
+    check script is *not* a learner's mistake; reporting it as one sends
+    them to debug their own correct work. So an unexpected exception exits
+    2 and prints a traceback, which labtrail reports as a broken lab (R4.3)
+    and an author can act on.
+
+    ``AssertionError`` is treated as a check failure so that ``assert x ==
+    y`` and ``assert_equal(x, y)`` behave the same way -- R2.16 names exit 1
+    "assertion failed", and an author who writes the plain spelling should
+    get the plain meaning. The cost is that an ``assert`` inside a library
+    the check imports is misreported as the learner being wrong. That is
+    the rarer case, and the fix (use ``CheckFailure``, or catch it) is
+    available to an author who hits it.
+
+    Called with no argument, returns a decorator, so a script may write
+    either ``main(check)`` or ``@main`` above the function.
+    """
+    if fn is None:
+        return entrypoint  # type: ignore[return-value]
+
+    try:
+        fn()
+    except CheckFailure as e:
+        print(str(e), file=sys.stderr)
+        sys.exit(EXIT_WRONG)
+    except AssertionError as e:
+        print(f"check failed: {e}" if str(e) else "check failed", file=sys.stderr)
+        traceback.print_exc()
+        sys.exit(EXIT_WRONG)
+    except KeyboardInterrupt:
+        print("check: interrupted", file=sys.stderr)
+        sys.exit(EXIT_COULD_NOT_RUN)
+    except SystemExit:
+        raise
+    except BaseException:  # noqa: BLE001 - deliberately broad; see the docstring
+        print("check: the check itself failed to run. This is a defect in the "
+              "lab, not in your work.", file=sys.stderr)
+        traceback.print_exc()
+        sys.exit(EXIT_COULD_NOT_RUN)
+    sys.exit(EXIT_CORRECT)
+
+
+def entrypoint(fn: Callable[[], object]) -> Callable[[], object]:
+    """Decorator form of :func:`main`.
+
+    ::
+
+        @entrypoint
+        def check():
+            ...
+    """
+    if os.environ.get("LABTRAIL_NO_AUTORUN"):
+        return fn
+    main(fn)
+    return fn
+
+
+@dataclass(frozen=True)
+class Completed:
+    """The result of :func:`run`."""
+
+    argv: list[str]
+    returncode: int
+    stdout: str
+    stderr: str
+
+    @property
+    def ok(self) -> bool:
+        return self.returncode == 0
+
+    @property
+    def output(self) -> str:
+        """Both streams, in the order a terminal would have shown them."""
+        return self.stdout + self.stderr
+
+
+def run(argv: list[str] | str, *, cwd: str | os.PathLike[str] | None = None,
+        env: dict[str, str] | None = None, timeout: float | None = None,
+        input: str | None = None) -> Completed:
+    """Run a command and capture it. No shell.
+
+    A string is split with ``shlex``; there is still no shell, so a pipeline
+    belongs in a script. That is the same rule the lab metadata follows, for
+    the same reason (R2.16).
+    """
+    import shlex
+
+    if isinstance(argv, str):
+        argv = shlex.split(argv)
+    full_env = {**os.environ, **(env or {})}
+    try:
+        proc = subprocess.run(
+            list(argv), cwd=cwd, env=full_env, capture_output=True, text=True,
+            timeout=timeout, input=input, errors="replace",
+        )
+    except FileNotFoundError as e:
+        # Missing tool: the check could not run. Raising (rather than
+        # returning 127) sends this to main()'s exit-2 path, which is
+        # where a missing tool belongs.
+        raise RuntimeError(f"{e.strerror}: {argv[0]}") from e
+    except subprocess.TimeoutExpired as e:
+        raise RuntimeError(f"timed out after {timeout}s: {' '.join(argv)}") from e
+    return Completed(argv=list(argv), returncode=proc.returncode,
+                     stdout=proc.stdout, stderr=proc.stderr)