digline 0.1.0__py3-none-any.whl

digline/__init__.py

@@ -0,0 +1,7 @@
+"""digline — a Python-native evaluation engine for LLM output.
+
+The core (`digline.core`) is pure and callable on its own. Upper layers
+depend on it, never the other way round.
+"""
+
+__version__ = "0.1.0"

digline/cli/__init__.py

@@ -0,0 +1,26 @@
+"""The command line. The last layer, and the only one that touches the world.
+
+It reads the clock and asks git; everything below receives those as values.
+"""
+
+from digline.cli.main import (
+    EXIT_OK,
+    EXIT_UNJUDGED,
+    EXIT_USAGE,
+    EXIT_WORSE,
+    OUTPUT_VERSION,
+    build_parser,
+    exit_code,
+    main,
+)
+
+__all__ = [
+    "EXIT_OK",
+    "EXIT_UNJUDGED",
+    "EXIT_USAGE",
+    "EXIT_WORSE",
+    "OUTPUT_VERSION",
+    "build_parser",
+    "exit_code",
+    "main",
+]

digline/cli/__main__.py

@@ -0,0 +1,5 @@
+"""`python -m digline.cli`, which is also how the tests drive it."""
+
+from digline.cli.main import main
+
+raise SystemExit(main())

digline/cli/environment.py

@@ -0,0 +1,65 @@
+"""The clock and git.
+
+**This is the only layer allowed to touch either.** The core takes `created_at`
+as an argument and `git_commit` as data precisely so that a run stays
+reproducible and every layer below stays testable without freezing time or
+building a repository. Everything that reads the outside world lives here, in
+one file, where it can be seen.
+"""
+
+from __future__ import annotations
+
+import subprocess
+from datetime import UTC, datetime
+from pathlib import Path
+
+__all__ = ["DIRTY_SUFFIX", "git_commit", "utc_now_iso"]
+
+DIRTY_SUFFIX = "-dirty"
+
+
+def utc_now_iso() -> str:
+    """The clock, read once per command and passed down as a value.
+
+    Microseconds are kept. They were truncated at first, for readability, until
+    listing runs showed what that costs: a run's key is derived from its
+    timestamp and its configuration, so two runs in the same second with the
+    same suite produced the same key and the second silently replaced the first.
+    A fast suite against a stubbed target does that on an ordinary Tuesday.
+
+    A collision now needs two runs in the same microsecond, which takes a real
+    coincidence rather than a normal one.
+    """
+    return datetime.now(UTC).isoformat()
+
+
+def _git(root: Path, *args: str) -> str | None:
+    try:
+        done = subprocess.run(
+            ["git", "-C", str(root), *args],
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+    except OSError:
+        return None  # git is not installed; not an error, just no commit
+    return done.stdout.strip() if done.returncode == 0 else None
+
+
+def git_commit(root: Path) -> str | None:
+    """The commit this run was produced from, or `None` outside a repository.
+
+    Being outside a repository is not an error: a run produced from a notebook
+    or a container legitimately has no commit, and refusing to work there would
+    make the tool unusable exactly where people try things first.
+
+    A dirty tree yields `"<sha>-dirty"`. The run is recorded, but the marker
+    travels with it, because such a run **cannot be reproduced from the
+    repository** — and a reader deciding whether to act on the numbers needs to
+    know that as a fact rather than as an assumption.
+    """
+    sha = _git(root, "rev-parse", "HEAD")
+    if sha is None:
+        return None
+    status = _git(root, "status", "--porcelain")
+    return f"{sha}{DIRTY_SUFFIX}" if status else sha

digline/cli/loader.py

@@ -0,0 +1,230 @@
+"""Loading a suite and a target, which are Python objects rather than data.
+
+A `Judge` is an object, a `Target` is a function, a `Disclosure` is declared in
+code by construction (ADR 0002). None of that is expressible in YAML without
+reinventing a language, and a configuration language grown one escape at a time
+ends up unable to express its own delimiters.
+
+So the CLI **imports** the suite; it does not interpret it. And it never goes
+looking: a file found by convention is a file that runs by accident.
+
+Two constraints on *how* it loads a suite given as a file path, both learned the
+hard way and both non-negotiable:
+
+- **The suite's directory goes on `sys.path`.** A suite imports the application
+  under test; that is the normal case. Only for a file path — a
+  `package.module:attr` spec is already importable and the path is left alone.
+- **It is compiled from source, never through the bytecode cache.** Writing
+  `__pycache__` dirties the user's repository; reading it can run a stale suite.
+- **So is everything that resolves in that directory**, through a finder scoped
+  to it alone — because the application under test is what changes between two
+  runs, and a stale copy of it hides the very regression the tool exists to
+  find.
+
+See ADR 0002 §9 for all three.
+"""
+
+from __future__ import annotations
+
+import importlib
+import os
+import sys
+from importlib.machinery import (
+    EXTENSION_SUFFIXES,
+    SOURCE_SUFFIXES,
+    ExtensionFileLoader,
+    FileFinder,
+    SourceFileLoader,
+)
+from pathlib import Path
+from types import CodeType, ModuleType
+from typing import TYPE_CHECKING
+
+from digline.run import Suite, Target
+
+if TYPE_CHECKING:
+    from _typeshed import ReadableBuffer
+
+__all__ = [
+    "SUITE_ATTR",
+    "TARGET_ATTR",
+    "SourceOnlyLoader",
+    "UsageError",
+    "load_suite",
+    "load_target",
+]
+
+SUITE_ATTR = "suite"
+TARGET_ATTR = "target"
+
+
+class SourceOnlyLoader(SourceFileLoader):
+    """A loader that always compiles from source, ignoring `__pycache__`.
+
+    Used **only** for the directory holding the suite, which is where the
+    application under test lives. Everything else — the standard library, site
+    packages, the rest of the user's environment — keeps the normal machinery,
+    because bytecode caching there is a performance win with no correctness
+    cost: those modules do not change between two runs of an evaluation.
+    """
+
+    def get_code(self, fullname: str) -> CodeType:
+        path = self.get_filename(fullname)
+        return self.source_to_code(self.get_data(path), path)  # type: ignore[return-value]
+
+    def set_data(self, path: str, data: ReadableBuffer, *, _mode: int = 0o666) -> None:
+        """Never write bytecode: the artifact is what dirties the user's tree."""
+        return None
+
+
+def _finder(directory: str) -> FileFinder:
+    # Deliberately without `BYTECODE_SUFFIXES`: in this directory a lone `.pyc`
+    # must not be importable at all.
+    return FileFinder(
+        directory,
+        (SourceOnlyLoader, SOURCE_SUFFIXES),
+        (ExtensionFileLoader, EXTENSION_SUFFIXES),
+    )
+
+
+_scoped: set[str] = set()
+
+
+def _scope_to_source(directory: str) -> None:
+    """Make imports resolving in `directory` read from disk, always.
+
+    The reason, found by running the tool and reading the wrong answer: the
+    suite itself is compiled from source, but everything it *imports* went
+    through the normal machinery — so a helper module edited within the same
+    second and to the same length was served from a stale `.pyc`, and a
+    comparison reported that nothing had got worse when something had.
+
+    Of every defect met in this project this is the only kind that can hide a
+    regression, which is why it is worth a finder rather than a warning.
+
+    The scope is one directory on purpose. A loader for every module would slow
+    every import to protect files that do not change during an evaluation, and
+    would reach far outside anything this tool has a right to alter.
+    """
+    real = os.path.realpath(directory)
+    if real in _scoped:
+        return
+    _scoped.add(real)
+
+    def hook(path: str) -> FileFinder:
+        if os.path.realpath(path) != real:
+            # Not ours: let the next hook — the normal one — answer.
+            raise ImportError(f"not the suite directory: {path}")
+        return _finder(path)
+
+    sys.path_hooks.insert(0, hook)
+    # The cache is consulted before the hooks, so the entry has to be replaced
+    # too — otherwise a finder built earlier would keep serving this directory.
+    sys.path_importer_cache[directory] = _finder(directory)
+
+
+class UsageError(Exception):
+    """Something the caller can fix by typing a different command."""
+
+
+def _split(spec: str) -> tuple[str, str | None]:
+    """`"pkg.mod:name"` or `"file.py:name"` into module part and attribute.
+
+    Only a trailing `:name` counts, and only when `name` is an identifier, so a
+    Windows path like `C:\\suites\\qa.py` is not mistaken for one.
+    """
+    head, sep, tail = spec.rpartition(":")
+    if sep and tail.isidentifier():
+        return head, tail
+    return spec, None
+
+
+def _import(module_part: str, spec: str) -> ModuleType:
+    if module_part.endswith(".py") or "/" in module_part or "\\" in module_part:
+        path = Path(module_part).resolve()
+        if not path.is_file():
+            raise UsageError(f"no such file: {path} (from {spec!r})")
+
+        # Compiled from source rather than imported through the normal
+        # machinery, which reads and writes `__pycache__`. Two reasons, both
+        # found by running this against a real repository:
+        #
+        # 1. Writing bytecode leaves artifacts in the user's repository, and a
+        #    repository with new untracked files is a *dirty* one — our own
+        #    import would have made every run report itself unreproducible.
+        # 2. Reading bytecode can run a stale suite. Python's freshness check is
+        #    (mtime, size) with one-second granularity, so a file edited within
+        #    the same second and to the same length runs from cache. For a tool
+        #    whose premise is reproducibility, "what is on disk" is the only
+        #    acceptable answer.
+        name = f"_digline_suite_{path.stem}"
+        try:
+            code = compile(path.read_text(encoding="utf-8"), str(path), "exec")
+        except SyntaxError as exc:
+            raise UsageError(f"{path} does not parse: {exc}") from exc
+
+        # The suite's own directory goes on the path before it runs.
+        #
+        # A suite imports the application it evaluates — that is what a suite
+        # *is*, not an exceptional case — so `from brief import judge` has to
+        # resolve the way it would under `python suite.py` or under pytest from
+        # its rootdir. Without this the first real suite fails on its first line
+        # with ModuleNotFoundError, and the tool looks broken because it is.
+        #
+        # Guarded against duplicates: several suites in one directory, or one
+        # loaded twice, must not make `sys.path` grow.
+        directory = str(path.parent)
+        if directory not in sys.path:
+            sys.path.insert(0, directory)
+        # …and what resolves there is read from disk, never from bytecode.
+        _scope_to_source(directory)
+
+        module = ModuleType(name)
+        module.__file__ = str(path)
+        # Registered before execution so a module using dataclasses that refer
+        # to their own names resolves normally.
+        sys.modules[name] = module
+        exec(code, module.__dict__)  # noqa: S102 — the user's own suite, by request
+        return module
+
+    try:
+        return importlib.import_module(module_part)
+    except ImportError as exc:
+        raise UsageError(f"cannot import module {module_part!r}: {exc}") from exc
+
+
+def _pick(module: ModuleType, attr: str, spec: str, kind: str) -> object:
+    value = getattr(module, attr, None)
+    if value is None:
+        available = ", ".join(sorted(n for n in vars(module) if not n.startswith("_")))
+        raise UsageError(
+            f"{spec!r} has no attribute {attr!r} to use as the {kind}. "
+            f"Define it, or name it explicitly as '{spec}:<attribute>'. "
+            f"Module defines: {available or '(nothing public)'}"
+        )
+    return value
+
+
+def load_suite(spec: str) -> tuple[Suite, ModuleType]:
+    """The `Suite`, and the module it came from — the target defaults to the
+    same module, so the caller usually needs only one path."""
+    module_part, attr = _split(spec)
+    module = _import(module_part, spec)
+    value = _pick(module, attr or SUITE_ATTR, spec, "suite")
+    if not isinstance(value, Suite):
+        raise UsageError(f"{spec!r} gave a {type(value).__name__}, not a Suite")
+    return value, module
+
+
+def load_target(spec: str | None, suite_module: ModuleType, suite_spec: str) -> Target:
+    """`--target` uses the same syntax as `--suite`. Omitted, it is looked up as
+    `target` in the suite's own module and nowhere else."""
+    if spec is None:
+        value = _pick(suite_module, TARGET_ATTR, suite_spec, "target")
+    else:
+        module_part, attr = _split(spec)
+        module = _import(module_part, spec)
+        value = _pick(module, attr or TARGET_ATTR, spec, "target")
+    if not callable(value):
+        raise UsageError(f"the target is a {type(value).__name__}, not callable")
+    return value  # type: ignore[return-value]