mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/ux/workload.py

@@ -0,0 +1,1077 @@
+"""The workload a build would run, judged against the ceilings that will judge it.
+
+``mr-data preflight`` used to ask only what this *machine* can do. That is a real question and it
+is still asked, but it is not the question a person who is about to deploy has: theirs is whether
+*this job* can run at all. Today the first honest answer to that arrives as an opaque failure
+inside a container, after the sources have been uploaded and the deployment approved, which is
+the shape of both #234 and #235.
+
+This module answers it before any of that, from two things and nothing else: the plan, and one
+confined ``stat`` per source. No source is opened or parsed, nothing is fetched, and no graph is
+executed. That bound is what makes the answer cheap enough to be the first thing a person runs.
+
+The stat is confined the way the build's own read is confined -- component by component, with
+``O_NOFOLLOW``, through the pipeline's own ``_require_source_directory`` and
+``_require_source_file``. A plain path join would let the kernel follow a directory link, which
+sizes a file the build refuses and stats something outside the folder this command was pointed at.
+
+**Every ceiling here is imported, never restated.** ``MAX_SOURCE_BYTES`` is the pipeline's own
+constant, ``HARD_MAX_RETAINED_BYTES`` is the graph kernel's own, ``ReaderBudgets`` is the Reader
+contract's own. A number written down twice is a number that drifts, and a preflight that judged
+a workload against a stale copy of a ceiling would be worse than no preflight: it would say a job
+fits and then the engine would refuse it. :class:`Ceiling` therefore carries the constant's own
+spelling beside the value, so the rendering names what a reader can go and look at.
+
+**A claim is only ever as strong as the observation under it.** Three kinds of claim are
+distinguished, because they license different conclusions:
+
+``MEASURED``
+    The exact number the engine will see -- a file's size on disk is the size the build reads.
+    Over the ceiling means refused, and under it means admitted.
+
+``AT_LEAST``
+    A floor. The sealed Build holds the raw sources *and* the derived members, so the raw total is
+    a floor under its size. A floor over the ceiling refuses, because the real value is at least
+    the floor. A floor *under* it settles nothing and is reported undecided -- the symmetric
+    mistake to the one below, and the one that let "at least 199 bytes, which is within
+    MAX_CANDIDATE_BYTES" be printed about a Build whose size nothing here had established.
+
+``AT_MOST``
+    A bound in the other direction. A CSV row costs at least one byte, so a source cannot hold
+    more rows than it holds bytes. A bound *under* the ceiling proves the ceiling cannot be
+    reached; a bound over it proves nothing at all, and is reported as undecided rather than as a
+    refusal. This is the direction that keeps the guarantee in
+    ``tests/test_ux_workload.py`` true: no verdict here says a job fits that the engine
+    then refuses.
+
+**What cannot be known here is said, not guessed.** How many rows a join emits, how much the
+kernel retains while it runs, and -- the one #234 was read as until its own follow-up found the
+harness's eager allocation instead -- how much memory the machine that will execute this job
+actually has, are all outside what a plan and a stat can answer. :func:`unknowns_for` names each
+of them with the ceiling that will be applied and where it is applied, so the report is honest
+about its own edges instead of quietly implying it checked. It
+is derived from the workload rather than fixed, because a sentence about the graph kernel printed
+over a tabular plan describes a check that did not happen.
+
+Nothing here prints, writes, or reaches a network. The result is a value
+:mod:`mostlyright.data_harness.ux.preflight` renders alongside its own checks.
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath
+from typing import Any
+
+from mostlyright.data_harness.local_contracts import (
+    GRAPH_PLAN_SCHEMA_VERSION,
+    MAX_COLUMN_COUNT,
+    MAX_GRAPH_NODE_COUNT,
+    MAX_OPERATION_COUNT,
+    MAX_OUTPUT_ROWS,
+    MAX_SOURCE_COUNT,
+    PLAN_SCHEMA_VERSION,
+)
+from mostlyright.data_harness.local_contracts import (
+    GraphTablePlan as GraphDatasetPlan,
+)
+from mostlyright.data_harness.local_contracts import (
+    TablePlan as DatasetPlan,
+)
+from mostlyright.data_harness.offline import (
+    _DEFINITION_FILES,
+    CONTROL_PATH,
+    DEFINITION_PATH,
+    INPUT_ROOT_PATH,
+)
+from mostlyright.data_harness.pipeline import (
+    MAX_CANDIDATE_BYTES,
+    MAX_CANDIDATE_MEMBER_BYTES,
+    MAX_COLUMNS,
+    MAX_FIELD_BYTES,
+    MAX_ROWS,
+    MAX_SOURCE_BYTES,
+)
+from mostlyright.data_harness.plan_graph import (
+    HARD_MAX_INTERMEDIATE_ROWS,
+    HARD_MAX_SOURCE_ROWS,
+    GraphResourcePolicy,
+)
+from mostlyright.data_harness.ux.path_kind import (
+    A_FOLDER,
+    NOTHING_THERE,
+    kind_at,
+    presence_at,
+)
+
+# The file a workbench folder keeps its plan in, taken from the coordinator's own table rather than
+# spelled again here: a rename there would otherwise leave this reporting that the workload could
+# not be read.
+_PLAN_DEFINITION = _DEFINITION_FILES["plan"]
+
+# What kind of workload was named, in the words the report uses. These are values a person reads.
+A_WORKBENCH = "a workbench folder"
+A_PLAN = "a plan"
+A_RECIPE = "a recipe"
+
+# Which engine a plan runs on. Both spellings are display words derived from the plan's own schema
+# version, which is the contract value and is never renamed here.
+GRAPH_EXECUTION = "the graph engine"
+TABULAR_EXECUTION = "the tabular engine"
+
+# How strong one observation is. See the module docstring: only the first two license a refusal.
+MEASURED = "measured"
+AT_LEAST = "at least"
+AT_MOST = "at most"
+
+# What one judged fact came out as.
+WITHIN = "within"
+OVER = "over"
+UNDECIDED = "undecided"
+
+
+# The two ways a named workload fails to become facts. Both are literals at their raise sites and
+# both carry their own remediation, because a person who typed a path that is not a plan is not
+# helped by a paragraph about packaging a Build for Studio, which is what the ``WORKLOAD`` prefix
+# family answers.
+WORKLOAD_UNREADABLE = "WORKLOAD_UNREADABLE"
+WORKLOAD_INVALID = "WORKLOAD_INVALID"
+
+
+class WorkloadUnreadable(Exception):
+    """The named workload could not be read, with the typed code and plain reason why.
+
+    Raised rather than reported, because preflight's caller turns it back into a *check* -- a
+    workload that cannot be read is an answer to the question preflight asks, and it belongs on
+    the same line as every other answer rather than as a traceback.
+
+    ``refused_as`` is the code of the refusal underneath, when there was one: the contract reader
+    and the recipe reader raise their own typed codes, and those are more specific than either of
+    the two above. It is carried beside the code rather than *as* it so that every raise site here
+    writes a literal, which is what keeps these two codes visible to the gate that proves every
+    typed code names a fix.
+    """
+
+    def __init__(self, code: str, detail: str, *, refused_as: str | None = None) -> None:
+        self.code = code
+        self.detail = detail
+        self.refused_as = refused_as
+        super().__init__(detail)
+
+
+@dataclass(frozen=True)
+class Ceiling:
+    """One fact about this workload, and the exact bound the engine will judge it against.
+
+    ``limit_name`` is the constant's own spelling -- ``plan_graph.HARD_MAX_RETAINED_BYTES``, not a
+    prose paraphrase -- so a reader who doubts the number can open the file it names. ``code`` is
+    the typed code the engine raises when the bound is crossed, so this report and the refusal it
+    predicts say one word rather than two.
+    """
+
+    fact: str
+    strength: str
+    observed: int | None
+    limit: int
+    limit_name: str
+    code: str
+    subject: str | None = None
+
+    @property
+    def verdict(self) -> str:
+        """``over`` only where the observation licenses it; see the module docstring."""
+
+        if self.observed is None:
+            return UNDECIDED
+        if self.observed <= self.limit:
+            # The mirror image of the rule below, and the one that was missing. A *floor* under the
+            # ceiling licenses nothing: the real value is somewhere above the floor and may be
+            # above the ceiling too. Reporting that as "within" would be this report saying a job
+            # fits that the engine then refuses, which is the one thing it may never do.
+            return UNDECIDED if self.strength == AT_LEAST else WITHIN
+        # A bound in the "at most" direction says nothing when it sits above the ceiling: the real
+        # number is somewhere under the bound and may be under the ceiling too. Calling that a
+        # refusal would stop work that would have finished, which is the expensive direction to be
+        # wrong in.
+        return UNDECIDED if self.strength == AT_MOST else OVER
+
+    @property
+    def refuses(self) -> bool:
+        return self.verdict == OVER
+
+    @property
+    def sentence(self) -> str:
+        """This fact, its bound, and what it came to, as one plain line."""
+
+        named = f"{self.subject}: " if self.subject else ""
+        if self.observed is None:
+            return (
+                f"{named}{self.fact} is not knowable here; the bound is {self.limit} "
+                f"({self.limit_name}), refused as {self.code}"
+            )
+        strength = "" if self.strength == MEASURED else f"{self.strength} "
+        judged = {
+            WITHIN: "which is within",
+            OVER: "which is over",
+            UNDECIDED: "which decides nothing about",
+        }[self.verdict]
+        return (
+            f"{named}{self.fact} is {strength}{self.observed}, {judged} the "
+            f"{self.limit} of {self.limit_name} ({self.code})"
+        )
+
+
+@dataclass(frozen=True)
+class Refused:
+    """A refusal this preflight established that is not a number against a ceiling.
+
+    Some of what stops a build is not a quantity. A source reached through a directory symlink, a
+    source with two hard links, and a recipe pinning a Reader coordinate this build does not ship
+    are each refused outright, with no number to compare -- and each is refused by the same typed
+    code the engine raises, because the whole point is that one answer is given twice rather than
+    two answers given once.
+    """
+
+    subject: str
+    code: str
+    sentence: str
+
+
+@dataclass(frozen=True)
+class SourceFacts:
+    """One source of this workload: where it is, how big it is, and what bounds it."""
+
+    source_id: str
+    path: str
+    data_format: str
+    looked_at: str
+    bytes_on_disk: int | None
+    unreadable: str | None
+    reader: str | None
+    reader_budgets: Mapping[str, int] | None
+    ceilings: tuple[Ceiling, ...]
+    refused: tuple[Refused, ...] = ()
+
+    @property
+    def sentence(self) -> str:
+        if self.unreadable is not None:
+            return f"{self.path} ({self.data_format}): {self.unreadable}"
+        opened_by = ""
+        if self.reader is not None:
+            # The budgets are part of the answer, not a detail behind it: "which Reader, under
+            # which budgets" is one of the facts this command was asked for, and a family name on
+            # its own does not say what that family will admit.
+            under = (
+                ", ".join(f"{name} {value}" for name, value in sorted(self.reader_budgets.items()))
+                if self.reader_budgets
+                else "budgets this build cannot resolve"
+            )
+            opened_by = f", read by {self.reader} under {under}"
+        return f"{self.path} ({self.data_format}): {self.bytes_on_disk} bytes{opened_by}"
+
+
+@dataclass(frozen=True)
+class Workload:
+    """Everything a plan and one stat per source can say about whether this job can run."""
+
+    kind: str
+    named: str
+    plan_schema_version: str
+    execution: str
+    sources: tuple[SourceFacts, ...]
+    ceilings: tuple[Ceiling, ...]
+    unknowns: tuple[tuple[str, str], ...]
+
+    @property
+    def every_ceiling(self) -> tuple[Ceiling, ...]:
+        """Every judged fact, per source first and then aggregate, in the order they are met."""
+
+        per_source = [ceiling for source in self.sources for ceiling in source.ceilings]
+        return (*per_source, *self.ceilings)
+
+    @property
+    def refusals(self) -> tuple[Ceiling, ...]:
+        return tuple(ceiling for ceiling in self.every_ceiling if ceiling.refuses)
+
+    @property
+    def refused(self) -> tuple[Refused, ...]:
+        return tuple(item for source in self.sources for item in source.refused)
+
+    @property
+    def blocked(self) -> bool:
+        return bool(self.refusals) or bool(self.refused)
+
+    @property
+    def sentence(self) -> str:
+        """The one line the preflight check carries: what was read, and what it came to."""
+
+        counted = len(self.sources)
+        noun = "source" if counted == 1 else "sources"
+        read = f"{self.kind} at {self.named}, {self.plan_schema_version} on {self.execution}, "
+        stopped: list[str] = [item.sentence for item in self.refused]
+        stopped.extend(ceiling.sentence for ceiling in self.refusals)
+        if not stopped:
+            return (
+                f"{read}{counted} {noun}; nothing this preflight can measure is over an "
+                "engine ceiling"
+            )
+        rest = len(stopped) - 1
+        more = f", and {rest} more" if rest else ""
+        return f"{read}{counted} {noun}; {stopped[0]}{more}"
+
+    @property
+    def remediation(self) -> tuple[str, ...]:
+        """What to do about every refusal, each line said once, in the order they are met."""
+
+        lines: list[str] = []
+        for item in self.refused:
+            for line in _remediation_for_refusal(item.code, item.sentence, item.subject):
+                if line not in lines:
+                    lines.append(line)
+        for ceiling in self.refusals:
+            for line in _remediation_for(ceiling):
+                if line not in lines:
+                    lines.append(line)
+        return tuple(lines)
+
+    def to_block(self) -> dict[str, Any]:
+        """The structured block preflight carries alongside its checks."""
+
+        width = len(str(max(len(self.sources), 1)))
+        bound_width = len(str(max(len(self.every_ceiling), 1)))
+        block: dict[str, Any] = {
+            "what it is": (
+                f"{self.kind} at {self.named}, written as {self.plan_schema_version}, "
+                f"which runs on {self.execution}"
+            ),
+            "its sources": {
+                f"source {index:0{width}d}": source.sentence
+                for index, source in enumerate(self.sources, start=1)
+            },
+            "what it needs, against what is allowed": {
+                f"bound {index:0{bound_width}d}": ceiling.sentence
+                for index, ceiling in enumerate(self.every_ceiling, start=1)
+            },
+            "what this cannot know": dict(self.unknowns),
+        }
+        if self.refused:
+            block["what stops it outright"] = {
+                f"{item.subject} ({item.code})": item.sentence for item in self.refused
+            }
+        return block
+
+
+# ------------------------------------------------------------------------------------------------
+# What cannot be answered from a plan and a stat
+# ------------------------------------------------------------------------------------------------
+#
+# Written down rather than left out. An omitted fact reads as a checked one, and the whole value of
+# this command is that a person can tell the difference between "this was judged and it is fine"
+# and "nothing here judged this".
+
+
+def unknowns_for(*, graph: bool, recipe: bool) -> tuple[tuple[str, str], ...]:
+    """What this reading of *this* workload could not answer, and what will answer it.
+
+    Derived rather than fixed. A static list said "accounted per node by the graph kernel" about a
+    tabular plan that never reaches the kernel, and pointed at "the bound above" for a bound only
+    the graph branch produces -- two sentences that described a check that had not happened. Each
+    entry below names the bound it is about by its own words rather than by its position, because
+    the rendering sorts these keys and "above" is not a place.
+    """
+
+    # Imported here rather than at the top: `hosted_worker` is the hosted execution surface and
+    # pulls the Studio boundary with it, which a local preflight has no business loading to print
+    # one number.
+    from mostlyright.data_harness.hosted_worker import MAX_HOSTED_CANDIDATE_BYTES
+
+    rows = HARD_MAX_SOURCE_ROWS if graph else MAX_ROWS
+    unknowns: list[tuple[str, str]] = [
+        (
+            "how many rows a source really holds",
+            "counted while the source is streamed, not before. What is knowable here is the bound "
+            f"its bytes allow, and the ceiling this plan runs under is {rows}",
+        ),
+        (
+            "how wide a source is, and how long its longest field is",
+            "both are read out of the source's own header and rows, which this command does not "
+            f"open. The build refuses a source over {MAX_COLUMNS} columns as CSV_COLUMNS and a "
+            f"field over {MAX_FIELD_BYTES} bytes as CSV_FIELD. The column bound reported above is "
+            "the plan's own declared output width, which is a different number",
+        ),
+        (
+            "how many rows this job finally produces",
+            "the result of running it. What is reported above is the row count the plan's own "
+            "quality gate demands, which is a floor the plan states rather than an estimate",
+        ),
+        (
+            "how much memory and processing the machine that runs this job has",
+            "no contract this command can read states it. Every bound above is the engine's own "
+            "ceiling, held whatever the container is given, so a workload that passes here has "
+            "not been shown to fit any particular hosted container -- see issue #234, where "
+            "hosted runs died with MemoryError while the worker's measured footprint sat far "
+            "below its container's limit",
+        ),
+        (
+            "whether the sealed Build fits the hosted handoff envelope",
+            f"a hosted Build member is capped at {MAX_HOSTED_CANDIDATE_BYTES} bytes "
+            "(hosted_worker.MAX_HOSTED_CANDIDATE_BYTES, refused as CANDIDATE_ENVELOPE_INVALID) "
+            "and the Courier contract caps are checked by the acquire path. Neither is judged "
+            "here: both are about the packaged Build, which does not exist until this job has run",
+        ),
+    ]
+    if graph:
+        unknowns.append(
+            (
+                "how many rows the steps in between hold",
+                "decided per node while the graph runs, because a join's output size is a "
+                "property of the data rather than of the plan; the ceiling is "
+                f"{HARD_MAX_INTERMEDIATE_ROWS} (plan_graph.HARD_MAX_INTERMEDIATE_ROWS), refused "
+                "as RESOURCE_POLICY_INVALID",
+            )
+        )
+        unknowns.append(
+            (
+                "how many bytes the run itself holds while it runs",
+                "accounted per node by the graph kernel. What is knowable here is the part "
+                "already spoken for, which is the bound named raw source bytes plus one decode "
+                "transient",
+            )
+        )
+    unknowns.append(
+        (
+            "how many bytes a source arrived as before it was normalized",
+            "a Reader's input budget is applied to the bytes fetched from a publisher, and what "
+            "is on disk here is the normalized CSV that decode produced. The fetched size is not "
+            "observable from a plan and a stat; it is refused as READER_BUDGET at acquisition"
+            + (
+                ", against the pinned family's max_input_bytes"
+                if recipe
+                else ", before a source becomes a plan source"
+            ),
+        )
+    )
+    if not recipe:
+        unknowns.append(
+            (
+                "which reader opens a source, and under which budgets",
+                "a plan source is already-normalized CSV and pins no Reader family. A family is "
+                "pinned on a recipe source; point --workload at the recipe to have its "
+                "coordinate and its narrowed budgets reported",
+            )
+        )
+    return tuple(unknowns)
+
+
+# ------------------------------------------------------------------------------------------------
+# Reading the workload
+# ------------------------------------------------------------------------------------------------
+
+
+def read_workload(workload: Path | str, *, input_root: Path | str | None = None) -> Workload:
+    """Derive the bounded facts of one workload, without opening a source or running anything.
+
+    ``workload`` is a workbench folder, a plan file, or a frozen recipe file. A workbench folder
+    carries its own sources under ``inputs/`` and its own plan under ``definitions/``; the other
+    two name their sources relative to ``input_root``, and report the sizes as unknowable when no
+    input folder was given, because a relative path with nothing to resolve it against is not a
+    file this command may guess at.
+    """
+
+    named = Path(workload)
+    if kind_at(named / CONTROL_PATH) == A_FOLDER:
+        return _from_workbench(named)
+    return _from_document(named, input_root=input_root)
+
+
+def _from_workbench(workspace: Path) -> Workload:
+    """A workbench folder: the plan it recorded, and the sources it already holds."""
+
+    plan_path = workspace / DEFINITION_PATH / _PLAN_DEFINITION
+    plan = _plan_from(_read_document(plan_path))
+    return _facts(
+        plan,
+        kind=A_WORKBENCH,
+        named=str(workspace),
+        sources_under=workspace / INPUT_ROOT_PATH,
+        readers={},
+    )
+
+
+def _from_document(document: Path, *, input_root: Path | str | None) -> Workload:
+    """A plan file or a recipe file, whose sources are read from the named input folder."""
+
+    value = _read_document(document)
+    schema_version = value.get("schema_version") if isinstance(value, Mapping) else None
+    if isinstance(schema_version, str) and schema_version.startswith("frozen-recipe."):
+        plan, readers, kind = _recipe(value)
+    else:
+        plan, readers, kind = _plan_from(value), {}, A_PLAN
+    return _facts(
+        plan,
+        kind=kind,
+        named=str(document),
+        sources_under=None if input_root is None else Path(input_root),
+        readers=readers,
+    )
+
+
+def _read_document(path: Path) -> Any:
+    """One JSON document, read through the command line's own bounded reader.
+
+    Not ``path.read_bytes()``. Every document reader in this package goes through
+    ``cli._read_json`` -- open once, ask the descriptor what it is, read it under a bound -- and a
+    preflight pointed at a FIFO would otherwise wait for a writer that never comes, which is the
+    recorded failure this reader exists to prevent.
+    """
+
+    # Imported here, not at the top: this package is loaded while the command line is still being
+    # set up.
+    from mostlyright.data_harness import cli
+
+    try:
+        return cli._read_json(path)
+    except Exception as error:
+        raise WorkloadUnreadable(
+            "WORKLOAD_UNREADABLE",
+            f"{path} could not be read as a plan or a recipe: {error}",
+            refused_as=_inner_code(error),
+        ) from error
+
+
+def _inner_code(error: BaseException) -> str | None:
+    """The typed code of the refusal underneath, when the reader that raised it carried one."""
+
+    code = getattr(error, "code", None)
+    return code if isinstance(code, str) else None
+
+
+def _plan_from(value: Any) -> DatasetPlan | GraphDatasetPlan:
+    """The plan this document holds, parsed by the contract's own reader."""
+
+    from mostlyright.data_harness.local_contracts import parse_plan_document
+
+    try:
+        return parse_plan_document(value)
+    except Exception as error:
+        raise WorkloadUnreadable(
+            "WORKLOAD_INVALID",
+            f"that is not a plan this build can read: {error}",
+            refused_as=_inner_code(error),
+        ) from error
+
+
+def _recipe(value: Any) -> tuple[DatasetPlan | GraphDatasetPlan, dict[str, Any], str]:
+    """A frozen recipe: the plan it froze, and the Reader each of its sources is pinned to."""
+
+    from mostlyright.data_harness.recipe import parse_frozen_recipe
+
+    try:
+        recipe = parse_frozen_recipe(value)
+    except Exception as error:
+        raise WorkloadUnreadable(
+            "WORKLOAD_INVALID",
+            f"that is not a recipe this build can read: {error}",
+            refused_as=_inner_code(error),
+        ) from error
+    readers = {source.plan_source_path: source for source in recipe.sources}
+    return recipe.transform_plan, readers, A_RECIPE
+
+
+# ------------------------------------------------------------------------------------------------
+# Judging it
+# ------------------------------------------------------------------------------------------------
+
+
+def _facts(
+    plan: DatasetPlan | GraphDatasetPlan,
+    *,
+    kind: str,
+    named: str,
+    sources_under: Path | None,
+    readers: Mapping[str, Any],
+) -> Workload:
+    graph = isinstance(plan, GraphDatasetPlan)
+    sources = tuple(
+        _source_facts(
+            source,
+            sources_under=sources_under,
+            reader=readers.get(source.path),
+            graph=graph,
+        )
+        for source in plan.sources
+    )
+    return Workload(
+        kind=kind,
+        named=named,
+        plan_schema_version=GRAPH_PLAN_SCHEMA_VERSION if graph else PLAN_SCHEMA_VERSION,
+        execution=GRAPH_EXECUTION if graph else TABULAR_EXECUTION,
+        sources=sources,
+        ceilings=_aggregate_ceilings(plan, sources, graph=graph),
+        unknowns=unknowns_for(graph=graph, recipe=kind == A_RECIPE),
+    )
+
+
+def _source_facts(
+    source: Any,
+    *,
+    sources_under: Path | None,
+    reader: Any,
+    graph: bool,
+) -> SourceFacts:
+    """One source, stated once by the build's own observation and judged against its ceilings.
+
+    The observation is the writer's, deliberately. ``pipeline._read_confined_source_from_handle``
+    walks every component of the relative path with ``O_NOFOLLOW`` and ``dir_fd``, applies
+    ``_require_source_directory`` to each folder and ``_require_source_file`` to the leaf, and only
+    then refuses ``SOURCE_TOO_LARGE`` from ``st_size`` -- before it reads a byte. A preflight that
+    resolved the path with an ordinary join instead would let the kernel follow a directory link,
+    which reports the size of a file the build refuses outright and also stats something outside
+    the folder this command was pointed at. So the walk is done here the same way, and the two
+    refusals it can raise are reported under the codes the build raises them under.
+    """
+
+    budgets, unresolved = _reader_facts(reader)
+    coordinate = _reader_coordinate(reader)
+    refused = () if unresolved is None else (Refused(source.source_id, *unresolved),)
+
+    def unmeasured(reason: str) -> SourceFacts:
+        return SourceFacts(
+            source_id=source.source_id,
+            path=source.path,
+            data_format=source.format,
+            looked_at="" if sources_under is None else str(sources_under / source.path),
+            bytes_on_disk=None,
+            unreadable=reason,
+            reader=coordinate,
+            reader_budgets=budgets,
+            ceilings=(),
+            refused=refused,
+        )
+
+    if sources_under is None:
+        return unmeasured(
+            "no input folder was named, so there is nothing to resolve this relative path "
+            "against and its size was not guessed at"
+        )
+    from mostlyright.data_harness.pipeline import BuildError
+
+    try:
+        info = _confined_stat(sources_under, source.path)
+    except BuildError as stopped:
+        # The build's own refusal, carried through as it was raised. No second class and no second
+        # vocabulary: the code a person quotes here is the code the build would have printed, and
+        # `finding_id` is where `pipeline.BuildError` keeps it.
+        return SourceFacts(
+            source_id=source.source_id,
+            path=source.path,
+            data_format=source.format,
+            looked_at=str(sources_under / source.path),
+            bytes_on_disk=None,
+            unreadable=str(stopped),
+            reader=coordinate,
+            reader_budgets=budgets,
+            ceilings=(),
+            refused=(
+                *refused,
+                Refused(source.source_id, stopped.finding_id, str(stopped)),
+            ),
+        )
+    except OSError as error:
+        where = sources_under / source.path
+        found = presence_at(where)
+        if found == NOTHING_THERE:
+            return unmeasured(f"there is {found} at {where} to read")
+        # A path that is not there is the commonest case by far, and it is worth its own sentence.
+        # Everything else is reported by the errno the walk actually met, and by nothing else: a
+        # second look at a path whose stat has just failed would put a noun in front of a person
+        # that this command did not establish, which `scripts/path_kind_gate.py` is right to refuse.
+        return unmeasured(f"{where} could not be measured: {error.strerror or error}")
+    size = info.st_size
+    return SourceFacts(
+        source_id=source.source_id,
+        path=source.path,
+        data_format=source.format,
+        looked_at=str(sources_under / source.path),
+        bytes_on_disk=size,
+        unreadable=None,
+        reader=coordinate,
+        reader_budgets=budgets,
+        ceilings=_source_ceilings(source.source_id, size, budgets=budgets, graph=graph),
+        refused=refused,
+    )
+
+
+def _confined_stat(root: Path, relative: str) -> os.stat_result:
+    """Stat one source the way the build reaches it: component by component, following nothing.
+
+    ``pipeline._require_source_directory`` and ``pipeline._require_source_file`` are the build's
+    own tests, called here rather than re-implemented, so the shapes this reports and the shapes
+    the build refuses cannot become two lists. Their ``BuildError`` is allowed to propagate exactly
+    as raised -- ``SOURCE_SYMLINK``, ``SOURCE_NOT_DIRECTORY``, ``SOURCE_NOT_REGULAR``,
+    ``SOURCE_HARDLINK`` -- because this command exists to say the build's refusal early rather than
+    to invent a second class and a second vocabulary for it.
+
+    Nothing is read and nothing is kept: the descriptors opened are folder descriptors on the way
+    down, and every one of them is closed before this returns.
+    """
+
+    from mostlyright.data_harness.pipeline import (
+        _require_source_directory,
+        _require_source_file,
+    )
+
+    flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
+    parts = PurePosixPath(relative).parts
+    opened: list[int] = []
+    try:
+        parent_fd = os.open(root, flags)
+        opened.append(parent_fd)
+        walked = ""
+        for part in parts[:-1]:
+            walked = f"{walked}/{part}" if walked else part
+            named = os.stat(part, dir_fd=parent_fd, follow_symlinks=False)
+            _require_source_directory(named, walked)
+            parent_fd = os.open(part, flags, dir_fd=parent_fd)
+            opened.append(parent_fd)
+        leaf = os.stat(parts[-1], dir_fd=parent_fd, follow_symlinks=False)
+        _require_source_file(leaf, relative)
+        return leaf
+    finally:
+        for descriptor in opened:
+            os.close(descriptor)
+
+
+def _source_ceilings(
+    source_id: str,
+    size: int,
+    *,
+    budgets: Mapping[str, int] | None,
+    graph: bool,
+) -> tuple[Ceiling, ...]:
+    ceilings = [
+        Ceiling(
+            fact="bytes on disk",
+            strength=MEASURED,
+            observed=size,
+            limit=MAX_SOURCE_BYTES,
+            limit_name="pipeline.MAX_SOURCE_BYTES",
+            code="SOURCE_TOO_LARGE",
+            subject=source_id,
+        ),
+        Ceiling(
+            fact="bytes it adds to the sealed Build as one member",
+            strength=MEASURED,
+            observed=size,
+            limit=MAX_CANDIDATE_MEMBER_BYTES,
+            limit_name="pipeline.MAX_CANDIDATE_MEMBER_BYTES",
+            code="CANDIDATE_TOO_LARGE",
+            subject=source_id,
+        ),
+        # One CSV row costs at least the byte that ends it, so a source cannot hold more rows than
+        # it holds bytes. Under the ceiling that settles the question; over it, nothing is decided
+        # -- see the module docstring on AT_MOST.
+        Ceiling(
+            fact="rows its bytes allow",
+            strength=AT_MOST,
+            observed=size,
+            limit=HARD_MAX_SOURCE_ROWS if graph else MAX_ROWS,
+            limit_name=("plan_graph.HARD_MAX_SOURCE_ROWS" if graph else "pipeline.MAX_ROWS"),
+            code="CSV_ROWS",
+            subject=source_id,
+        ),
+    ]
+    if budgets is not None:
+        # The *output* budget, not the input one. What is on disk at a recipe source's
+        # `plan_source_path` is the normalized CSV a decode produced, and
+        # `ReaderResult.validate_for` holds exactly that against `max_output_bytes`. The input
+        # budget applies to the bytes
+        # fetched from the publisher, which are gone by the time this file exists -- judging this
+        # file by that ceiling refused normalized sources between the two, and the bulk families
+        # put 16x between them. The fetched size is named in the unknowns instead.
+        #
+        # This is where a raster refuses: the weather.grib2 families take the plain ReaderBudgets
+        # defaults rather than the bulk ones, so both of their ceilings are 16 MiB, which no
+        # published model file fits under.
+        ceilings.append(
+            Ceiling(
+                fact="bytes its pinned Reader may emit",
+                strength=MEASURED,
+                observed=size,
+                limit=budgets["max_output_bytes"],
+                limit_name="the pinned Reader family's max_output_bytes",
+                code="READER_BUDGET",
+                subject=source_id,
+            )
+        )
+    return tuple(ceilings)
+
+
+def _aggregate_ceilings(
+    plan: DatasetPlan | GraphDatasetPlan,
+    sources: Sequence[SourceFacts],
+    *,
+    graph: bool,
+) -> tuple[Ceiling, ...]:
+    """The bounds that are about the whole job rather than about one source of it."""
+
+    sized = [source.bytes_on_disk for source in sources if source.bytes_on_disk is not None]
+    complete = len(sized) == len(sources) and bool(sources)
+    raw_total = sum(sized) if complete else None
+    decode_peak = max(sized, default=0) if complete else None
+    ceilings = [
+        Ceiling(
+            fact="sources this plan declares",
+            strength=MEASURED,
+            observed=len(plan.sources),
+            limit=MAX_SOURCE_COUNT,
+            limit_name="local_contracts.MAX_SOURCE_COUNT",
+            code="COLLECTION_LIMIT",
+        ),
+        # The plan's own declared output width against the contract's ceiling for it. It is
+        # deliberately *not* judged against `CSV_COLUMNS`, which the build raises about a source's
+        # header width -- a different number, unknowable from a stat, and named in the unknowns.
+        Ceiling(
+            fact="output columns this plan declares",
+            strength=MEASURED,
+            observed=len(plan.select),
+            limit=MAX_COLUMN_COUNT,
+            limit_name="local_contracts.MAX_COLUMN_COUNT",
+            code="COLLECTION_LIMIT",
+        ),
+    ]
+    if graph:
+        assert isinstance(plan, GraphDatasetPlan)
+        policy = GraphResourcePolicy()
+        ceilings.extend(
+            (
+                Ceiling(
+                    fact="graph nodes this plan declares",
+                    strength=MEASURED,
+                    observed=len(plan.nodes),
+                    limit=MAX_GRAPH_NODE_COUNT,
+                    limit_name="local_contracts.MAX_GRAPH_NODE_COUNT",
+                    code="COLLECTION_LIMIT",
+                ),
+                Ceiling(
+                    fact="graph operations this plan declares",
+                    strength=MEASURED,
+                    observed=sum(1 for node in plan.nodes if node.operation != "source"),
+                    limit=MAX_GRAPH_NODE_COUNT,
+                    limit_name="local_contracts.MAX_GRAPH_NODE_COUNT",
+                    code="COLLECTION_LIMIT",
+                ),
+                # `pipeline._replay_graph_derivations` refuses this before it executes a node.
+                Ceiling(
+                    fact="rows the plan's own quality gate demands",
+                    strength=MEASURED,
+                    observed=plan.quality.min_rows,
+                    limit=policy.max_output_rows,
+                    limit_name="plan_graph.HARD_MAX_OUTPUT_ROWS",
+                    code="ROW_BUDGET_EXCEEDED",
+                ),
+                Ceiling(
+                    fact="raw source bytes the run must hold at once",
+                    strength=MEASURED,
+                    observed=raw_total,
+                    limit=policy.max_retained_bytes,
+                    limit_name="plan_graph.HARD_MAX_RETAINED_BYTES",
+                    code="BYTE_BUDGET_EXCEEDED",
+                ),
+                # The kernel keeps the raw bytes and one decoded-text transient beside its own
+                # tables, and refuses when what is left of the envelope falls below one byte. Said
+                # here as the sum against one byte less than the envelope, which is the same test.
+                Ceiling(
+                    fact="raw source bytes plus one decode transient",
+                    strength=MEASURED,
+                    observed=None if raw_total is None else raw_total + (decode_peak or 0),
+                    limit=policy.max_retained_bytes - 1,
+                    limit_name="plan_graph.HARD_MAX_RETAINED_BYTES, less the one byte the "
+                    "execution itself must be left",
+                    code="BYTE_BUDGET_EXCEEDED",
+                ),
+            )
+        )
+    else:
+        ceilings.extend(
+            (
+                Ceiling(
+                    fact="cleaning steps this plan declares",
+                    strength=MEASURED,
+                    observed=len(plan.cleaning),
+                    limit=MAX_OPERATION_COUNT,
+                    limit_name="local_contracts.MAX_OPERATION_COUNT",
+                    code="COLLECTION_LIMIT",
+                ),
+                Ceiling(
+                    fact="rows the plan's own quality gate demands",
+                    strength=MEASURED,
+                    observed=plan.quality.min_rows,
+                    limit=MAX_OUTPUT_ROWS,
+                    limit_name="local_contracts.MAX_OUTPUT_ROWS",
+                    code="ROW_BUDGET_EXCEEDED",
+                ),
+            )
+        )
+    ceilings.append(
+        # A floor, not a measurement: the sealed Build holds every raw source as a member and the
+        # derived members besides. Over the ceiling it still refuses, because the real size is at
+        # least this.
+        Ceiling(
+            fact="bytes the sealed Build holds",
+            strength=AT_LEAST,
+            observed=raw_total,
+            limit=MAX_CANDIDATE_BYTES,
+            limit_name="pipeline.MAX_CANDIDATE_BYTES",
+            code="CANDIDATE_TOO_LARGE",
+        )
+    )
+    return tuple(ceilings)
+
+
+def _reader_coordinate(reader: Any) -> str | None:
+    """``family@version`` for a recipe source that pins one, and ``None`` for anything else."""
+
+    if reader is None or getattr(reader, "family_id", None) is None:
+        return None
+    return f"{reader.family_id}@{reader.family_version}"
+
+
+def _reader_facts(reader: Any) -> tuple[dict[str, int] | None, tuple[str, str] | None]:
+    """The budgets one decode will really run under, or the refusal that says there will be none.
+
+    This is ``acquisition.sandbox``'s own call -- ``family.default_budgets.narrowed_by(caps)`` --
+    made against the same registry, so the numbers reported here are the numbers the clean room
+    will apply rather than a second opinion about them.
+
+    The two ways it can fail are answered differently, and conflating them was a false pass. A
+    coordinate the Toolbox does not hold is a *refusal*: ``recipe.py`` resolves the same pin on the
+    execution path and ``READER_FAMILY_UNKNOWN`` is what it gets, so a preflight that dropped the
+    budgets and said nothing reported "read by weather.grib2@99.0.0" about a Reader that does not
+    exist, and passed. A family whose optional dependency is simply not installed here is not that:
+    the pin is real and this machine cannot answer for its budgets, which is said in the source's
+    own sentence rather than left blank.
+    """
+
+    if reader is None or getattr(reader, "family_id", None) is None:
+        return None, None
+    from mostlyright.data_harness.readers.contracts import ReaderError
+    from mostlyright.data_harness.readers.registry import TOOLBOX
+
+    coordinate = f"{reader.family_id}@{reader.family_version}"
+    try:
+        family = TOOLBOX.resolve(reader.family_id, reader.family_version)
+    except ReaderError as error:
+        if error.code == "READER_DEPENDENCY_UNAVAILABLE":
+            return None, None
+        offered = ", ".join(f"{name}@{version}" for name, version in TOOLBOX.families())
+        return None, (
+            error.code,
+            f"this recipe pins {coordinate}, which this build does not have. It has: {offered}",
+        )
+    caps = None if reader.resource_caps is None else dict(reader.resource_caps.to_dict())
+    try:
+        budgets = family.default_budgets.narrowed_by(caps)
+    except ReaderError as error:
+        return None, (error.code, f"the caps {coordinate} is narrowed by are refused: {error}")
+    return {
+        "max_input_bytes": budgets.max_input_bytes,
+        "max_output_bytes": budgets.max_output_bytes,
+        "max_declared_cells": budgets.max_declared_cells,
+        "max_rows": budgets.max_rows,
+        "max_columns": budgets.max_columns,
+    }, None
+
+
+# ------------------------------------------------------------------------------------------------
+# What to do about a refusal
+# ------------------------------------------------------------------------------------------------
+
+
+def _remediation_for_refusal(code: str, sentence: str, subject: str) -> tuple[str, ...]:
+    """What to do about a refusal that is not a number, in the words of the refusal it predicts."""
+
+    from mostlyright.data_harness.ux.remediation import remediation_for
+
+    return (
+        f"{subject}: {sentence}. A build refuses this as {code}. Nothing was uploaded and "
+        "nothing was run.",
+        *remediation_for(code),
+    )
+
+
+def _remediation_for(ceiling: Ceiling) -> tuple[str, ...]:
+    """What to do about one crossed bound, in the words of the refusal it predicts.
+
+    The error-remediation map answers by code, and these codes are shared with the mid-run
+    refusals, so the map's paragraph is the right one and is used. What it cannot say is the thing
+    a person most needs to hear here, which is that a bigger machine is not the answer: every
+    bound above is a semantic ceiling of the engine, and CPU and RAM do not move any of them. That
+    sentence is added for the byte and row bounds, where the instinct to ask for more memory is
+    strongest.
+    """
+
+    from mostlyright.data_harness.ux.remediation import remediation_for
+
+    lines = list(remediation_for(ceiling.code))
+    named = f" ({ceiling.subject})" if ceiling.subject else ""
+    # The typed code goes in the sentence, not only in the record: this payload never passes
+    # through the command line's error boundary, which is where a refusal's `Code:` line is
+    # printed, so a person reading the plain rendering would otherwise have nothing to quote.
+    lines.insert(
+        0,
+        f"This workload's {ceiling.fact}{named} is {ceiling.observed}, over the "
+        f"{ceiling.limit} that {ceiling.limit_name} allows, which a build refuses as "
+        f"{ceiling.code}. Nothing was uploaded and nothing was run.",
+    )
+    if ceiling.code in _NOT_A_MACHINE_SIZE:
+        lines.append(_PARTITIONED)
+    return tuple(lines)
+
+
+# The bounds a bigger machine does not move. Every one of them is a ceiling of the deterministic
+# engine, held whatever the container is given, so answering one with more RAM is answering the
+# wrong question -- which is the mistake #234 records being made in the other direction.
+_NOT_A_MACHINE_SIZE = frozenset(
+    {
+        "BYTE_BUDGET_EXCEEDED",
+        "CANDIDATE_TOO_LARGE",
+        "CSV_ROWS",
+        "READER_BUDGET",
+        "ROW_BUDGET_EXCEEDED",
+        "SOURCE_TOO_LARGE",
+    }
+)
+
+_PARTITIONED = (
+    "A larger machine does not raise this bound: it is a ceiling of the deterministic engine, "
+    "which runs in memory and holds it whatever CPU and RAM it is given. A source of this size "
+    "needs partitioned execution -- bounded shards manifested, executed and merged -- which is "
+    "separate, unbuilt work, and is not more memory."
+)
+
+
+__all__ = [
+    "AT_LEAST",
+    "AT_MOST",
+    "A_PLAN",
+    "A_RECIPE",
+    "A_WORKBENCH",
+    "GRAPH_EXECUTION",
+    "MEASURED",
+    "OVER",
+    "TABULAR_EXECUTION",
+    "UNDECIDED",
+    "WITHIN",
+    "WORKLOAD_INVALID",
+    "WORKLOAD_UNREADABLE",
+    "Ceiling",
+    "Refused",
+    "SourceFacts",
+    "Workload",
+    "WorkloadUnreadable",
+    "read_workload",
+    "unknowns_for",
+]