mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/ux/plain_file.py

@@ -0,0 +1,178 @@
+"""One rule for reading a file somebody named on the command line, in one place.
+
+Every command that takes a path from a person has the same three questions to answer before it
+reads a byte: is there anything there, is what is there a plain file, and is it small enough. The
+obvious way to answer them -- ``path.stat()``, then ``path.is_file()``, then ``path.read_bytes()``
+-- gets all three wrong at once. ``st_size`` is 0 for a FIFO, a pipe and a character device, so a
+cap taken off the path governs nothing; the stat follows links, so a link to ``/dev/zero`` reads
+the same way; and the file the size was measured on is not the file the read then opens, so a
+declared bound is a bound on a moment that has passed.
+
+So the rule here is the open-then-fstat rule the sealed tree opens everything with: open once, ask
+the descriptor what it is, and read that same descriptor under a bound. ``O_NONBLOCK`` is part of
+the rule rather than a flourish -- opening a FIFO for reading blocks until somebody opens the other
+end, so without it the refusal below could never be reached and the command would hang instead of
+answering. On a plain file the flag does nothing at all.
+
+**Why the refusal is untyped here.** :class:`PlainFileRefusal` carries a reason and the path and no
+wording, because the wording belongs to the command the person ran: ``author`` says a Recipe is
+written in a file, ``peek`` says there is nothing there to look at, and neither sentence would read
+right on the other surface. What the callers share is the rule, and sharing the rule is the whole
+point: one implementation keeps every command on the same file-type and byte-boundary contract.
+"""
+
+from __future__ import annotations
+
+import os
+import stat
+from pathlib import Path
+
+# Why a named file was not read. Four reasons, so a caller answers four sentences rather than
+# branching on an errno.
+MISSING = "missing"
+NOT_PLAIN = "not a plain file"
+TOO_LARGE = "too large"
+
+# ``os.open`` on a link that leads nowhere raises ``FileNotFoundError``, but the named entry still
+# exists and needs link-specific repair. It is a distinct *reason* rather than a shade of missing
+# because absence and a dangling link name different fixes.
+DANGLING = "a link that leads nowhere"
+
+# The non-blocking open is the FIFO part of the rule, and it is a POSIX part: Windows has no
+# ``O_NONBLOCK`` and no FIFO a plain ``open`` could hang on, so there the flag is simply absent.
+# Looked up once here rather than at the call, so the thin client (ADR 0021) answers ``whoami``
+# on Windows instead of failing before it reads a byte -- the any-OS smoke matrix found this.
+_NONBLOCKING = getattr(os, "O_NONBLOCK", 0)
+
+# ⚠ THE OTHER HALF OF THAT SENTENCE, AND THE HALF THE MATRIX CANNOT SEE. ``O_BINARY`` exists only
+# on Windows, where ``os.open`` otherwise leaves the C runtime free to translate CRLF on the way
+# through -- so the bytes this rule returned would not be the bytes on disk. Every caller reads
+# something whose EXACT bytes are the point: a canonical-JSON credential that is parsed back, a
+# Recipe that is about to be digested. A silently rewritten byte is not a crash the smoke job
+# could catch; it is a digest that no longer matches, on one operating system only.
+_BINARY = getattr(os, "O_BINARY", 0)
+
+
+class PlainFileRefusal(Exception):
+    """A named file this rule will not read, with the reason and the path but no wording.
+
+    Every caller translates this into its own vocabulary and its own error class. Nothing catches
+    it at the command line, so a caller that forgets to translate fails loudly in its own tests
+    rather than reaching a person as an untyped traceback.
+
+    ``mode`` is the ``st_mode`` the descriptor reported, for :data:`NOT_PLAIN`; the ``st_mode`` the
+    one ``lstat`` reported, for :data:`DANGLING`; and ``None`` for the other two reasons. It is not
+    a reason of its own and it is not a case added to the rule: it is the fact this rule already
+    looked at to decide, handed back rather than thrown away, so a caller whose vocabulary
+    distinguishes a folder from a pipe can say which it found. A caller whose vocabulary does not
+    -- ``peek`` answers all four with one sentence -- ignores it.
+    """
+
+    def __init__(self, reason: str, path: Path, *, mode: int | None = None) -> None:
+        self.reason = reason
+        self.path = path
+        self.mode = mode
+        super().__init__(f"{path}: {reason}")
+
+
+def open_plain_file(path: Path | str) -> tuple[int, int]:
+    """A descriptor on a plain file and the size that descriptor reports.
+
+    The caller owns the descriptor and must close it. Only :data:`MISSING`, :data:`DANGLING` and
+    :data:`NOT_PLAIN` are answered here; every other ``OSError`` -- a permission refusal, a
+    component of the path that is not a folder -- is left to travel, because those are the command
+    line's own boundary to answer and inventing a sentence for them here would be a second answer
+    to a question that already has one.
+    """
+
+    target = Path(path)
+    try:
+        descriptor = os.open(target, os.O_RDONLY | _NONBLOCKING | _BINARY)
+    except FileNotFoundError:
+        # One `lstat` before the answer, because `ENOENT` is two findings and only one of them is
+        # "nothing is there". The open resolved the name and found nothing at the end of it; this
+        # asks about the name itself, without following it, and an entry that answers is an entry
+        # that exists. Only a look that also fails is `MISSING`.
+        try:
+            entry = os.lstat(target)
+        except (FileNotFoundError, NotADirectoryError):
+            raise PlainFileRefusal(MISSING, target) from None
+        except OSError as refused_look:
+            # The name resolved to nothing and the look itself was refused, so this rule has no
+            # finding of its own; the operating system's refusal is the answer, and the command
+            # line's boundary already has a sentence for it. `MISSING` here would be a guess.
+            raise refused_look from None
+        raise PlainFileRefusal(DANGLING, target, mode=entry.st_mode) from None
+    except OSError as open_error:
+        # Some kernels refuse to open a Unix-domain socket even with ``O_NONBLOCK``.  The
+        # descriptor-first rule therefore cannot reach ``fstat`` for that path kind.  A
+        # best-effort fallback classification is safe here because it is used only to refuse a
+        # non-regular target; it never licenses a file for reading.  If the path vanished, is a
+        # regular file, or cannot be inspected, preserve the original system refusal.
+        try:
+            refused = target.stat()
+        except OSError:
+            raise open_error from None
+        if not stat.S_ISREG(refused.st_mode):
+            raise PlainFileRefusal(NOT_PLAIN, target, mode=refused.st_mode) from None
+        raise
+    try:
+        opened = os.fstat(descriptor)
+    except BaseException:
+        os.close(descriptor)
+        raise
+    if not stat.S_ISREG(opened.st_mode):
+        os.close(descriptor)
+        raise PlainFileRefusal(NOT_PLAIN, target, mode=opened.st_mode)
+    return descriptor, opened.st_size
+
+
+def read_bounded(descriptor: int, *, max_bytes: int) -> bytes:
+    """Everything the descriptor holds, up to one byte past the cap and never further.
+
+    The cap governs the read as well as the size check above it. A file that grows between the
+    ``fstat`` and the read is the case the size check alone cannot see, and it is the case a
+    declared bound exists for.
+    """
+
+    chunks: list[bytes] = []
+    remaining = max_bytes + 1
+    while remaining > 0:
+        chunk = os.read(descriptor, min(remaining, 1 << 16))
+        if not chunk:
+            break
+        chunks.append(chunk)
+        remaining -= len(chunk)
+    return b"".join(chunks)
+
+
+def read_plain_file(path: Path | str, *, max_bytes: int) -> bytes:
+    """The whole rule in one call: open it, check it, read it under the bound.
+
+    Both refusals a size can produce are :data:`TOO_LARGE`: the one the descriptor reports before
+    the read, and the one the read itself finds when the file grew in between.
+    """
+
+    target = Path(path)
+    descriptor, size = open_plain_file(target)
+    try:
+        if size > max_bytes:
+            raise PlainFileRefusal(TOO_LARGE, target)
+        raw = read_bounded(descriptor, max_bytes=max_bytes)
+    finally:
+        os.close(descriptor)
+    if len(raw) > max_bytes:
+        raise PlainFileRefusal(TOO_LARGE, target)
+    return raw
+
+
+__all__ = [
+    "DANGLING",
+    "MISSING",
+    "NOT_PLAIN",
+    "TOO_LARGE",
+    "PlainFileRefusal",
+    "open_plain_file",
+    "read_bounded",
+    "read_plain_file",
+]

mostlyright/data_harness/ux/plan_check.py

@@ -0,0 +1,311 @@
+"""Try a plan by building it for real into a folder that is then thrown away.
+
+The command exercises the real build path without retaining an immutable output directory. It is
+the safe way to test a plan before choosing the path for a durable Build.
+
+**There is one build path, and this is it.** :func:`pipeline.build_candidate` is called unmodified,
+with the plan value exactly as it was handed in. Nothing here re-implements a lighter check, skips a
+gate, or relaxes a limit: a check that is easier to pass than execution is invalid
+you wanted to know. A recording-wrapper test asserts that call happens exactly once, so this
+paragraph is machine-checked rather than believed.
+
+**Nobody ran this build, so its fingerprint is nobody's.** A build's own fingerprint binds who ran
+it and under which attempt; those two values are synthesised here (:data:`CHECK_PRINCIPAL`,
+:data:`CHECK_ATTEMPT`) because a dry run has no real runner. The fingerprint that comes out is
+therefore not the one the real build will have, and reporting it would send somebody chasing a
+mismatch that was never a mismatch. :class:`PlanCheck` has no field for it -- absent, not empty --
+and every run says so in plain words. The *data* fingerprint is a different
+matter: it depends on the plan and the sources and nothing else, so it is reported, and a test
+measures it against a real build of the same plan rather than against a written-down constant.
+
+**The folder goes away, whatever happens.** The cleanup runs in a ``finally``, so an interrupt takes
+the folder with it. It also has to survive a recorded quirk of this repository: the folder a build
+seals is mode 0555 on purpose, so removing it means restoring owner write permission from the bottom
+up first. A cleanup that fails is reported as a fact rather than raised, because a folder left
+behind is worth knowing about but is not a reason to call a build that worked a failure.
+
+**Two things about the folder are checked before anything is built into it.** It must be private to
+its owner, and it must not sit inside the folder the sources are read from -- the second because a
+build refuses to read from the folder it writes to, and meeting that refusal through a path you
+never chose would be a puzzle rather than an answer. Both are refusals that name themselves, and
+both clean up the folder they refused.
+
+One more thing, recorded because it costs an afternoon otherwise: the system temporary folder is
+often reached through a link, and sealing refuses an output whose parent chain contains one. The
+folder is therefore resolved to its real path before anything is built into it.
+
+Nothing here prints. :meth:`PlanCheck.to_dict` returns the one payload both renderings are built
+from, so the same facts can be shown in a browser later without re-deriving them.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import stat
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from mostlyright.data_harness import pipeline
+from mostlyright.data_harness.local_contracts import (
+    GraphTablePlan,
+    TablePlan,
+    parse_fresh_plan_document,
+    refuse_inconsistent_units,
+    refuse_retired_operations,
+)
+from mostlyright.data_harness.pipeline import REFERENCE_BACKEND_NAME, BuildError
+
+# Calling the command's check builder rather than restating its wording keeps one
+# check reading the same in `mr-data plan-check` as it does in `mr-data review --local`. Two
+# spellings of one check are two things a person has to learn.
+from mostlyright.data_harness.ux.local_review import CHECKS_KEY, LocalCheck, _quality_checks
+
+# The one status this command reports. A plan that does not build raises instead, so this word is
+# never on a failure.
+PLAN_CHECK_STATUS = "plan_checked"
+
+# The synthetic identity every dry run builds under. Marked as a check rather than borrowed from
+# the caller: these two values are the reason the resulting build fingerprint is not the real
+# build's, and a value that came from the operator would blur that distinction.
+CHECK_PRINCIPAL = "plan-check"
+CHECK_ATTEMPT = "throwaway"
+
+# The name the folder is made under, so a leftover one can be recognised for what it is.
+TEMP_PREFIX = "mr-data-plan-check-"
+
+# What `mkdtemp` promises. Asserted rather than assumed: a whole build lands inside it.
+PRIVATE_MODE = 0o700
+
+# The two refusals this module makes on its own. They are spelled out again at the raise sites
+# below, because the gate that proves every code names its fix reads code literals out of the
+# source; a test asserts the raised identifier is the one named here, so the pair cannot drift.
+TEMP_OVERLAP_CODE = "PLAN_CHECK_TEMP_OVERLAP"
+TEMP_NOT_PRIVATE_CODE = "PLAN_CHECK_TEMP_NOT_PRIVATE"
+
+# Where the closing sentence sits in the payload. It is a key a person reads, and it renders last.
+UNPREDICTED_KEY = "what this does not tell you"
+
+# Why no build fingerprint is shown. Said in the same breath as the data fingerprint that is
+# shown, because the distinction between the two is the whole of what this command has to teach.
+WHY_NO_BUILD_FINGERPRINT = (
+    "The data fingerprint above is the one your real build will produce. The build's own "
+    "fingerprint is not shown, because it depends on who runs the build and under which attempt, "
+    "and a fingerprint that will not match is worse than none."
+)
+
+_DELETED = "This was a real build, into a folder that has now been deleted."
+_KEPT = (
+    "This was a real build, into the folder named above, which was left in place as you asked. "
+    "Run chmod -R u+w on it before removing it: the folder a build seals is read-only on purpose."
+)
+_NOT_REMOVED = (
+    "This was a real build, into the folder named above, which could not be removed afterwards. "
+    "Run chmod -R u+w on it before removing it: the folder a build seals is read-only on purpose."
+)
+
+
+@dataclass(frozen=True)
+class PlanCheck:
+    """What a real build of one plan produced, from a folder that no longer exists.
+
+    ``built`` is a property rather than a field because a plan that does not build raises: this
+    object exists only when the build happened, so there is no state in which it could be false.
+    There is deliberately no field for the build's own fingerprint because the synthetic runner
+    identity guarantees that it will differ from the durable Build's fingerprint.
+    """
+
+    row_count: int
+    columns: tuple[str, ...]
+    table_sha256: str
+    quality: tuple[LocalCheck, ...]
+    engine: str
+    temp_root: Path
+    cleanup_failed: bool = False
+    kept: bool = False
+
+    @property
+    def built(self) -> bool:
+        return True
+
+    @property
+    def removed(self) -> bool:
+        """Whether the throwaway folder is gone. False when it was kept, or would not go."""
+
+        return not self.kept and not self.cleanup_failed
+
+    def to_dict(self) -> dict[str, Any]:
+        """The one payload both renderings are built from."""
+
+        width = len(str(max(len(self.quality), 1)))
+        payload: dict[str, Any] = {
+            "status": PLAN_CHECK_STATUS,
+            "built": self.built,
+            "table_sha256": self.table_sha256,
+            "row_count": self.row_count,
+            "columns": list(self.columns),
+            "engine": self.engine,
+            "throwaway_folder_removed": self.removed,
+        }
+        if not self.removed:
+            # Named only when there is something to go and look at, and named before the sentence
+            # that tells you what to do with it.
+            payload["throwaway_folder"] = str(self.temp_root)
+        payload[CHECKS_KEY] = {
+            f"check {index:0{width}d}": check.line
+            for index, check in enumerate(self.quality, start=1)
+        }
+        payload[UNPREDICTED_KEY] = f"{self._folder_sentence()} {WHY_NO_BUILD_FINGERPRINT}"
+        return payload
+
+    def _folder_sentence(self) -> str:
+        if self.removed:
+            return _DELETED
+        return _KEPT if self.kept else _NOT_REMOVED
+
+
+def check_plan(
+    plan_value: Any,
+    *,
+    input_root: Path | str,
+    engine: str = REFERENCE_BACKEND_NAME,
+    keep: bool = False,
+) -> PlanCheck:
+    """Build ``plan_value`` for real into a throwaway folder and report what came out.
+
+    ``keep`` leaves the folder in place for someone looking at a failure; every other path removes
+    it, including the one an interrupt takes. A refusal from the build is raised exactly as the
+    real build raises it, so a plan that would fail fails here in the same words.
+    """
+
+    # The same gate `build` applies to the same bytes: a dry run of a plan nobody could build is
+    # a dry run that lies about what the real build would do.  The isinstance short-circuit mirrors
+    # `build_candidate`, which this function hands the value straight to: it accepts an
+    # already-parsed plan, and so must the exported reuse point in front of it.
+    if isinstance(plan_value, (TablePlan, GraphTablePlan)):
+        refuse_retired_operations(plan_value)
+        refuse_inconsistent_units(plan_value)
+    else:
+        parse_fresh_plan_document(plan_value)
+    root = Path(os.path.realpath(input_root))
+    # Resolved to its real path: the system temporary folder is often reached through a link, and
+    # sealing refuses an output whose parent chain contains one.
+    temp_root = Path(os.path.realpath(tempfile.mkdtemp(prefix=TEMP_PREFIX)))
+    try:
+        _require_private(temp_root)
+        _refuse_overlap(temp_root, root)
+    except BaseException:
+        # A folder nobody asked for and nothing was built into is never left behind, whatever the
+        # caller asked to keep.
+        _remove_tree(temp_root)
+        raise
+
+    cleanup_failed = False
+    try:
+        try:
+            # The real build, unmodified, on the plan value exactly as it was given. Called
+            # through the module so a test can record the call without a second build path
+            # existing for it to record.
+            result = pipeline.build_candidate(
+                plan_value,
+                input_root=root,
+                output_dir=temp_root / "check",
+                producer_principal=CHECK_PRINCIPAL,
+                producer_attempt=CHECK_ATTEMPT,
+                engine=engine,
+            )
+            # One reading of the sealed tree, for the facts a plan's author asked about: what the
+            # columns came out as, and which checks it passed.
+            inspection = pipeline.inspect_candidate(result.run_dir)
+        except BuildError as error:
+            raise _named_if_kept(error, temp_root, keep=keep) from error
+    finally:
+        if not keep:
+            cleanup_failed = not _remove_tree(temp_root)
+
+    return PlanCheck(
+        row_count=inspection.row_count,
+        columns=tuple(inspection.columns),
+        table_sha256=inspection.table_sha256,
+        quality=tuple(_quality_checks(inspection.quality)),
+        engine=engine,
+        temp_root=temp_root,
+        cleanup_failed=cleanup_failed,
+        kept=keep,
+    )
+
+
+def _named_if_kept(error: BuildError, temp_root: Path, *, keep: bool) -> BuildError:
+    """The build's own refusal, saying where the folder was left when one was left.
+
+    The kind, the identifier and the severity are the real build's; only the message gains the
+    place to go and look. A refusal that keeps evidence and does not say where it put it is a
+    refusal somebody has to guess at.
+    """
+
+    if not keep:
+        return error
+    return BuildError(
+        error.finding_id,
+        f"{error}; the folder it was building into was kept at {temp_root}",
+        severity=error.severity,
+    )
+
+
+def _require_private(temp_root: Path) -> None:
+    """Refuse a folder other people can read. A whole build is about to land inside it."""
+
+    mode = stat.S_IMODE(temp_root.stat().st_mode)
+    if mode != PRIVATE_MODE:
+        raise BuildError(
+            "PLAN_CHECK_TEMP_NOT_PRIVATE",
+            f"the temporary folder {temp_root} is not private to its owner",
+        )
+
+
+def _refuse_overlap(temp_root: Path, root: Path) -> None:
+    """Refuse a temporary folder inside the input folder, in words about the folder that moved.
+
+    A build refuses to read from the folder it writes to. That refusal is right, but meeting it
+    through a path the caller never chose is a puzzle, so the overlap is named here instead.
+    """
+
+    if temp_root == root or root in temp_root.parents or temp_root in root.parents:
+        raise BuildError(
+            "PLAN_CHECK_TEMP_OVERLAP",
+            f"the temporary folder {temp_root} sits inside the input folder {root}",
+        )
+
+
+def _remove_tree(temp_root: Path) -> bool:
+    """Remove the throwaway folder, restoring write permission first. True when it is gone.
+
+    A sealed build is mode 0555 on purpose, so a plain removal fails with a permission error part
+    way through and leaves the rest behind. Owner write is added from the bottom up -- the
+    directories are what the removal needs, and a directory has to be walked before it is fixed.
+    """
+
+    try:
+        for directory, _dirnames, _filenames in os.walk(temp_root, topdown=False):
+            os.chmod(directory, PRIVATE_MODE)
+        shutil.rmtree(temp_root)
+    except OSError:
+        return False
+    return True
+
+
+__all__ = [
+    "CHECK_ATTEMPT",
+    "CHECK_PRINCIPAL",
+    "PLAN_CHECK_STATUS",
+    "PRIVATE_MODE",
+    "TEMP_NOT_PRIVATE_CODE",
+    "TEMP_OVERLAP_CODE",
+    "TEMP_PREFIX",
+    "UNPREDICTED_KEY",
+    "WHY_NO_BUILD_FINGERPRINT",
+    "PlanCheck",
+    "check_plan",
+]