mostlyright-data 0.9.8__tar.gz → 0.9.9__tar.gz

{mostlyright_data-0.9.8 → mostlyright_data-0.9.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mostlyright-data
-Version: 0.9.8
+Version: 0.9.9
 Summary: Mostly Right CLI for reproducible, reviewed Parquet datasets
 Project-URL: Homepage, https://mostlyright.md/
 Project-URL: Documentation, https://mostlyright.md/docs/guides/cli/

{mostlyright_data-0.9.8 → mostlyright_data-0.9.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mostlyright-data"
-version = "0.9.8"
+version = "0.9.9"
 description = "Mostly Right CLI for reproducible, reviewed Parquet datasets"
 readme = "README.md"
 requires-python = ">=3.13,<3.14"

{mostlyright_data-0.9.8 → mostlyright_data-0.9.9}/src/mostlyright/data_harness/thin/download.py

@@ -70,6 +70,11 @@ class DownloadedArtifact:
     media_type: str
     size_bytes: int
     content_digest: str
+    #: False when the bytes were already on disk, carried the digest the signed session declared,
+    #: and were adopted rather than fetched again. Reported so that "it was already here" can
+    #: never be read as "it was fetched and checked just now" -- the same reason ``verify`` prints
+    #: ``checked_by`` and ``acquire`` prints ``reader_binding``.
+    transferred: bool = True
 
     def to_receipt(self) -> dict[str, Any]:
         return {
@@ -78,6 +83,7 @@ class DownloadedArtifact:
             "media_type": self.media_type,
             "size_bytes": self.size_bytes,
             "content_digest": self.content_digest,
+            "transferred": self.transferred,
         }
 
 
@@ -103,6 +109,48 @@ def _required_headers(session: Mapping[str, Any]) -> dict[str, str]:
     return headers
 
 
+def _adopted(
+    destination: Path,
+    *,
+    expected_digest: Any,
+    expected_preimage: ExpectedPreimage | None,
+) -> tuple[bool, str, int]:
+    """Whether the file already on disk IS this artifact, by the digest the session declared.
+
+    ⚠ A DIGEST SKIP, NOT AN OVERWRITE. `deploy-dataset` fetches one artifact per kind in a loop,
+    and every one was created ``O_EXCL``, so a transfer interrupted after the first of two left a
+    folder holding one complete, verified artifact and no way to finish the job except a fresh
+    folder that re-downloaded the first as well. On a released dataset of several gigabytes the
+    cost of a dropped connection was paid twice (#438).
+
+    The skip is decided by the SAME fact the download would be verified against, so adopting
+    cannot be weaker than fetching: the bytes on disk must hash to the digest the signed session
+    declares, in whichever of the two authorities this download uses. Anything present that does
+    not match stays ``THIN_DESTINATION_EXISTS``: that file is not this artifact, and replacing it
+    silently is exactly what that refusal exists to prevent.
+    """
+
+    size = destination.stat().st_size
+    if size > MAX_ARTIFACT_BYTES:
+        return False, "", size
+    raw = bytearray()
+    observed = hashlib.sha256()
+    with destination.open("rb") as handle:
+        while chunk := handle.read(DOWNLOAD_CHUNK_BYTES):
+            observed.update(chunk)
+            if expected_preimage is not None:
+                raw.extend(chunk)
+    digest = observed.hexdigest()
+    if expected_preimage is None:
+        return f"sha256:{digest}" == expected_digest, f"sha256:{digest}", size
+    # The bundle seals its own preimage, so the authority is the excised hash, not the raw one.
+    body = bytes(raw)
+    if body.count(expected_preimage.excised_field) != 1:
+        return False, "", size
+    preimage = hashlib.sha256(body.replace(expected_preimage.excised_field, b"", 1)).hexdigest()
+    return preimage == expected_preimage.digest, f"sha256:{digest}", size
+
+
 def download_signed_artifact(
     session: Mapping[str, Any],
     destination: Path,
@@ -153,6 +201,30 @@ def download_signed_artifact(
 
     selected = transport or UrlLibTransport()
     destination.parent.mkdir(parents=True, exist_ok=True)
+    # ⚠ A SYMLINK IS NEVER ADOPTED. The write path opens with ``O_NOFOLLOW`` precisely so a
+    # symlinked destination is refused rather than followed, and adopting through one would walk
+    # around that. Only a regular file in the place the artifact belongs can stand in for it; a
+    # symlink falls through to the open below and is refused there, as before.
+    #
+    # The remaining race is benign: a file swapped between this check and the read would have to
+    # hash to the sealed digest to be adopted, and a file that does is the artifact.
+    if destination.is_file() and not destination.is_symlink():
+        matched, observed_digest, size = _adopted(
+            destination, expected_digest=expected_digest, expected_preimage=expected_preimage
+        )
+        if matched:
+            media_type = session.get("media_type")
+            artifact_id = session.get("artifact_id")
+            return DownloadedArtifact(
+                path=destination,
+                artifact_id=artifact_id if isinstance(artifact_id, str) else "",
+                media_type=(
+                    media_type if isinstance(media_type, str) else "application/octet-stream"
+                ),
+                size_bytes=size,
+                content_digest=(expected_digest if expected_preimage is None else observed_digest),
+                transferred=False,
+            )
     flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
     if hasattr(os, "O_NOFOLLOW"):
         flags |= os.O_NOFOLLOW
@@ -165,7 +237,8 @@ def download_signed_artifact(
     except FileExistsError as error:
         raise ThinLaneError(
             "THIN_DESTINATION_EXISTS",
-            f"{destination} already exists; a hosted download never overwrites a local file",
+            f"{destination} already exists and does not carry this artifact's sealed digest, so "
+            "it is a different file; a hosted download never overwrites a local file",
         ) from error
 
     try:

{mostlyright_data-0.9.8 → mostlyright_data-0.9.9}/src/mostlyright/data_harness/thin/propose.py

@@ -2099,6 +2099,7 @@ def propose(
     recipe = bundle.recipe
     assert recipe is not None
     _check_recipe_sources(recipe, source_ids=source_ids, configurations=configurations)
+    _check_recipe_query_templates(recipe, configurations=configurations)
     research_run_id: str | None = None
     if research_session_id is not None:
         record = _lookup_session(
@@ -2359,6 +2360,149 @@ def _refuse_a_registered_source_the_bundle_no_longer_names(
         )
 
 
+#: The Reader families a thin install can canonicalize in process. The other five certified
+#: coordinates -- ``spreadsheet.xlsx`` (two), ``parquet.tabular``, ``weather.grib2`` (two) --
+#: import openpyxl, pyarrow or mostlyright-grib at module scope, which the thin profile does not
+#: install. ``propose`` therefore refuses such a pin by name rather than sealing one it cannot
+#: check; see :func:`_admitted_reader_pin`.
+_READER_CANONICALIZATION_UNAVAILABLE = (
+    "this Reader family needs a native wheel the hosted profile does not install, so propose "
+    "cannot admit its options here; author the Recipe from an install that has it "
+    '(pip install "mostlyright-data[local]") so the sealed pin is the admitted one'
+)
+
+
+def _admitted_reader_pin(pin: Any, *, name: str, path: str) -> Any:
+    """One Reader pin in the form the CRAWLER will recompute, or a refusal naming why not.
+
+    ⚠ WHY AUTHORING HAS TO DO THIS. A pin is digested, and the digest is over the ADMITTED options
+    -- ``family.validate_options`` with every family-defaulted key filled in. An author who writes
+    ``{"delimiter": ","}`` and omits ``encoding``, ``header``, ``expand`` or ``required`` seals a
+    document that digests differently from the one the crawler will reconstruct, and every crawl
+    then fails ``reader_identity`` with nothing naming the cause. That is what killed r13 and r14
+    (#411). Every gate accepted those Recipes -- propose, Studio admission, approval -- and only
+    the crawl refused, unnamed.
+
+    The principle the issue states is the one applied here: an agent authors against ``propose``,
+    so anything Studio or the crawler will refuse at run time must be refused HERE, by name,
+    before a person is asked to approve it.
+
+    ⚠ THE THIN PROFILE CAN DO THIS FOR MOST FAMILIES, AND SAYS SO WHEN IT CANNOT. Importing
+    ``readers.registry`` is stdlib-only, and the registry loads a family lazily, so the ten
+    delimited/JSON/archive coordinates -- ``delimited_text`` among them, which is the family the
+    failing Recipes used -- canonicalize on a bare interpreter. The five that need a native wheel
+    are refused rather than passed through unchecked.
+    """
+
+    if pin is None:
+        return None
+    if not isinstance(pin, dict):
+        raise _invalid(name, path, "must be null or a Reader pin object")
+    _keys(pin, name, path, ("family_id", "family_version", "decode_options"))
+    family_id = pin["family_id"]
+    family_version = pin["family_version"]
+    options = pin["decode_options"]
+    if not isinstance(family_id, str) or not isinstance(family_version, str):
+        raise _invalid(name, path, "family_id and family_version must be strings")
+    if not isinstance(options, dict):
+        raise _invalid(name, f"{path}.decode_options", "must be an object")
+
+    # Imported here rather than at module scope: `propose` is reached on a bare interpreter, and
+    # the registry is only needed when a Recipe actually pins a Reader.
+    from mostlyright.data_harness.readers.registry import TOOLBOX
+
+    try:
+        family = TOOLBOX.resolve(family_id, family_version)
+    except ImportError as error:  # the family's native wheel is absent on this install
+        raise ProposeError(
+            "THIN_PROPOSE_READER_PIN",
+            f"{name} {path} pins {family_id}@{family_version}: "
+            f"{_READER_CANONICALIZATION_UNAVAILABLE} ({error})",
+        ) from error
+    except Exception as error:  # unknown family, unknown version, or a dependency refusal
+        raise ProposeError(
+            "THIN_PROPOSE_READER_PIN",
+            f"{name} {path} pins {family_id}@{family_version}, which is not a certified Reader "
+            f"coordinate this Harness can admit: {error}",
+        ) from error
+    try:
+        admitted = dict(family.validate_options(options))
+    except Exception as error:
+        raise ProposeError(
+            "THIN_PROPOSE_READER_PIN",
+            f"{name} {path} pins Reader options {family_id}@{family_version} refuses: {error}",
+        ) from error
+    return {
+        "family_id": family_id,
+        "family_version": family_version,
+        "decode_options": admitted,
+    }
+
+
+def _reader_pin_of(query: Any) -> Any:
+    return query.get("reader_pin") if isinstance(query, dict) else None
+
+
+def _check_recipe_query_templates(
+    recipe: Mapping[str, Any],
+    *,
+    configurations: Mapping[str, Mapping[str, Any]],
+) -> None:
+    """Every recipe query is the query its registered connector holds, in the admitted form.
+
+    ⚠ TWO REFUSALS, BECAUSE THEY ARE TWO MISTAKES. Studio binds ``query_template`` byte-for-byte
+    to the registered connector configuration, so a Recipe whose query has drifted from its source
+    is refused at run time with nothing naming the field. And a raw (non-admitted) Reader pin
+    digests differently from the one the crawler rebuilds, failing ``reader_identity`` on every
+    crawl. Both were accepted by every authoring gate before this (#411); both are named here,
+    with the exact value to write.
+
+    Sources whose adapter carries no connector query -- ``user.file`` snapshots -- are skipped:
+    there is nothing registered to compare against.
+    """
+
+    for source in recipe["sources"]:
+        source_id = source["source_id"]
+        template = source.get("query_template")
+        registered = configurations.get(source_id, {}).get("query")
+        if template is None or registered is None:
+            continue
+        if not isinstance(template, dict):
+            raise _invalid("recipe.json", f"sources[{source_id}].query_template", "must be object")
+
+        admitted_pin = _admitted_reader_pin(
+            _reader_pin_of(template),
+            name="recipe.json",
+            path=f"sources[{source_id}].query_template.reader_pin",
+        )
+        if admitted_pin != _reader_pin_of(template):
+            raise ProposeError(
+                "THIN_PROPOSE_READER_PIN",
+                f"recipe.json source {source_id} pins Reader options that are not the admitted "
+                "form, so its sealed digest is not the one every crawl will recompute and "
+                "reader_identity would fail on each run; write the pin exactly as: "
+                f"{json.dumps(admitted_pin, sort_keys=True)}",
+            )
+
+        differing = sorted(
+            key
+            for key in set(template) | set(registered)
+            if canonical_json_bytes(template.get(key)) != canonical_json_bytes(registered.get(key))
+        )
+        if differing:
+            detail = "; ".join(
+                f"{key}: recipe.json has {json.dumps(template.get(key), sort_keys=True)}, "
+                f"the registered connector has {json.dumps(registered.get(key), sort_keys=True)}"
+                for key in differing
+            )
+            raise ProposeError(
+                "THIN_PROPOSE_SOURCE_QUERY",
+                f"recipe.json source {source_id} carries a query its registered connector "
+                f"configuration does not hold, and Studio binds the two byte-for-byte, so this "
+                f"Recipe would be refused when it ran -- {detail}",
+            )
+
+
 def _check_recipe_sources(
     recipe: Mapping[str, Any],
     *,

{mostlyright_data-0.9.8 → mostlyright_data-0.9.9}/src/mostlyright/data_harness/ux/remediation.py

@@ -5,7 +5,7 @@ for the commonest failure of all -- pointing a command at a folder that is not t
 engineering wording reads like a security incident. This module turns a code into the two or three
 plain sentences that say what happened and what to do next.
 
-There are two layers, because 1763 typed codes across 209 prefixes can reach a person at a
+There are two layers, because 1765 typed codes across 209 prefixes can reach a person at a
 workbench, and a hand-written sentence for each one would be neither affordable nor honest:
 
 * **Families.** Every code prefix is assigned to a family. A family is one paragraph: what this