mostlyright-data 0.26.0__tar.gz → 0.26.1__tar.gz

{mostlyright_data-0.26.0 → mostlyright_data-0.26.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mostlyright-data
-Version: 0.26.0
+Version: 0.26.1
 Summary: Mostly Right hosted CLI for reviewed datasets
 Project-URL: Homepage, https://mostlyright.md/
 Project-URL: Documentation, https://mostlyright.md/docs/guides/cli/

{mostlyright_data-0.26.0 → mostlyright_data-0.26.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mostlyright-data"
-version = "0.26.0"
+version = "0.26.1"
 description = "Mostly Right hosted CLI for reviewed datasets"
 readme = "README.md"
 requires-python = ">=3.11"

{mostlyright_data-0.26.0 → mostlyright_data-0.26.1}/skills/mr-data-build/references/readers.md

@@ -90,6 +90,7 @@ decoder. Zero matches and multiple matches both refuse.
 | `delimited_text@2.0.0` | Explicit delimiter, encoding, quoting, header presence, skip_rows, skip_blank_rows, and indexed columns with output names and optional expected_header; unselected_columns requires empty cells unless explicitly set to allow. |
 | `json.tabular@1.0.0` | `columns` with output names and RFC 6901 pointers; optional `document_format` (`json` or `ndjson`), `records_pointer`, and sequential `expand` pointers. Columns are required unless they state `required: false`. |
 | `json.tabular@1.1.0` | Same; also accepts a response served as `text/plain` (common for public agency JSON and NDJSON). |
+| `json.tabular@1.2.0` | Same as 1.1.0. An `expand` entry may be `{"pointer": "/x", "required": false}`. A record whose array there is null, missing or too short adds no rows instead of refusing the document. A present member that is not an array still refuses. |
 | `xml.tabular@1.0.0` | Absolute record element path, declared namespace prefixes, and relative field selectors ending in `text()` or `@attribute`; fields are required unless `required: false`. |
 | `html.tabular@1.0.0` | Absolute record element path and relative text/attribute field selectors; fields are required unless `required: false`. |
 | `html.web_extract@1.0.0` | `mode: dom` with closed CSS-like record/field selectors, or `mode: json` with an inert JSON script selector and RFC 6901 field pointers. |

{mostlyright_data-0.26.0 → mostlyright_data-0.26.1}/skills/mr-data-build/references/sources.md

@@ -190,6 +190,8 @@ Write the request by these rules.
 - `method` is `GET` or `POST`. A `POST` states `body_template`, and a `GET` states none.
 - Each `{name}` placeholder takes the upstream row's cell in the column of that name.
 - Every placeholder is a declared parameter, and every parameter is used.
+- A literal brace is written twice, `{{` or `}}`. Any other brace is refused. A SPARQL query can
+  ride in a query value this way.
 - Every parameter and every entry of `carry_columns` is a column of the upstream source.
 - A header value is a literal. The worker writes `Content-Type` itself.
 - Every literal the request sends is printable ASCII, with no trailing newline.
@@ -251,7 +253,9 @@ and a redirect. A keyless 401 or 403 is one of them. Tell the user each run fail
 and one sentence.
 
 When the upstream source has a request window, the collection inherits it. Name the window's
-`merge.partition.column` in `carry_columns`, or registration refuses the recipe. A refresh then
+`merge.partition.column` in `carry_columns`, or registration refuses the recipe. A per-row
+request collection over another per-row request collection inherits the same window when every
+hop carries that column. `mr-data recipe explain` names each hop that drops it. A refresh then
 sends requests only for rows in the refreshed partitions. A request the predecessor answered
 successfully is never sent again, whatever the row's carried cells or partition now are. A full
 build leaves out an upstream row whose partition cell does not read. Read this source alone in the

{mostlyright_data-0.26.0 → mostlyright_data-0.26.1}/src/mostlyright/data_harness/thin/recipe.py

@@ -478,55 +478,45 @@ def _named_source(source: Mapping[str, Any], index: int) -> str:
 def _inherits_a_direct_window(source: Mapping[str, Any], sources: Sequence[Any]) -> bool:
     """Whether this collection refreshes with a direct request window of the same recipe.
 
-    Studio's ``_inherits_a_direct_window``, read the way it reads it. The discovery is
-    ``source_rows`` and states ``request``, so the collection sends one request per upstream row.
-    The upstream it names is a direct window of this plan: not a stream, not ``closed``, not a
-    collection, with a ``window`` that carries ``request`` and no ``snapshot`` and replaces whole
-    partitions. And ``request.carry_columns`` names that window's ``merge.partition.column``, so
-    every member is keyed to its upstream row's partition and a refresh re-derives exactly the
-    members of the partitions the window replaces. Every other collection restores a ledger that
-    cannot join that proof.
+    Studio's ``_inherits_a_direct_window``, read the way it reads it. The collection is a per-row
+    request collection, a ``source_rows`` discovery that states ``request``, and its chain of
+    such collections up ``discovery.source`` (:func:`recipe_explain.inherited_window_chain`)
+    ends at a direct window of this plan: not a stream, not ``closed``, not a collection, with a
+    ``window`` that carries ``request`` and no ``snapshot`` and replaces whole partitions. And
+    every hop, this collection included, names that window's ``merge.partition.column`` in
+    ``request.carry_columns``, so every member is keyed to its upstream row's partition and a
+    refresh re-derives exactly the members of the partitions the window replaces. A chain that
+    loops, or that passes a ``url_column`` collection, inherits nothing.
 
     A name two sources share is nobody's upstream here. That document is malformed, and the
     refusal for it is the registration route's.
     """
 
-    collection = source.get("collection")
-    discovery = collection.get("discovery") if isinstance(collection, Mapping) else None
-    if not isinstance(discovery, Mapping) or discovery.get("kind") != "source_rows":
-        return False
-    request = discovery.get("request")
-    upstream_name = discovery.get("source")
-    if not isinstance(request, Mapping) or not isinstance(upstream_name, str):
-        return False
-    upstreams = [
-        candidate
+    counted: dict[str, int] = {}
+    for candidate in sources:
+        if isinstance(candidate, Mapping) and isinstance(candidate.get("name"), str):
+            counted[candidate["name"]] = counted.get(candidate["name"], 0) + 1
+    by_name = {
+        candidate["name"]: candidate
         for candidate in sources
-        if isinstance(candidate, Mapping) and candidate.get("name") == upstream_name
-    ]
-    if len(upstreams) != 1:
+        if isinstance(candidate, Mapping)
+        and isinstance(candidate.get("name"), str)
+        and counted[candidate["name"]] == 1
+    }
+    chain = recipe_explain.inherited_window_chain(source, by_name)
+    if chain is None:
         return False
-    (upstream,) = upstreams
+    root, hops = chain
+    upstream = by_name[root]
     if (
         upstream.get("source_class") == "stream"
         or upstream.get("closed") is True
         or isinstance(upstream.get("collection"), Mapping)
     ):
         return False
-    window = upstream.get("window")
-    merge = window.get("merge") if isinstance(window, Mapping) else None
-    partition = merge.get("partition") if isinstance(merge, Mapping) else None
-    column = partition.get("column") if isinstance(partition, Mapping) else None
-    carried = request.get("carry_columns")
-    return (
-        isinstance(window, Mapping)
-        and "snapshot" not in window
-        and isinstance(window.get("request"), Mapping)
-        and isinstance(merge, Mapping)
-        and merge.get("materialization") == "partition_replace"
-        and isinstance(column, str)
-        and isinstance(carried, list)
-        and column in carried
+    column = recipe_explain.window_partition_column(upstream)
+    return column is not None and all(
+        column in recipe_explain.row_request_carried(hop) for hop in hops
     )
 
 
@@ -730,8 +720,9 @@ def unrefreshable_plan_finding(
             "collection restores its ledger from the raw snapshot and cannot join the "
             "replacement-partition proof a refresh writes, so it refuses the whole table. The one "
             "collection a refresh admits sends one request per row of a direct request window "
-            "in this recipe, through a `source_rows` discovery that states `request`, and names "
-            "that window's `merge.partition.column` in `request.carry_columns`. "
+            "in this recipe, directly or through other such collections, each through a "
+            "`source_rows` discovery that states `request` and names that window's "
+            "`merge.partition.column` in `request.carry_columns`. "
             f"{UNREFRESHABLE_PLAN_TAIL}"
         )
     if "window" in held:

{mostlyright_data-0.26.0 → mostlyright_data-0.26.1}/src/mostlyright/data_harness/thin/recipe_explain.py

@@ -493,34 +493,57 @@ def declared_rebuild(document: Mapping[str, Any]) -> Mapping[str, Any] | None:
     return declaration
 
 
-def _inherits_a_direct_window(
-    source: Mapping[str, Any],
-    sources: Mapping[str, Mapping[str, Any]],
-    direct_windows: Sequence[str],
-) -> bool:
-    """Whether this per-row request collection inherits a request window of this plan.
-
-    Studio's rule: the upstream is a direct window of the same plan, and that window states
-    ``request`` (not only ``snapshot``), replaces whole partitions, and has its partition column
-    carried by the request. A snapshot window, a closed source and another collection are none.
+def inherited_window_chain(
+    source: Mapping[str, Any], sources: Mapping[str, Mapping[str, Any]]
+) -> tuple[str, tuple[Mapping[str, Any], ...]] | None:
+    """The source a per-row request chain reads its rows from, and every hop on the way.
+
+    Studio's rule: walk ``discovery.source`` through per-row request collections, starting at
+    ``source``, until a source that is not one. Answers that source's name and every hop,
+    ``source`` first, or ``None`` when ``source`` is no per-row request collection, a hop names an
+    undeclared source, or the chain loops back on itself.
     """
 
-    collection = source.get("collection")
-    discovery = collection.get("discovery") if isinstance(collection, Mapping) else None
-    request = discovery.get("request") if isinstance(discovery, Mapping) else None
-    if (
-        not isinstance(discovery, Mapping)
-        or discovery.get("kind") != "source_rows"
-        or not isinstance(request, Mapping)
-        or discovery.get("source") not in direct_windows
-    ):
-        return False
-    upstream = sources.get(str(discovery["source"]))
-    window = upstream.get("window") if isinstance(upstream, Mapping) else None
+    hops: list[Mapping[str, Any]] = []
+    visited: set[str] = set()
+    current = source
+    while True:
+        discovery = _row_discovery(current)
+        request = discovery.get("request") if discovery is not None else None
+        upstream = discovery.get("source") if discovery is not None else None
+        if not isinstance(request, Mapping) or not isinstance(upstream, str):
+            return None
+        hops.append(current)
+        following = sources.get(upstream)
+        if following is None or upstream in visited:
+            return None
+        visited.add(upstream)
+        following_discovery = _row_discovery(following)
+        if following_discovery is None or not isinstance(
+            following_discovery.get("request"), Mapping
+        ):
+            return upstream, tuple(hops)
+        current = following
+
+
+def row_request_carried(source: Mapping[str, Any]) -> tuple[str, ...]:
+    """The columns a per-row request collection carries, or none for any other source."""
+
+    discovery = _row_discovery(source)
+    request = discovery.get("request") if discovery is not None else None
+    carried = request.get("carry_columns") if isinstance(request, Mapping) else None
+    if not isinstance(carried, list):
+        return ()
+    return tuple(name for name in carried if isinstance(name, str))
+
+
+def window_partition_column(source: Mapping[str, Any]) -> str | None:
+    """The partition column of a request window that replaces whole partitions, else ``None``."""
+
+    window = source.get("window")
     merge = window.get("merge") if isinstance(window, Mapping) else None
     partition = merge.get("partition") if isinstance(merge, Mapping) else None
-    carried = request.get("carry_columns")
-    return (
+    if (
         isinstance(window, Mapping)
         and "snapshot" not in window
         and isinstance(window.get("request"), Mapping)
@@ -528,9 +551,30 @@ def _inherits_a_direct_window(
         and merge.get("materialization") == "partition_replace"
         and isinstance(partition, Mapping)
         and isinstance(partition.get("column"), str)
-        and isinstance(carried, list)
-        and partition["column"] in carried
-    )
+    ):
+        return str(partition["column"])
+    return None
+
+
+def _inherits_a_direct_window(
+    source: Mapping[str, Any],
+    sources: Mapping[str, Mapping[str, Any]],
+    direct_windows: Sequence[str],
+) -> bool:
+    """Whether this per-row request collection inherits a request window of this plan.
+
+    Studio's rule: the end of the per-row request chain (:func:`inherited_window_chain`) is a
+    direct window of the same plan that states ``request`` (not only ``snapshot``) and replaces
+    whole partitions, and every hop, this collection included, carries its partition column. A
+    snapshot window, a closed source, a ``url_column`` collection and a loop are none.
+    """
+
+    chain = inherited_window_chain(source, sources)
+    if chain is None or chain[0] not in direct_windows:
+        return False
+    root, hops = chain
+    column = window_partition_column(sources[root])
+    return column is not None and all(column in row_request_carried(hop) for hop in hops)
 
 
 def _layout_is_consistent(
@@ -573,9 +617,10 @@ def _not_inheriting(
     upstream = discovery.get("source") if discovery is not None else None
     if isinstance(upstream, str) and upstream in sources:
         detail = (
-            f"{name} derives its rows from {upstream}, which is not a request window "
-            f"replacing whole partitions whose partition column {name} carries, so a refresh "
-            f"cannot tell which of its members changed"
+            f"{name} derives its rows from {upstream}, which is neither a request window "
+            f"replacing whole partitions nor a per-row request collection inheriting one, with "
+            f"that window's partition column carried by {name} and every collection between, so "
+            f"a refresh cannot tell which of its members changed"
         )
     else:
         detail = (
@@ -927,37 +972,51 @@ def _rebuild(facts: _Facts) -> list[dict[str, Any]]:
 
 
 def _collection_suggestions(names: Sequence[str], facts: _Facts) -> list[dict[str, Any]]:
+    """What would let each collection inherit a request window of its own table.
+
+    A per-row request chain inherits the window its first non-collection upstream states, so the
+    window and the missing carries are judged at the end of the chain, not one hop up.
+    """
+
+    by_name = {name: facts.source(name) for name in facts.indices}
     suggestions: list[dict[str, Any]] = []
     for name in names:
-        discovery = _row_discovery(facts.source(name)) or {}
+        source = facts.source(name)
+        discovery = _row_discovery(source) or {}
         upstream_name = discovery.get("source")
+        chain = inherited_window_chain(source, by_name)
+        if chain is not None:
+            upstream_name = chain[0]
         upstream = facts.source(upstream_name) if isinstance(upstream_name, str) else {}
-        window = upstream.get("window")
-        merge = window.get("merge") if isinstance(window, Mapping) else None
-        partition = merge.get("partition") if isinstance(merge, Mapping) else None
-        column = partition.get("column") if isinstance(partition, Mapping) else None
-        if (
-            isinstance(discovery.get("request"), Mapping)
-            and isinstance(window, Mapping)
-            and isinstance(window.get("request"), Mapping)
-            and isinstance(merge, Mapping)
-            and merge.get("materialization") == "partition_replace"
-            and isinstance(column, str)
-        ):
-            suggestions.append(
-                _suggestion(
-                    "carry_partition_column",
-                    f"add {column} to {name}'s request.carry_columns so its rows key to the "
-                    f"partitions of {upstream_name}",
-                    facts.pointer(name, "/collection/discovery/request/carry_columns"),
+        column = window_partition_column(upstream)
+        missing_carry = False
+        if chain is not None and column is not None:
+            for hop in chain[1]:
+                if column in row_request_carried(hop):
+                    continue
+                missing_carry = True
+                hop_name = str(hop.get("name"))
+                suggestions.append(
+                    _suggestion(
+                        "carry_partition_column",
+                        f"add {column} to {hop_name}'s request.carry_columns so its rows key to "
+                        f"the partitions of {upstream_name}",
+                        facts.pointer(hop_name, "/collection/discovery/request/carry_columns"),
+                    )
                 )
+        if not missing_carry and isinstance(upstream_name, str) and upstream:
+            detail = (
+                f"let {upstream_name} refresh as a request window of its own, neither "
+                f"closed nor a page collection, so {name} re-derives only the rows of the "
+                "refreshed partitions"
+                if chain is not None and column is not None
+                else f"make {upstream_name} a request window that replaces whole partitions, "
+                f"so {name} re-derives only the rows of the refreshed partitions"
             )
-        elif isinstance(upstream_name, str) and upstream:
             suggestions.append(
                 _suggestion(
                     "make_upstream_a_request_window",
-                    f"make {upstream_name} a request window that replaces whole partitions, "
-                    f"so {name} re-derives only the rows of the refreshed partitions",
+                    detail,
                     facts.pointer(upstream_name, "/window"),
                     date_pair_parameters(upstream),
                 )
@@ -1492,11 +1551,14 @@ __all__ = [
     "explain_exit_code",
     "explain_table_at_studio",
     "gated_grammar",
+    "inherited_window_chain",
     "mirror_findings",
     "registration_gate",
     "route_unavailable",
+    "row_request_carried",
     "row_request_parameter_names",
     "served_capabilities",
     "snapshot_sources",
     "source_action",
+    "window_partition_column",
 ]

{mostlyright_data-0.26.0 → mostlyright_data-0.26.1}/src/mostlyright/data_harness/thin/recipe_lint.py

@@ -158,6 +158,12 @@ COHERENCE_CREDENTIAL_MODES: frozenset[str] = frozenset({"none", "opaque_referenc
 #: ``request`` instead of ``url_column``, on a source declared through this adapter.
 ROW_REQUEST_ADAPTER_ID = "public.https.collection@2.0.0"
 
+#: One per-row template token, read left to right: an escaped brace, a ``{name}`` placeholder, a
+#: lone brace, or a run of text with no brace. The registration route and the worker read every
+#: per-row template with this same expression, and no per-row template pattern of
+#: ``recipe.schema.json`` admits a string in which a lone brace is read.
+ROW_TEMPLATE_TOKEN = re.compile(r"\{\{|\}\}|\{([A-Za-z][A-Za-z0-9_]{0,63})\}|[{}]|[^{}]+")
+
 #: ``recipe.schema.json#/$defs/collection_revisit``'s bounds on an ``interval``, which the
 #: collection lane's ``MIN_REVISIT_SECONDS`` and ``MAX_REVISIT_SECONDS`` also hold: an hour to a
 #: year. A per-row request admits ``interval`` since Studio 4.14.0 (``CONTRACT.md`` §1.1).
@@ -226,6 +232,7 @@ def lint_document(document: Mapping[str, Any]) -> list[Finding]:
     findings.extend(_sources(document.get("sources")))
     findings.extend(_row_filters(document.get("sources")))
     findings.extend(_row_request_credentials(document.get("sources")))
+    findings.extend(_row_request_templates(document.get("sources")))
     findings.extend(_units(document.get("units")))
     findings.extend(_coherence(document, declared=declared))
     findings.extend(_checks(document.get("checks"), declared=declared, grain=_grain(document)))
@@ -758,6 +765,68 @@ def _row_request_credentials(sources: Any) -> list[Finding]:
     return findings
 
 
+def row_template_is_well_formed(text: str) -> bool:
+    """Whether every brace in one per-row template is doubled or part of a ``{name}`` placeholder.
+
+    ``{{`` is a literal ``{`` and ``}}`` a literal ``}``, read left to right, so ``{{{id}}}`` is a
+    literal brace, the ``id`` placeholder and a literal brace.
+    """
+
+    return all(match.group(0) not in ("{", "}") for match in ROW_TEMPLATE_TOKEN.finditer(text))
+
+
+def _row_request_templates(sources: Any) -> list[Finding]:
+    """The brace grammar of every per-row request template, held to the schema's.
+
+    A per-row request renders ``{name}`` from the upstream row in its path, in each query value
+    and in each string of its body. A literal brace is written twice, and any other brace is the
+    schema's refusal, so each finding carries :data:`DOCUMENT_INVALID`. Header values, query names
+    and body keys take no placeholder and are not read here.
+    """
+
+    if not isinstance(sources, list):
+        return []
+    findings: list[Finding] = []
+
+    def check(pointer: str, text: Any) -> None:
+        if isinstance(text, str) and not row_template_is_well_formed(text):
+            findings.append(
+                Finding(
+                    DOCUMENT_INVALID,
+                    pointer,
+                    "holds a brace that is neither doubled nor part of a placeholder; a "
+                    "placeholder is a parameter name in braces, and a literal brace is written "
+                    "twice, {{ or }}",
+                )
+            )
+
+    def walk(pointer: str, value: Any) -> None:
+        if isinstance(value, str):
+            check(pointer, value)
+        elif isinstance(value, list):
+            for position, item in enumerate(value):
+                walk(f"{pointer}/{position}", item)
+        elif isinstance(value, Mapping):
+            for key, item in value.items():
+                walk(f"{pointer}/{_escape(str(key))}", item)
+
+    for index, source in enumerate(sources):
+        if not isinstance(source, Mapping):
+            continue
+        _stated, request = _row_request(source)
+        if request is None:
+            continue
+        at = f"/sources/{index}/collection/discovery/request"
+        check(f"{at}/path_template", request.get("path_template"))
+        static_query = request.get("static_query")
+        for position, pair in enumerate(static_query if isinstance(static_query, list) else ()):
+            if isinstance(pair, Mapping):
+                check(f"{at}/static_query/{position}/value", pair.get("value"))
+        if "body_template" in request:
+            walk(f"{at}/body_template", request["body_template"])
+    return findings
+
+
 def _units(units: Any) -> list[Finding]:
     """Each declared unit's own two members. Whether the column exists is :func:`_coherence`'s."""
 
@@ -1430,8 +1499,10 @@ __all__ = [
     "MIN_BINS",
     "MIN_REVISIT_SECONDS",
     "PRESENTATION_BUCKETS",
+    "ROW_TEMPLATE_TOKEN",
     "Finding",
     "lint_document",
     "lint_refusal",
+    "row_template_is_well_formed",
     "snapshot_horizon_notes",
 ]

{mostlyright_data-0.26.0 → mostlyright_data-0.26.1}/src/mostlyright/data_harness/thin/v4.py

@@ -138,7 +138,7 @@ from mostlyright.data_harness.thin.transport import ThinLaneError
 #: ``JOB_INVALID`` outright, which in production was every refresh a collection epoch was offered
 #: for rather than only the collection runs. This package must not reach a Studio older than the
 #: commit it pins; ``docs/V4-WORKER-PROTOCOL.md`` states it beside the layout's own ordering rule.
-PINNED_V4_OPENAPI_SOURCE_SHA256 = "f14b52c537caac9cb065c5aa6399fcb7077e2eaafe6b42f981189059f1a260b2"
+PINNED_V4_OPENAPI_SOURCE_SHA256 = "afe1b4d7af60c5296f44e00a6bd84dc11a852fcbf4228bbc63a4c2d2b3b00829"
 PINNED_V4_CONTRACT_VERSION = "4.14.0"
 
 #: The ``schema_version`` every command body that predates 4.14.0 stamps: ``dataset activity``,