mostlyright-data 0.30.3__tar.gz → 0.30.5__tar.gz

{mostlyright_data-0.30.3 → mostlyright_data-0.30.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mostlyright-data
-Version: 0.30.3
+Version: 0.30.5
 Summary: Mostly Right hosted CLI for reviewed datasets
 Project-URL: Homepage, https://mostlyright.md/
 Project-URL: Documentation, https://mostlyright.md/docs/guides/cli/

{mostlyright_data-0.30.3 → mostlyright_data-0.30.5}/skills/mr-data-build/SKILL.md

@@ -44,6 +44,7 @@ session finds it. Then choose the source shape that fits.
 | One request per row of another source | a per-row `request`, with `batch` when a `GET` takes a list of ids | the gap | [per-row](references/patterns.md#one-request-per-row-many-ids-per-request) |
 | Bulk dump beside a windowed source | `reference` | `recheck_seconds` | [reference](references/patterns.md#bulk-dump-beside-a-windowed-source) |
 | Live `wss://` feed | a recorded stream | see [Streams](references/recording-a-stream-venue.md) | none |
+| A join of tables this workspace already built, rebuilt when one promotes | `table.version@1.0.0` sources with `refresh: upstream_promoted` and `rebuild` (Studio 4.38) | one shared `data_classification`, `debounce_seconds` | [table sources](references/sources.md#reading-another-table-of-this-workspace) |
 
 [Patterns](references/patterns.md) has one complete recipe per row, with the pace set.
 [Sources](references/sources.md#pace-and-width) has the pace rules and the limits of parallel

{mostlyright_data-0.30.3 → mostlyright_data-0.30.5}/skills/mr-data-build/references/sources.md

@@ -383,3 +383,30 @@ with `epoch_millis_value` on the raw millisecond time column, and size the reque
 for all slices and pages. Do not substitute a whole-day query:
 1,440 one-minute bars exceed Binance's 1,000-row response cap. Existing credential-free
 `public.https` file fetches and `public.https.collection@2.0.0` keep their own contracts.
+
+### Reading another table of this workspace
+
+Needs Studio 4.38. When the user wants a table that joins or reshapes tables this workspace has
+already built, do not fetch the upstream data again. Read each upstream as a source:
+
+- `source_class: mostlyright_table`, and a `table` locator with `{table_id, version:
+  latest_promoted}`.
+- Adapter `table.version@1.0.0`, with `credential_mode: none` and the Reader pin
+  `parquet.tabular` `1.0.0`, decode options `{}`.
+- `limits.max_source_bytes` above the upstream's real bytes. Take them from `mr-data table show`;
+  Studio refuses a run whose pinned upstream version is larger.
+
+To rebuild the join whenever an upstream promotes, declare `rebuild` and add `refresh: {trigger:
+upstream_promoted, debounce_seconds: N}`. Pick N so upstreams that promote together start one
+rebuild; 900 is a sound default.
+
+Three rules cost a round trip when they are missed:
+
+- Every source states the same `data_classification`. Use the strictest upstream's class.
+- The upstreams must be in this workspace, promoted, not archived, and not this table. No edge may
+  close a cycle.
+- The trigger runs only after the user chose updates with `mr-data promote --cadence`. That
+  cadence stays the fallback schedule.
+
+The full member list is in `docs/RECIPE-DOCUMENT.md`, "Needs Studio 4.38".
+

{mostlyright_data-0.30.3 → mostlyright_data-0.30.5}/src/mostlyright/data_harness/thin/recipe_lint.py

@@ -231,6 +231,8 @@ def lint_document(document: Mapping[str, Any]) -> list[Finding]:
     findings.extend(_columns(document.get("table")))
     findings.extend(_identifiers(document))
     findings.extend(_sources(document.get("sources")))
+    findings.extend(_table_sources(document.get("sources")))
+    findings.extend(_refresh_trigger(document))
     findings.extend(_row_filters(document.get("sources")))
     findings.extend(_row_request_credentials(document.get("sources")))
     findings.extend(_row_request_templates(document.get("sources")))
@@ -551,6 +553,272 @@ def _sources(sources: Any) -> list[Finding]:
     return findings
 
 
+#: ``recipe.schema.json``'s Mostly Right table source (Studio 4.38.0, ADR 0100): the one adapter
+#: that reads another table of this workspace, its source class and its locator kind.
+TABLE_SOURCE_ADAPTER_ID = "table.version@1.0.0"
+TABLE_SOURCE_CLASS = "mostlyright_table"
+TABLE_LOCATOR_KIND = "table"
+
+#: ``#/$defs/table_source_locator/properties/version``: the one selector v1 admits.
+TABLE_SOURCE_VERSIONS: frozenset[str] = frozenset({"latest_promoted"})
+
+#: Members a table source never carries: it is read whole, presents no credential and names no
+#: origin, so the schema refuses each of these beside it.
+TABLE_SOURCE_FORBIDDEN: tuple[str, ...] = (
+    "credential",
+    "window",
+    "collection",
+    "closed",
+    "reference",
+    "document_archive",
+)
+TABLE_CONNECTOR_FORBIDDEN: tuple[str, ...] = (
+    "origin",
+    "generic_api",
+    "stream_connector_document_digest",
+)
+
+#: ``#/$defs/table_refresh_trigger``.
+REFRESH_TRIGGERS: frozenset[str] = frozenset({"upstream_promoted"})
+MAX_REFRESH_DEBOUNCE_SECONDS = 86_400
+
+_UUID = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")
+
+
+def is_table_source(source: Any) -> bool:
+    """Whether a source is a Mostly Right table source by any of the three marks the schema reads.
+
+    The schema applies the table-source rules when the adapter, the locator kind OR the source
+    class says so, so a source that states only one of them is held to all of them here too.
+    """
+
+    if not isinstance(source, Mapping):
+        return False
+    connector = source.get("connector")
+    locator = source.get("locator")
+    return (
+        (isinstance(connector, Mapping) and connector.get("adapter_id") == TABLE_SOURCE_ADAPTER_ID)
+        or (isinstance(locator, Mapping) and locator.get("kind") == TABLE_LOCATOR_KIND)
+        or source.get("source_class") == TABLE_SOURCE_CLASS
+    )
+
+
+def _table_sources(sources: Any) -> list[Finding]:
+    """A Mostly Right table source's shape, which the schema refuses at the wire (Studio 4.38.0).
+
+    It reads the latest promoted version of another table in this workspace: source class
+    ``mostlyright_table``, a ``table`` locator naming the upstream ``table_id`` and
+    ``version: latest_promoted``, adapter ``table.version@1.0.0`` with ``credential_mode: none``
+    and the Reader pin that decodes the upstream's Parquet parts. Which table it names, whether that
+    table is in this workspace, promoted, archived, the table itself or closes a cycle, is Studio's
+    to judge at registration; nothing here can see the workspace.
+    """
+
+    findings: list[Finding] = []
+    for index, source in enumerate(sources if isinstance(sources, list) else ()):
+        if not is_table_source(source):
+            continue
+        pointer = f"/sources/{index}"
+        if source.get("source_class") != TABLE_SOURCE_CLASS:
+            findings.append(
+                Finding(
+                    DOCUMENT_INVALID,
+                    f"{pointer}/source_class",
+                    f"must be {TABLE_SOURCE_CLASS!r} for a source that reads another table",
+                )
+            )
+        locator = source.get("locator")
+        locator = locator if isinstance(locator, Mapping) else {}
+        if locator.get("kind") != TABLE_LOCATOR_KIND:
+            findings.append(
+                Finding(
+                    DOCUMENT_INVALID,
+                    f"{pointer}/locator/kind",
+                    f"must be {TABLE_LOCATOR_KIND!r} for a source that reads another table",
+                )
+            )
+        table = locator.get("table")
+        if not isinstance(table, Mapping):
+            findings.append(
+                Finding(
+                    DOCUMENT_INVALID,
+                    f"{pointer}/locator/table",
+                    "is absent; it names the upstream table as {table_id, version: "
+                    "latest_promoted}",
+                )
+            )
+        else:
+            table_id = table.get("table_id")
+            if not isinstance(table_id, str) or _UUID.match(table_id) is None:
+                findings.append(
+                    Finding(
+                        DOCUMENT_INVALID,
+                        f"{pointer}/locator/table/table_id",
+                        "is not a lowercase UUID naming a table of this workspace",
+                    )
+                )
+            if _outside(table.get("version"), TABLE_SOURCE_VERSIONS):
+                findings.append(
+                    Finding(
+                        DOCUMENT_INVALID,
+                        f"{pointer}/locator/table/version",
+                        "must be 'latest_promoted', the one version selector Studio admits",
+                    )
+                )
+            for member in sorted(set(table) - {"table_id", "version"}):
+                findings.append(
+                    Finding(
+                        DOCUMENT_INVALID,
+                        f"{pointer}/locator/table/{_escape(str(member))}",
+                        "is not a member of a table locator",
+                    )
+                )
+        connector = source.get("connector")
+        connector = connector if isinstance(connector, Mapping) else {}
+        if connector.get("adapter_id") != TABLE_SOURCE_ADAPTER_ID:
+            findings.append(
+                Finding(
+                    DOCUMENT_INVALID,
+                    f"{pointer}/connector/adapter_id",
+                    f"must be {TABLE_SOURCE_ADAPTER_ID!r} for a source that reads another table",
+                )
+            )
+        if connector.get("credential_mode") != "none":
+            findings.append(
+                Finding(
+                    DOCUMENT_INVALID,
+                    f"{pointer}/connector/credential_mode",
+                    "must be 'none': a table source reads this workspace's own sealed parts",
+                )
+            )
+        parameters = connector.get("parameters")
+        if not isinstance(parameters, list) or len(parameters) < 2:
+            findings.append(
+                Finding(
+                    DOCUMENT_INVALID,
+                    f"{pointer}/connector/parameters",
+                    "must pin the Reader that decodes the upstream's Parquet parts "
+                    "(reader.family_id parquet.tabular, reader.family_version, "
+                    "reader.decode_options)",
+                )
+            )
+        for member in TABLE_CONNECTOR_FORBIDDEN:
+            if member in connector:
+                findings.append(
+                    Finding(
+                        DOCUMENT_INVALID,
+                        f"{pointer}/connector/{member}",
+                        "is not stated on a table source, which reaches no origin",
+                    )
+                )
+        for member in TABLE_SOURCE_FORBIDDEN:
+            if member in source:
+                findings.append(
+                    Finding(
+                        DOCUMENT_INVALID,
+                        f"{pointer}/{member}",
+                        "is not stated on a table source, which is read whole and presents no "
+                        "credential",
+                    )
+                )
+    return findings
+
+
+def _refresh_trigger(document: Mapping[str, Any]) -> list[Finding]:
+    """``refresh: {trigger: upstream_promoted, debounce_seconds}`` and what it requires.
+
+    The trigger asks Studio to rebuild this table when an upstream table it reads promotes a new
+    version, so the schema requires ``rebuild`` beside it (the rebuild is what answers the request,
+    under its own interval and byte bounds) and at least one ``table.version@1.0.0`` source.
+    """
+
+    if "refresh" not in document:
+        return []
+    refresh = document.get("refresh")
+    if not isinstance(refresh, Mapping):
+        return [Finding(DOCUMENT_INVALID, "/refresh", "is not one JSON object")]
+    findings: list[Finding] = []
+    if _outside(refresh.get("trigger"), REFRESH_TRIGGERS):
+        findings.append(
+            Finding(DOCUMENT_INVALID, "/refresh/trigger", "must be 'upstream_promoted'")
+        )
+    debounce = refresh.get("debounce_seconds")
+    if (
+        isinstance(debounce, bool)
+        or not isinstance(debounce, int)
+        or not 0 <= debounce <= MAX_REFRESH_DEBOUNCE_SECONDS
+    ):
+        findings.append(
+            Finding(
+                DOCUMENT_INVALID,
+                "/refresh/debounce_seconds",
+                f"must be a whole number of seconds from 0 to {MAX_REFRESH_DEBOUNCE_SECONDS}",
+            )
+        )
+    for member in sorted(set(refresh) - {"trigger", "debounce_seconds"}):
+        findings.append(
+            Finding(
+                DOCUMENT_INVALID,
+                f"/refresh/{_escape(str(member))}",
+                "is not a member of refresh",
+            )
+        )
+    if "rebuild" not in document:
+        findings.append(
+            Finding(
+                DOCUMENT_INVALID,
+                "/rebuild",
+                "is absent, and refresh requires it: the declared whole-table rebuild is what "
+                "answers an upstream promotion",
+            )
+        )
+    sources = document.get("sources")
+    if not any(
+        isinstance(source, Mapping)
+        and isinstance(source.get("connector"), Mapping)
+        and source["connector"].get("adapter_id") == TABLE_SOURCE_ADAPTER_ID
+        for source in (sources if isinstance(sources, list) else ())
+    ):
+        findings.append(
+            Finding(
+                DOCUMENT_INVALID,
+                "/refresh",
+                f"needs at least one {TABLE_SOURCE_ADAPTER_ID} source: only an upstream table "
+                "this recipe reads can promote",
+            )
+        )
+    return findings
+
+
+def _classification_coherence(sources: Any) -> list[Finding]:
+    """Every source of one run declares one ``data_classification``.
+
+    The worker's governance refuses a run whose sources declare two classifications
+    (``RUN_RECIPE_INVALID`` at acquire), after the document registered and a run was queued, so
+    it is named here. It matters most for a join of Mostly Right tables, whose sources inherit
+    each upstream's classification and must still agree.
+    """
+
+    stated = [
+        (index, source.get("data_classification"))
+        for index, source in enumerate(sources if isinstance(sources, list) else ())
+        if isinstance(source, Mapping) and isinstance(source.get("data_classification"), str)
+    ]
+    if len({value for _index, value in stated}) <= 1:
+        return []
+    first = stated[0][1]
+    return [
+        Finding(
+            DOCUMENT_INCOHERENT,
+            f"/sources/{index}/data_classification",
+            f"is {value!r} while /sources/{stated[0][0]} is {first!r}; the worker refuses a run "
+            "whose sources declare more than one classification, at acquire",
+        )
+        for index, value in stated
+        if value != first
+    ]
+
+
 def _row_filters(sources: Any) -> list[Finding]:
     """``collection.discovery.row_filter``: where one may stand, and what it must be.
 
@@ -905,6 +1173,7 @@ def _coherence(document: Mapping[str, Any], *, declared: Sequence[str]) -> list[
     findings.extend(_table_partition_coherence(document, known=known))
     findings.extend(_reference_coherence(document.get("sources")))
     findings.extend(_rebuild_coherence(document))
+    findings.extend(_classification_coherence(document.get("sources")))
     return findings
 
 

{mostlyright_data-0.30.3 → mostlyright_data-0.30.5}/src/mostlyright/data_harness/thin/table_search_contract.json

@@ -1,6 +1,6 @@
 {
  "schema_version": "mostlyright-thin-table-search-contract.v1",
- "source_commit": "b308bc2e9ebb424b4cfe1130438ab9f7d0b6736d",
+ "source_commit": "f61f08e29fc94e2fb1c2d16e0de2e08522a9f993",
  "recipe_search_definition": "spec",
  "$defs": {
   "evaluation_case": {

{mostlyright_data-0.30.3 → mostlyright_data-0.30.5}/src/mostlyright/data_harness/thin/v4.py

@@ -148,8 +148,8 @@ from mostlyright.data_harness.thin.transport import ThinLaneError
 #: ``JOB_INVALID`` outright, which in production was every refresh a collection epoch was offered
 #: for rather than only the collection runs. This package must not reach a Studio older than the
 #: commit it pins; ``docs/V4-WORKER-PROTOCOL.md`` states it beside the layout's own ordering rule.
-PINNED_V4_OPENAPI_SOURCE_SHA256 = "001a5e132e8af9872ac61168e41df4465627f34ac062066b212f1b48a88385f4"
-PINNED_V4_CONTRACT_VERSION = "4.35.0"
+PINNED_V4_OPENAPI_SOURCE_SHA256 = "67cc1e66ae3610d9ae5c8b5469bb05e51a8519e55ecef502387fdbb4185b084e"
+PINNED_V4_CONTRACT_VERSION = "4.38.0"
 
 #: The ``schema_version`` every command body that predates 4.14.0 stamps: ``dataset activity``,
 #: ``table rename``, ``table archive`` and ``dataset archive``. It is the number the 0.25 line

{mostlyright_data-0.30.3 → mostlyright_data-0.30.5}/src/mostlyright/data_harness/thin/v4_stream.py

@@ -255,8 +255,9 @@ def watch(
     rates = RateTracker()
 
     def _observe(body: Mapping[str, Any], frame_name: str) -> bool:
-        nonlocal seen_any
+        nonlocal seen_any, quiet_since
         seen_any = True
+        quiet_since = clock()
         if announce:
             print(render_v4_frame(body, frame_name), flush=True)
         moved = rates.observe(body)
@@ -271,10 +272,20 @@ def watch(
 
     def _quiet() -> bool:
         # Asked on every heartbeat of a quiet stream. Never stops the watch; it only says, once
-        # per quiet stretch, that nothing has arrived yet and why that is the ordinary case.
+        # per quiet interval, what was (and was not) observed. Heartbeats prove only
+        # stream connectivity, never worker liveness or progress.
         nonlocal quiet_since
         now = clock()
-        if announce and not seen_any and now - quiet_since >= QUIET_NOTE_SECONDS:
+        if announce and now - quiet_since >= QUIET_NOTE_SECONDS:
+            if seen_any:
+                print(
+                    f"no new progress report for {int(now - quiet_since)} seconds: "
+                    "work may still be running; stream connectivity is not worker progress; "
+                    "ETA unknown",
+                    flush=True,
+                )
+                quiet_since = now
+                return False
             waited = int(now - started)
             print(
                 f"nothing reported yet after {waited} seconds: the run was {run.get('status')} "