mostlyright-data 0.13.0__tar.gz → 0.14.1__tar.gz

{mostlyright_data-0.13.0 → mostlyright_data-0.14.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mostlyright-data
-Version: 0.13.0
+Version: 0.14.1
 Summary: Mostly Right CLI for reproducible, reviewed Parquet datasets
 Project-URL: Homepage, https://mostlyright.md/
 Project-URL: Documentation, https://mostlyright.md/docs/guides/cli/

{mostlyright_data-0.13.0 → mostlyright_data-0.14.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mostlyright-data"
-version = "0.13.0"
+version = "0.14.1"
 description = "Mostly Right CLI for reproducible, reviewed Parquet datasets"
 readme = "README.md"
 requires-python = ">=3.13,<3.14"

{mostlyright_data-0.13.0 → mostlyright_data-0.14.1}/skills/mr-data-build/SKILL.md

@@ -525,6 +525,71 @@ which Studio fixes. Do it once per dataset, silently, and do not repeat it for a
 subject changed. A dataset presented without a cover is a defect, the same as a one-line
 description.
 
+### Say what each source is
+
+Every entry in `sources` takes an optional `description`, and every one you write gets one. It is
+a single sentence saying what that source **is** — `Daily climatological reports, the official
+highs and lows.` — and the dataset page draws it beside the endpoint, under the publisher it came
+from. That line is the only place a reader ever learns what a URL full of query parameters
+actually returned, so it is written for somebody who has never seen the recipe.
+
+Write it about the DATA, never about the run: `Hourly airport surface observations` and not `the
+2024 half of the window`, `fetched as CSV` or `unioned in step one`. Do not restate the address in
+words, and do not repeat the publisher's product code where it says nothing a reader can use. Up
+to 240 characters, which is one sentence rather than two. Where one window is split across several
+sources on the same endpoint, all of them describe the same thing and get the same sentence — the
+years are already in their names.
+
+Leaving it out is a defect in the same way a one-line dataset description is: the source is then
+drawn with its endpoint alone, and a reader is left to guess from a query string.
+
+### Say what each column shows
+
+Decide, per column, what a reader should see, and write it on the column as `presentation`. This is
+your judgement about the data and nobody else can make it: the Builder can see the values but not
+the question they were gathered to answer.
+
+    {
+      "name": "observed_on",
+      "type": "date",
+      "description": "The day the readings were taken.",
+      "presentation": {
+        "chart": "timeline",
+        "story": "The archive runs unbroken across three summer months.",
+        "bucket": "month"
+      }
+    }
+
+`chart` says what to draw and is one of `histogram`, `timeline`, `top_values`, `share` and `none`.
+Rules of thumb, and they hold most of the time:
+
+- Identifiers, free text, notes, anything one row carries its own copy of — `none`. Counting the
+  twelve commonest case numbers says only that the sample is a sample.
+- Measurements — `histogram`. Temperatures, prices, durations, counts of things.
+- Event times — `timeline`, with `bucket` matched to the grain the rows arrive at: hourly readings
+  get `hour` or `day`, a decade of annual filings gets `year`.
+- Categories — `top_values`. Cities, statuses, publishers, anything a reader would group by.
+- Flags — `share`. One split bar, the true part against the false part.
+
+`story` is the sentence a reader should be able to say after looking. Write it about the DATA —
+"rain is concentrated in a handful of days and most days record almost none" — and never about the
+chart, because the reader can see that it is a chart. Leave it out rather than write a caption.
+
+Every member is optional, including `presentation` itself. A column that says nothing still gets a
+chart: the Builder picks one from the type you declared and the dataset page says the choice was
+its own rather than yours. Say something anyway wherever you have an opinion — a column you
+thought about reads differently from one nobody did.
+
+Three things can make the Builder replace the chart you asked for, and each of them drops your
+`story` with it: a column that turns out to be entirely missing, an event time spanning more than
+twelve years (no grain draws that in twelve buckets), and a build whose sealed output type
+contradicts what the column declared. A `bucket` the span outgrew is not one of them — the grain
+is widened and the chart stays yours. So `none` is the right answer to write for a column you
+already know has no shape, and a wide date range is worth a thought before you promise a timeline.
+
+[The recipe document](../../docs/RECIPE-DOCUMENT.md) carries which types each chart admits, what
+the defaults are, and the two members only some charts take.
+
 ### What the statement must actually return
 
 Every acquired source is bound as all-text CSV — `read_csv(header = true, auto_detect = false,

{mostlyright_data-0.13.0 → mostlyright_data-0.14.1}/src/mostlyright/data_harness/hosted_run_worker.py

@@ -1162,6 +1162,9 @@ def v4_artifact_payloads(
             workspace_id=job.workspace_id,
             run_id=job.run_id,
             candidate_digest=candidate_digest,
+            # The parsed document, not a second read of the bytes: the recipe is already in hand
+            # here, and a profile that re-read it could disagree with the run that produced it.
+            declared_columns={column.name: column for column in job.recipe.table.columns},
         ),
         "preview": _table_preview(snapshot["data/table.parquet"], candidate_digest),
         "receipt": _receipt_document(job, verified, coverage, raw_members, snapshot),

{mostlyright_data-0.13.0 → mostlyright_data-0.14.1}/src/mostlyright/data_harness/hosted_stream_subscription_worker.py

@@ -17,6 +17,7 @@ its own.
 from __future__ import annotations
 
 import asyncio
+import logging
 import os
 import shutil
 import stat
@@ -281,15 +282,7 @@ def _materialized_claim_connector(
     root = Path(tempfile.mkdtemp(prefix="stream-subscription-feed-", dir=work_root))
     try:
         os.chmod(root, 0o700)
-        registry_root = root / "registry"
-        registry_root.mkdir(mode=0o700)
-        materialize_connector_registry(
-            transport=objects,
-            bucket=_claim_text(claim.stream_venue_bucket, "stream venue bucket"),
-            registry_prefix=_claim_text(claim.connector_registry_prefix, "registry prefix"),
-            registry_digest=_claim_text(claim.connector_registry_digest, "registry digest"),
-            root=registry_root,
-        )
+        registry_root = _materialized_registry_root(claim, objects, root)
         registry = load_stream_connector_registry(registry_root)
         entry = registry.resolve(
             _claim_text(claim.connector_id, "connector id"),
@@ -314,6 +307,33 @@ def _materialized_claim_connector(
         shutil.rmtree(root, ignore_errors=True)
 
 
+def _materialized_registry_root(
+    claim: ClaimedSubscription, objects: GCSObjectTransport, root: Path
+) -> Path:
+    """Materialize the pinned registry under ``root`` and file its series documents.
+
+    ONE MATERIALIZATION SHAPE FOR BOTH LOADERS.  A published registry is one flat name-to-digest
+    table; the connector loader reads the documents directly beneath the root and the series
+    loader reads the ``series`` directory under it.  A subscription that follows a series is
+    pinned to a registry that carries both kinds by construction, so a root left unsorted is not
+    a connector registry at all and the feed is refused before it opens.
+    """
+
+    registry_root = root / "registry"
+    registry_root.mkdir(mode=0o700)
+    series_root = registry_root / SERIES_DIRECTORY_NAME
+    series_root.mkdir(mode=0o700)
+    materialize_connector_registry(
+        transport=objects,
+        bucket=_claim_text(claim.stream_venue_bucket, "stream venue bucket"),
+        registry_prefix=_claim_text(claim.connector_registry_prefix, "registry prefix"),
+        registry_digest=_claim_text(claim.connector_registry_digest, "registry digest"),
+        root=registry_root,
+    )
+    _sort_series_documents(registry_root, series_root)
+    return registry_root
+
+
 def _claim_text(value: str | None, label: str) -> str:
     if not isinstance(value, str) or not value:
         raise StreamSubscriptionError("STREAM_SUBSCRIPTION_CONNECTOR_INVALID", label)
@@ -385,18 +405,7 @@ def _materialized_claim_series(
     root = Path(tempfile.mkdtemp(prefix="stream-subscription-series-", dir=work_root))
     try:
         os.chmod(root, 0o700)
-        registry_root = root / "registry"
-        registry_root.mkdir(mode=0o700)
-        series_root = registry_root / SERIES_DIRECTORY_NAME
-        series_root.mkdir(mode=0o700)
-        materialize_connector_registry(
-            transport=objects,
-            bucket=_claim_text(claim.stream_venue_bucket, "stream venue bucket"),
-            registry_prefix=_claim_text(claim.connector_registry_prefix, "registry prefix"),
-            registry_digest=_claim_text(claim.connector_registry_digest, "registry digest"),
-            root=registry_root,
-        )
-        _sort_series_documents(registry_root, series_root)
+        registry_root = _materialized_registry_root(claim, objects, root)
         documents = load_stream_series_documents(registry_root)
         held = documents.get(series.series_id)
         if held is None:
@@ -625,6 +634,7 @@ async def run_subscription_worker(
 def stream_subscription_worker_main(argv: Sequence[str] | None = None) -> int:
     """Cloud Run Worker Pool executable; it intentionally accepts no mutable CLI authority."""
 
+    _configure_worker_logging()
     arguments = tuple(sys.argv[1:] if argv is None else argv)
     if arguments:
         raise HostedStreamSubscriptionWorkerError(
@@ -636,6 +646,25 @@ def stream_subscription_worker_main(argv: Sequence[str] | None = None) -> int:
     return 0
 
 
+def _configure_worker_logging() -> None:
+    """Configure the root logger for the pool executable, on a stream its host collects.
+
+    A library module configures nothing; an executable must.  Unconfigured, this process fell
+    through to the last-resort handler, which emits a bare message with no level and no logger
+    name, so the pool's own refusals arrived unattributable.
+
+    Plain text on either stream is collected at DEFAULT severity, so the level is carried in the
+    line rather than in the record -- which is why the format names it.  Stdout is chosen over the
+    sibling executables' stderr for the same reason it is chosen anywhere: a stream that carries
+    ordinary progress as well as refusals is not the error stream.  An already-configured root is
+    left alone, which is what lets a smoke or a test harness own it instead.
+    """
+
+    logging.basicConfig(
+        level=logging.INFO, stream=sys.stdout, format="%(levelname)s %(name)s %(message)s"
+    )
+
+
 def _studio_origin(value: str | None) -> str:
     if not isinstance(value, str) or not 1 <= len(value) <= 2048 or value.endswith("/"):
         raise HostedStreamSubscriptionWorkerError(