mostlyright-data 0.9.14__tar.gz → 0.10.0__tar.gz

{mostlyright_data-0.9.14 → mostlyright_data-0.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mostlyright-data
-Version: 0.9.14
+Version: 0.10.0
 Summary: Mostly Right CLI for reproducible, reviewed Parquet datasets
 Project-URL: Homepage, https://mostlyright.md/
 Project-URL: Documentation, https://mostlyright.md/docs/guides/cli/
@@ -9,28 +9,41 @@ Provides-Extra: build
 Requires-Dist: hatchling==1.28.0; extra == 'build'
 Requires-Dist: uv-build==0.11.3; extra == 'build'
 Provides-Extra: dev
+Requires-Dist: astropy==8.0.1; extra == 'dev'
 Requires-Dist: cmarkgfm>=2025.10.22; extra == 'dev'
 Requires-Dist: croniter==6.2.4; extra == 'dev'
 Requires-Dist: cryptography<51,>=50.0.0; extra == 'dev'
+Requires-Dist: duckdb==1.5.5; extra == 'dev'
+Requires-Dist: eccodes==2.48.0; extra == 'dev'
+Requires-Dist: eccodeslib==2.48.0.26; (sys_platform != 'win32') and extra == 'dev'
 Requires-Dist: et-xmlfile==2.0.0; extra == 'dev'
+Requires-Dist: h5py==3.16.0; extra == 'dev'
 Requires-Dist: idna<4,>=3.10; extra == 'dev'
 Requires-Dist: jsonschema[format]<5,>=4.23; extra == 'dev'
+Requires-Dist: netcdf4==1.7.4; extra == 'dev'
 Requires-Dist: numpy==2.5.1; extra == 'dev'
 Requires-Dist: openpyxl==3.1.5; extra == 'dev'
 Requires-Dist: packaging<27,>=24; extra == 'dev'
 Requires-Dist: pandas==3.0.3; extra == 'dev'
+Requires-Dist: pillow==12.3.0; extra == 'dev'
 Requires-Dist: polars-runtime-32==1.42.1; extra == 'dev'
 Requires-Dist: polars==1.42.1; extra == 'dev'
 Requires-Dist: pyarrow==23.0.1; extra == 'dev'
+Requires-Dist: pyhdf==0.11.7; ((sys_platform == 'linux' and platform_machine == 'x86_64') or (sys_platform == 'darwin' and platform_machine == 'arm64')) and extra == 'dev'
+Requires-Dist: pypublicdecompwt==2.8.1.6; extra == 'dev'
+Requires-Dist: pyreadstat==1.3.6; extra == 'dev'
 Requires-Dist: pytest-xdist>=3.6; extra == 'dev'
 Requires-Dist: pytest>=8.3; extra == 'dev'
+Requires-Dist: rasterio==1.5.1; extra == 'dev'
 Requires-Dist: referencing<1,>=0.35; extra == 'dev'
 Requires-Dist: ruff>=0.9; extra == 'dev'
+Requires-Dist: zarr==3.3.0; extra == 'dev'
 Provides-Extra: hosted
 Requires-Dist: mostlyright-studio-client-v3==3.0.0; extra == 'hosted'
 Provides-Extra: local
 Requires-Dist: croniter==6.2.4; extra == 'local'
 Requires-Dist: cryptography<51,>=50.0.0; extra == 'local'
+Requires-Dist: duckdb==1.5.5; extra == 'local'
 Requires-Dist: et-xmlfile==2.0.0; extra == 'local'
 Requires-Dist: idna<4,>=3.10; extra == 'local'
 Requires-Dist: numpy==2.5.1; extra == 'local'
@@ -40,6 +53,7 @@ Requires-Dist: pandas==3.0.3; extra == 'local'
 Requires-Dist: polars-runtime-32==1.42.1; extra == 'local'
 Requires-Dist: polars==1.42.1; extra == 'local'
 Requires-Dist: pyarrow==23.0.1; extra == 'local'
+Requires-Dist: pyreadstat==1.3.6; extra == 'local'
 Provides-Extra: local-search-neural
 Requires-Dist: anyio==4.14.2; extra == 'local-search-neural'
 Requires-Dist: certifi==2026.7.22; extra == 'local-search-neural'
@@ -61,9 +75,12 @@ Requires-Dist: pyyaml==6.0.3; extra == 'local-search-neural'
 Requires-Dist: tokenizers==0.23.1; extra == 'local-search-neural'
 Requires-Dist: tqdm==4.70.0; extra == 'local-search-neural'
 Requires-Dist: typing-extensions==4.16.0; extra == 'local-search-neural'
+Provides-Extra: recorder
+Requires-Dist: cryptography<51,>=50.0.0; extra == 'recorder'
 Provides-Extra: release
 Requires-Dist: croniter==6.2.4; extra == 'release'
 Requires-Dist: cryptography<51,>=50.0.0; extra == 'release'
+Requires-Dist: duckdb==1.5.5; extra == 'release'
 Requires-Dist: et-xmlfile==2.0.0; extra == 'release'
 Requires-Dist: idna<4,>=3.10; extra == 'release'
 Requires-Dist: jsonschema[format]<5,>=4.23; extra == 'release'
@@ -74,6 +91,18 @@ Requires-Dist: pandas==3.0.3; extra == 'release'
 Requires-Dist: polars-runtime-32==1.42.1; extra == 'release'
 Requires-Dist: polars==1.42.1; extra == 'release'
 Requires-Dist: pyarrow==23.0.1; extra == 'release'
+Requires-Dist: pyreadstat==1.3.6; extra == 'release'
+Provides-Extra: scientific
+Requires-Dist: astropy==8.0.1; extra == 'scientific'
+Requires-Dist: eccodes==2.48.0; extra == 'scientific'
+Requires-Dist: eccodeslib==2.48.0.26; (sys_platform != 'win32') and extra == 'scientific'
+Requires-Dist: h5py==3.16.0; extra == 'scientific'
+Requires-Dist: netcdf4==1.7.4; extra == 'scientific'
+Requires-Dist: pillow==12.3.0; extra == 'scientific'
+Requires-Dist: pyhdf==0.11.7; ((sys_platform == 'linux' and platform_machine == 'x86_64') or (sys_platform == 'darwin' and platform_machine == 'arm64')) and extra == 'scientific'
+Requires-Dist: pypublicdecompwt==2.8.1.6; extra == 'scientific'
+Requires-Dist: rasterio==1.5.1; extra == 'scientific'
+Requires-Dist: zarr==3.3.0; extra == 'scientific'
 Provides-Extra: viewer
 Requires-Dist: nbconvert==7.17.1; extra == 'viewer'
 Requires-Dist: nbformat==5.10.4; extra == 'viewer'
@@ -84,8 +113,9 @@ Description-Content-Type: text/markdown
 Browse datasets built by our community or create your own with agents that find the sources and
 configure the pipeline. Mostly Right keeps the dataset live.
 
-`mostlyright-data` is the command-line client. It opens hosted research sessions, starts and
-follows hosted dataset runs, and downloads verified datasets after they finish.
+`mostlyright-data` is the command-line client. It probes sources, registers one recipe document,
+starts and follows hosted dataset runs, reads what they delivered, and downloads the datasets after
+they finish.
 
 ## Requirements
 
@@ -115,8 +145,20 @@ mr-data --version
 mr-data --help
 ```
 
-Use `mr-data research-open`, `mr-data research-status`, and `mr-data research-close` for hosted
-research sessions. `research-status` shows the current state.
+## Build a dataset
+
+```bash
+mr-data probe SOURCE_ID "does this feed carry city grain before 2021"
+mr-data recipe RECIPE.json
+mr-data run RECIPE_ID --sample --window 2025-03-01..2025-04-01 --max-rows 50000
+mr-data checks RUN
+mr-data download RUN --output ./out
+```
+
+`probe` asks one registered source one question; the session behind it is opened, reused and closed
+for you. `recipe` registers one document and prints the identifiers the server derived from it.
+`run` starts a run and `mr-data status RUN`, `mr-data runs`, and `mr-data watch RUN` report on it.
+`mr-data promote TABLE_ID` makes a table live, and it is the one step that asks a person first.
 
 ## Local execution
 
@@ -147,8 +189,10 @@ Use `mr-data --help` for the full command list and options. Commands that suppor
 
 - `mr-data review RUN_DIR --local` runs the deterministic checks and writes no review decision. It exits 1 when a check fails. Write the flag after the command name; before it, `--local` is the lane selector.
 - `mr-data review RUN_DIR --report REPORT` seals a review decision from signed reviewer reports.
-- The weather reader accepts one extracted GRIB2 record. It refuses a multi-record GFS or HRRR file.
+- Weather Reader versions 1 and 2 accept one extracted GRIB2 record and refuse a multi-record GFS
+  or HRRR file. Version 3 reads a bounded collection and selects exactly one sealed record.
 - Hosted credential-free acquisition supports one single-address public HTTPS object. Bounded collections require both local attestations.
+- A keyed stream venue is named by a credential reference the backend holds, never by a value. No command flag takes a secret as an argument; the local recorder lane reads one from a `0600` file named by `--credential-file`. Venues reached over an actual WebSocket are covered; a venue whose credential comes from an interactive human login, or that is not WebSocket at all, is not.
 
 ## Verification
 

{mostlyright_data-0.9.14 → mostlyright_data-0.10.0}/README.md

@@ -3,8 +3,9 @@
 Browse datasets built by our community or create your own with agents that find the sources and
 configure the pipeline. Mostly Right keeps the dataset live.
 
-`mostlyright-data` is the command-line client. It opens hosted research sessions, starts and
-follows hosted dataset runs, and downloads verified datasets after they finish.
+`mostlyright-data` is the command-line client. It probes sources, registers one recipe document,
+starts and follows hosted dataset runs, reads what they delivered, and downloads the datasets after
+they finish.
 
 ## Requirements
 
@@ -34,8 +35,20 @@ mr-data --version
 mr-data --help
 ```
 
-Use `mr-data research-open`, `mr-data research-status`, and `mr-data research-close` for hosted
-research sessions. `research-status` shows the current state.
+## Build a dataset
+
+```bash
+mr-data probe SOURCE_ID "does this feed carry city grain before 2021"
+mr-data recipe RECIPE.json
+mr-data run RECIPE_ID --sample --window 2025-03-01..2025-04-01 --max-rows 50000
+mr-data checks RUN
+mr-data download RUN --output ./out
+```
+
+`probe` asks one registered source one question; the session behind it is opened, reused and closed
+for you. `recipe` registers one document and prints the identifiers the server derived from it.
+`run` starts a run and `mr-data status RUN`, `mr-data runs`, and `mr-data watch RUN` report on it.
+`mr-data promote TABLE_ID` makes a table live, and it is the one step that asks a person first.
 
 ## Local execution
 
@@ -66,8 +79,10 @@ Use `mr-data --help` for the full command list and options. Commands that suppor
 
 - `mr-data review RUN_DIR --local` runs the deterministic checks and writes no review decision. It exits 1 when a check fails. Write the flag after the command name; before it, `--local` is the lane selector.
 - `mr-data review RUN_DIR --report REPORT` seals a review decision from signed reviewer reports.
-- The weather reader accepts one extracted GRIB2 record. It refuses a multi-record GFS or HRRR file.
+- Weather Reader versions 1 and 2 accept one extracted GRIB2 record and refuse a multi-record GFS
+  or HRRR file. Version 3 reads a bounded collection and selects exactly one sealed record.
 - Hosted credential-free acquisition supports one single-address public HTTPS object. Bounded collections require both local attestations.
+- A keyed stream venue is named by a credential reference the backend holds, never by a value. No command flag takes a secret as an argument; the local recorder lane reads one from a `0600` file named by `--credential-file`. Venues reached over an actual WebSocket are covered; a venue whose credential comes from an interactive human login, or that is not WebSocket at all, is not.
 
 ## Verification
 

{mostlyright_data-0.9.14 → mostlyright_data-0.10.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mostlyright-data"
-version = "0.9.14"
+version = "0.10.0"
 description = "Mostly Right CLI for reproducible, reviewed Parquet datasets"
 readme = "README.md"
 requires-python = ">=3.13,<3.14"
@@ -50,6 +50,9 @@ mostlyright-grib = [
     { path = "vendor/wheels/mostlyright_grib-0.1.0-py3-none-manylinux_2_36_x86_64.whl", marker = "sys_platform == 'linux'" },
 ]
 mostlyright-studio-client-v3 = { path = "vendor/wheels/mostlyright_studio_client_v3-3.0.0-py3-none-any.whl" }
+pyhdf = [
+    { path = "vendor/wheels/pyhdf-0.11.7-cp313-cp313-macosx_15_0_arm64.whl", marker = "sys_platform == 'darwin' and platform_machine == 'arm64'" },
+]
 
 [project.scripts]
 # ⚠ NOT `cli:main`. `cli` imports `hosted_worker`, which imports pyarrow and cryptography at module
@@ -71,8 +74,12 @@ mr-data-catalog-job = "mostlyright.data_harness.thin.workers:catalog_job_main"
 mr-data-crawler = "mostlyright.data_harness.thin.workers:crawler_main"
 mr-data-crawler-job = "mostlyright.data_harness.thin.workers:crawler_job_main"
 mr-data-producer = "mostlyright.data_harness.thin.workers:producer_main"
+# The v4 run worker's batch shape. It names what the executable does -- it runs a run -- rather
+# than a v3 role, and it routes through the shim for the same reason every name above does:
+# `hosted_run_worker` imports `hosted_worker`, so a thin install would fail inside the entry
+# point's own import before the entry point could say anything.
+mr-data-run-worker = "mostlyright.data_harness.thin.workers:run_worker_main"
 mr-data-generic-api = "mostlyright.data_harness.thin.workers:generic_api_probe_main"
-mr-data-verifier = "mostlyright.data_harness.thin.workers:verifier_main"
 # The legacy Cloud Run service entrypoint remains available for rollback. The Job entrypoint is a
 # separate executable so an image revision cannot accidentally start the retired HTTP service.
 # Both go through the same shim because `hosted_session_worker` imports `hosted_worker`.
@@ -103,6 +110,16 @@ build = [
 local = [
     "croniter==6.2.4",
     "cryptography>=50.0.0,<51",
+    # The warm session worker's bounded-query engine, and the ONLY reason it is here rather than
+    # in the base `dependencies`: duckdb ships a compiled extension, and `tests/test_thin_profile.py`
+    # installs the base closure into a fresh environment and refuses any `.so`/`.pyd`/`.dylib` it
+    # finds under site-packages. `bounded_query.py` is what uses it, and it is REQUIRED there
+    # rather than convenient: `docs/V4-WORKER-PROTOCOL.md` puts the security boundary for an
+    # agent-supplied statement in this repository -- a read-only connection with
+    # `enable_external_access=false` -- and says a worker that cannot enforce that must not claim
+    # queries. polars' `SQLContext` was considered and rejected: it adds no package and offers no
+    # equivalent of that setting.
+    "duckdb==1.5.5",
     # UTS46 with URL Standard validity checks. `acquisition/url_policy` requires the same
     # domain-to-ASCII behavior as the URL parser; Python's `idna` codec implements IDNA2003.
     "idna>=3.10,<4",
@@ -114,13 +131,54 @@ local = [
     "pandas==3.0.3",
     "polars==1.42.1",
     "polars-runtime-32==1.42.1",
+    # The bounded statistical-microdata Readers use the reviewed ReadStat binding for Stata,
+    # SPSS, and SAS Transport files.  It stays in the local lane: the base hosted client must
+    # remain dependency-free, and the compiled binding is never imported before exact Reader
+    # resolution.
+    "pyreadstat==1.3.6",
+]
+# What the hosted stream recorder needs beyond the base distribution, and nothing else. The two
+# asymmetric signing algorithms in the keyed-wss connector grammar -- `rsa-pss-sha256` for Kalshi
+# and `ed25519` for Binance `session.logon` -- are the whole of it; the three HMAC algorithms are
+# stdlib.
+#
+# Deliberately a separate extra from `local` even though `local` already carries the same pin and
+# the producer image installs both. The recorder is a hosted lane and `local` is a retired one; an
+# image that stops installing `local` must not silently lose the ability to authenticate to a
+# venue, and stating the requirement where it is used is what makes that impossible. The specifier
+# is character-identical to `local`'s on purpose, asserted as equal strings by
+# `tests/test_thin_profile.py`: two specifiers that resolve to one version today and drift
+# tomorrow is exactly the failure the load-bearing-pin warning above exists for.
+recorder = [
+    "cryptography>=50.0.0,<51",
+]
+# Native scientific decoders are kept out of the ordinary local and thin-client closures. The
+# hosted Builder and networkless Courier install this extra explicitly. Every Reader still gates
+# the exact format/profile before it crosses one of these native boundaries.
+scientific = [
+    "astropy==8.0.1",
+    "eccodes==2.48.0",
+    # The binding's wheel metadata does not make this native runtime visible to uv on every
+    # platform. State it directly so a worker never falls back to an ambient libecCodes.
+    # The native runtime wheel is separate on POSIX. The governed Windows eccodes wheel embeds its
+    # runtime and eccodeslib publishes no Windows artifact.
+    "eccodeslib==2.48.0.26; sys_platform != 'win32'",
+    "h5py==3.16.0",
+    "netCDF4==1.7.4",
+    # Linux resolves the exact upstream wheel. macOS arm64 uses the reviewed source build,
+    # because upstream's CP313 wheel is built with an unsupported macOS 26 deployment floor.
+    "pyhdf==0.11.7; (sys_platform == 'linux' and platform_machine == 'x86_64') or (sys_platform == 'darwin' and platform_machine == 'arm64')",
+    "Pillow==12.3.0",
+    "pypublicdecompwt==2.8.1.6",
+    "rasterio==1.5.1",
+    "zarr==3.3.0",
 ]
 dev = [
     # Development and every `verify-*` leg exercise the complete product, so the development extra
     # carries the local lane rather than making each Make target and each workflow remember to ask
     # for it. A leg that must prove the THIN closure installs the base distribution into its own
     # environment instead; see `tests/test_thin_profile.py`.
-    "mostlyright-data[local]",
+    "mostlyright-data[local,scientific]",
     "cmarkgfm>=2025.10.22",
     "jsonschema[format]>=4.23,<5",
     "pytest>=8.3",

{mostlyright_data-0.9.14 → mostlyright_data-0.10.0}/src/mostlyright/data_harness/acquisition/url_policy.py

@@ -416,6 +416,39 @@ def canonical_public_hostname(parsed: SplitResult) -> str:
     return _normalize_hostname(hostname)
 
 
+def names_internal_destination(hostname: str) -> bool:
+    """Whether ``hostname`` names somewhere only this deployment's own network can answer.
+
+    The four shapes an SSRF payload takes when the only thing under an author's control is a
+    hostname: the cloud metadata service by name, a private/loopback/link-local/reserved literal, a
+    single-label or ``.internal``/``.local`` name that no public resolver serves, and an obfuscated
+    numeric literal (``0x7f.1``, ``2130706433``) that ``ipaddress`` will not read while a resolver
+    will.
+
+    A PREDICATE OVER A NAME, and deliberately not a resolution: it is what a caller applies to an
+    address BEFORE a socket exists, so a refusal costs no packet. :func:`approved_public_addresses`
+    is the resolution rule, and it belongs to a transport that can pin the peer it dialled -- the
+    two are complementary and neither replaces the other. Callers holding a URL rather than a bare
+    name should pass it through :func:`canonical_public_hostname` first, because this asks about the
+    machine name a browser would resolve and not about whatever text sat in the authority.
+    """
+
+    if hostname in _METADATA_HOSTS or hostname.endswith((".internal", ".local")):
+        return True
+    if "." not in hostname and ":" not in hostname:
+        return True
+    if _looks_like_obfuscated_ip(hostname):
+        return True
+    literal = _parse_ip_literal(hostname)
+    if literal is None:
+        return False
+    try:
+        _require_globally_routable(literal)
+    except AcquisitionSecurityError:
+        return True
+    return False
+
+
 def _validate_url_structure(parsed: SplitResult) -> None:
     if parsed.scheme != "https":
         raise AcquisitionSecurityError("URL_SCHEME", "only HTTPS URLs are allowed")