mudm-mcp 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. mudm_mcp-0.1.0/LICENSE +21 -0
  2. mudm_mcp-0.1.0/PKG-INFO +147 -0
  3. mudm_mcp-0.1.0/README.md +117 -0
  4. mudm_mcp-0.1.0/pyproject.toml +53 -0
  5. mudm_mcp-0.1.0/setup.cfg +4 -0
  6. mudm_mcp-0.1.0/src/mudm_mcp/__init__.py +4 -0
  7. mudm_mcp-0.1.0/src/mudm_mcp/catalog_client.py +236 -0
  8. mudm_mcp-0.1.0/src/mudm_mcp/facets.py +68 -0
  9. mudm_mcp-0.1.0/src/mudm_mcp/geometry.py +56 -0
  10. mudm_mcp-0.1.0/src/mudm_mcp/sandbox.py +147 -0
  11. mudm_mcp-0.1.0/src/mudm_mcp/server.py +234 -0
  12. mudm_mcp-0.1.0/src/mudm_mcp/spatial.py +164 -0
  13. mudm_mcp-0.1.0/src/mudm_mcp/tools.py +541 -0
  14. mudm_mcp-0.1.0/src/mudm_mcp/units.py +46 -0
  15. mudm_mcp-0.1.0/src/mudm_mcp.egg-info/PKG-INFO +147 -0
  16. mudm_mcp-0.1.0/src/mudm_mcp.egg-info/SOURCES.txt +30 -0
  17. mudm_mcp-0.1.0/src/mudm_mcp.egg-info/dependency_links.txt +1 -0
  18. mudm_mcp-0.1.0/src/mudm_mcp.egg-info/entry_points.txt +2 -0
  19. mudm_mcp-0.1.0/src/mudm_mcp.egg-info/requires.txt +7 -0
  20. mudm_mcp-0.1.0/src/mudm_mcp.egg-info/top_level.txt +1 -0
  21. mudm_mcp-0.1.0/tests/test_catalog_client.py +128 -0
  22. mudm_mcp-0.1.0/tests/test_download_query.py +127 -0
  23. mudm_mcp-0.1.0/tests/test_facets.py +92 -0
  24. mudm_mcp-0.1.0/tests/test_geometry.py +48 -0
  25. mudm_mcp-0.1.0/tests/test_live.py +64 -0
  26. mudm_mcp-0.1.0/tests/test_patches.py +111 -0
  27. mudm_mcp-0.1.0/tests/test_response_size.py +71 -0
  28. mudm_mcp-0.1.0/tests/test_sandbox.py +99 -0
  29. mudm_mcp-0.1.0/tests/test_server.py +99 -0
  30. mudm_mcp-0.1.0/tests/test_spatial.py +109 -0
  31. mudm_mcp-0.1.0/tests/test_tools.py +144 -0
  32. mudm_mcp-0.1.0/tests/test_units.py +45 -0
mudm_mcp-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Novagen Research
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,147 @@
1
+ Metadata-Version: 2.4
2
+ Name: mudm-mcp
3
+ Version: 0.1.0
4
+ Summary: Local MCP server to discover, access, and locally analyze the muDM static spatial-data catalog.
5
+ Author: Bengt Ljungquist
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://ngrf.mudm.io
8
+ Project-URL: Repository, https://github.com/NovagenResearch/mudm-mcp
9
+ Project-URL: Issues, https://github.com/NovagenResearch/mudm-mcp/issues
10
+ Keywords: mcp,mudm,spatial-data,spatial-transcriptomics,connectomics,llm,agents,duckdb,parquet
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Intended Audience :: Science/Research
13
+ Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.11
16
+ Classifier: Programming Language :: Python :: 3.12
17
+ Classifier: Programming Language :: Python :: 3.13
18
+ Classifier: Programming Language :: Python :: 3.14
19
+ Requires-Python: <3.15,>=3.11
20
+ Description-Content-Type: text/markdown
21
+ License-File: LICENSE
22
+ Requires-Dist: mcp<3,>=2.3
23
+ Requires-Dist: duckdb<3,>=1.4
24
+ Requires-Dist: pyarrow>=16.0
25
+ Requires-Dist: httpx>=0.27
26
+ Requires-Dist: numpy>=2.4.6
27
+ Requires-Dist: brotli>=1.2.0
28
+ Requires-Dist: certifi>=2024.2.2
29
+ Dynamic: license-file
30
+
31
+ # mudm-mcp
32
+
33
+ Ask an AI assistant about the [muDM](https://ngrf.mudm.io) spatial-data catalog: about 100 public datasets of
34
+ spatial transcriptomics and proteomics (Xenium, CODEX, CosMx, MERSCOPE, MIBI, Visium), *Drosophila* EM connectomes
35
+ and 3D reference anatomy.
36
+
37
+ `mudm-mcp` is a small [Model Context Protocol](https://modelcontextprotocol.io) server that runs **on your own
38
+ machine**. Your assistant (Claude Code, Claude Desktop, Cursor, …) starts it, and through it can find datasets,
39
+ describe them, query per-cell and per-neuron attributes, count cells in a region, and download data for local
40
+ analysis. There is no account, no API key and no hosted backend: it reads the public static catalog.
41
+
42
+ ## Quickstart
43
+
44
+ You need [uv](https://docs.astral.sh/uv/getting-started/installation/).
45
+
46
+ **Claude Code**
47
+
48
+ ```bash
49
+ claude mcp add mudm -- uvx mudm-mcp
50
+ ```
51
+
52
+ Start a new session and type `/mcp`: `mudm` should be listed as connected.
53
+
54
+ **Claude Desktop** — add this to `claude_desktop_config.json` (Settings → Developer → Edit Config) and restart:
55
+
56
+ ```json
57
+ {
58
+ "mcpServers": {
59
+ "mudm": { "command": "uvx", "args": ["mudm-mcp"] }
60
+ }
61
+ }
62
+ ```
63
+
64
+ If Claude Desktop reports that it cannot find `uvx`, give its full path instead (run `which uvx`; typically
65
+ `/Users/you/.local/bin/uvx`).
66
+
67
+ **Any other MCP host** — register a stdio server with the command `uvx mudm-mcp`.
68
+
69
+ **From a source checkout** — no PyPI needed; use this to run an unreleased version. Clone this repository, then
70
+ give the host the checkout's path instead of `uvx mudm-mcp`:
71
+
72
+ ```bash
73
+ claude mcp add mudm -- uv run --directory /path/to/mudm-mcp mudm-mcp
74
+ ```
75
+
76
+ The first launch downloads the Python dependencies (about 0.5 GB into uv's cache, a few seconds on a fast
77
+ connection); later launches start in under a second. The first `query_facets` call also fetches DuckDB's
78
+ `httpfs` extension (about 20 MB) from DuckDB's extension server.
79
+
80
+ Then ask, for example:
81
+
82
+ - *"Find a mouse brain spatial transcriptomics dataset in muDM and describe it."*
83
+ - *"How many cells are in the 500 µm square from (2000, 2000) to (2500, 2500) of that section?"*
84
+ - *"How many Kenyon cells are in the hemibrain connectome, by type?"*
85
+ - *"Which genes have the most transcripts in the Xenium mouse brain coronal section?"*
86
+
87
+ [`docs/demo.md`](docs/demo.md) is a ten-minute walk-through of these prompts with the answers to expect.
88
+
89
+ To remove the server again: `claude mcp remove mudm` (or delete the entry from the Claude Desktop config).
90
+
91
+ ## Tools
92
+
93
+ | Tool | What it does | Downloads data? |
94
+ |---|---|---|
95
+ | `search_datasets` | Find datasets by free text (`"mouse brain"`) or by modality, organism, technique, tissue | no |
96
+ | `describe_dataset` | What a dataset contains: assay, license, citation, layers, columns, attributes, units, sizes per zoom | no |
97
+ | `query_facets` | SQL over the per-cell / per-neuron attribute table (gene counts, markers, cell types), read in place | no |
98
+ | `count_features` | Count cells inside a rectangle given in micrometres, optionally grouped by a tag (2D datasets) | a few MB, once |
99
+ | `get_access` | URLs and a decode snippet for reading the data directly, outside this server | no |
100
+ | `download` | Fetch one zoom level of the feature Parquet into the local cache | yes (size-gated) |
101
+ | `query` | SQL (DuckDB) over the downloaded features | no, but needs `download` first |
102
+ | `sample_patches` | Sample tiles as spatial patches, with per-feature geometry summaries (for ML) | no, but needs `download` first |
103
+ | `get_lod` | Summarise one level of detail of the downloaded data | no, but needs `download` first |
104
+
105
+ ## Things worth knowing
106
+
107
+ **Units.** Datasets store their vertices in different units: Xenium in image pixels (0.2125 µm each), the other
108
+ 2D platforms in micrometres, the connectomes in nanometres, the reference anatomy in metres. `describe_dataset`
109
+ reports the stored unit and the factor to micrometres, and every tool argument or result named `*_um` is in
110
+ micrometres, already converted.
111
+
112
+ **Geometry is not GeoParquet.** Feature geometry is muDM binary tile geometry (packed float32 vertices), so SQL
113
+ cannot filter on coordinates. Use `count_features` for regions and `sample_patches` for geometry;
114
+ `get_access` returns a three-line Python snippet for decoding it yourself.
115
+
116
+ **Downloads are size-gated.** A download estimated above 500 MB is not started; the tool reports the size and
117
+ cheaper options, and proceeds only with `confirm_large=true`. (The hemibrain at full resolution is about 300 GB.)
118
+ Most questions need no download at all: `query_facets` reads the remote file in place.
119
+
120
+ **SQL is sandboxed and bounded.** `query` and `query_facets` accept one read-only `SELECT`. The DuckDB connection
121
+ can read only the dataset it was opened for: no other files or URLs, no writes. Results are capped (200 rows by
122
+ default) and flagged `truncated` when more exist.
123
+
124
+ **Cache.** Downloads and catalog metadata live in `~/.cache/mudm-mcp/` (override with `MUDM_CACHE_DIR` or
125
+ `--cache-dir`). Catalog metadata refreshes after 24 hours. Delete the directory to reclaim space.
126
+
127
+ **Another catalog.** `--base-url` (or `MUDM_BASE_URL`) points the server at a different muDM static catalog,
128
+ including a local build directory.
129
+
130
+ ## Development
131
+
132
+ ```bash
133
+ uv sync
134
+ uv run pytest tests/ -x --tb=short # offline tests
135
+ uv run pytest -m live # contract tests against the live catalog
136
+ uv run python scripts/host_smoke.py # drive the server over stdio, end to end
137
+ ```
138
+
139
+ ## Part of the muDM family
140
+
141
+ - [`mudm`](https://github.com/NovagenResearch/mudm) — the data model
142
+ - [`mudm-tools`](https://github.com/NovagenResearch/mudm-tools) — the tiling/conversion engine and viewers
143
+ - `mudm-data` — produces and serves the static catalog this client reads
144
+
145
+ ## License
146
+
147
+ MIT. The datasets carry their own licenses and citations; `describe_dataset` reports them.
@@ -0,0 +1,117 @@
1
+ # mudm-mcp
2
+
3
+ Ask an AI assistant about the [muDM](https://ngrf.mudm.io) spatial-data catalog: about 100 public datasets of
4
+ spatial transcriptomics and proteomics (Xenium, CODEX, CosMx, MERSCOPE, MIBI, Visium), *Drosophila* EM connectomes
5
+ and 3D reference anatomy.
6
+
7
+ `mudm-mcp` is a small [Model Context Protocol](https://modelcontextprotocol.io) server that runs **on your own
8
+ machine**. Your assistant (Claude Code, Claude Desktop, Cursor, …) starts it, and through it can find datasets,
9
+ describe them, query per-cell and per-neuron attributes, count cells in a region, and download data for local
10
+ analysis. There is no account, no API key and no hosted backend: it reads the public static catalog.
11
+
12
+ ## Quickstart
13
+
14
+ You need [uv](https://docs.astral.sh/uv/getting-started/installation/).
15
+
16
+ **Claude Code**
17
+
18
+ ```bash
19
+ claude mcp add mudm -- uvx mudm-mcp
20
+ ```
21
+
22
+ Start a new session and type `/mcp`: `mudm` should be listed as connected.
23
+
24
+ **Claude Desktop** — add this to `claude_desktop_config.json` (Settings → Developer → Edit Config) and restart:
25
+
26
+ ```json
27
+ {
28
+ "mcpServers": {
29
+ "mudm": { "command": "uvx", "args": ["mudm-mcp"] }
30
+ }
31
+ }
32
+ ```
33
+
34
+ If Claude Desktop reports that it cannot find `uvx`, give its full path instead (run `which uvx`; typically
35
+ `/Users/you/.local/bin/uvx`).
36
+
37
+ **Any other MCP host** — register a stdio server with the command `uvx mudm-mcp`.
38
+
39
+ **From a source checkout** — no PyPI needed; use this to run an unreleased version. Clone this repository, then
40
+ give the host the checkout's path instead of `uvx mudm-mcp`:
41
+
42
+ ```bash
43
+ claude mcp add mudm -- uv run --directory /path/to/mudm-mcp mudm-mcp
44
+ ```
45
+
46
+ The first launch downloads the Python dependencies (about 0.5 GB into uv's cache, a few seconds on a fast
47
+ connection); later launches start in under a second. The first `query_facets` call also fetches DuckDB's
48
+ `httpfs` extension (about 20 MB) from DuckDB's extension server.
49
+
50
+ Then ask, for example:
51
+
52
+ - *"Find a mouse brain spatial transcriptomics dataset in muDM and describe it."*
53
+ - *"How many cells are in the 500 µm square from (2000, 2000) to (2500, 2500) of that section?"*
54
+ - *"How many Kenyon cells are in the hemibrain connectome, by type?"*
55
+ - *"Which genes have the most transcripts in the Xenium mouse brain coronal section?"*
56
+
57
+ [`docs/demo.md`](docs/demo.md) is a ten-minute walk-through of these prompts with the answers to expect.
58
+
59
+ To remove the server again: `claude mcp remove mudm` (or delete the entry from the Claude Desktop config).
60
+
61
+ ## Tools
62
+
63
+ | Tool | What it does | Downloads data? |
64
+ |---|---|---|
65
+ | `search_datasets` | Find datasets by free text (`"mouse brain"`) or by modality, organism, technique, tissue | no |
66
+ | `describe_dataset` | What a dataset contains: assay, license, citation, layers, columns, attributes, units, sizes per zoom | no |
67
+ | `query_facets` | SQL over the per-cell / per-neuron attribute table (gene counts, markers, cell types), read in place | no |
68
+ | `count_features` | Count cells inside a rectangle given in micrometres, optionally grouped by a tag (2D datasets) | a few MB, once |
69
+ | `get_access` | URLs and a decode snippet for reading the data directly, outside this server | no |
70
+ | `download` | Fetch one zoom level of the feature Parquet into the local cache | yes (size-gated) |
71
+ | `query` | SQL (DuckDB) over the downloaded features | no, but needs `download` first |
72
+ | `sample_patches` | Sample tiles as spatial patches, with per-feature geometry summaries (for ML) | no, but needs `download` first |
73
+ | `get_lod` | Summarise one level of detail of the downloaded data | no, but needs `download` first |
74
+
75
+ ## Things worth knowing
76
+
77
+ **Units.** Datasets store their vertices in different units: Xenium in image pixels (0.2125 µm each), the other
78
+ 2D platforms in micrometres, the connectomes in nanometres, the reference anatomy in metres. `describe_dataset`
79
+ reports the stored unit and the factor to micrometres, and every tool argument or result named `*_um` is in
80
+ micrometres, already converted.
81
+
82
+ **Geometry is not GeoParquet.** Feature geometry is muDM binary tile geometry (packed float32 vertices), so SQL
83
+ cannot filter on coordinates. Use `count_features` for regions and `sample_patches` for geometry;
84
+ `get_access` returns a three-line Python snippet for decoding it yourself.
85
+
86
+ **Downloads are size-gated.** A download estimated above 500 MB is not started; the tool reports the size and
87
+ cheaper options, and proceeds only with `confirm_large=true`. (The hemibrain at full resolution is about 300 GB.)
88
+ Most questions need no download at all: `query_facets` reads the remote file in place.
89
+
90
+ **SQL is sandboxed and bounded.** `query` and `query_facets` accept one read-only `SELECT`. The DuckDB connection
91
+ can read only the dataset it was opened for: no other files or URLs, no writes. Results are capped (200 rows by
92
+ default) and flagged `truncated` when more exist.
93
+
94
+ **Cache.** Downloads and catalog metadata live in `~/.cache/mudm-mcp/` (override with `MUDM_CACHE_DIR` or
95
+ `--cache-dir`). Catalog metadata refreshes after 24 hours. Delete the directory to reclaim space.
96
+
97
+ **Another catalog.** `--base-url` (or `MUDM_BASE_URL`) points the server at a different muDM static catalog,
98
+ including a local build directory.
99
+
100
+ ## Development
101
+
102
+ ```bash
103
+ uv sync
104
+ uv run pytest tests/ -x --tb=short # offline tests
105
+ uv run pytest -m live # contract tests against the live catalog
106
+ uv run python scripts/host_smoke.py # drive the server over stdio, end to end
107
+ ```
108
+
109
+ ## Part of the muDM family
110
+
111
+ - [`mudm`](https://github.com/NovagenResearch/mudm) — the data model
112
+ - [`mudm-tools`](https://github.com/NovagenResearch/mudm-tools) — the tiling/conversion engine and viewers
113
+ - `mudm-data` — produces and serves the static catalog this client reads
114
+
115
+ ## License
116
+
117
+ MIT. The datasets carry their own licenses and citations; `describe_dataset` reports them.
@@ -0,0 +1,53 @@
1
+ [build-system]
2
+ requires = ["setuptools>=77.0"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "mudm-mcp"
7
+ version = "0.1.0"
8
+ description = "Local MCP server to discover, access, and locally analyze the muDM static spatial-data catalog."
9
+ readme = "README.md"
10
+ authors = [{ name = "Bengt Ljungquist" }]
11
+ license = "MIT"
12
+ license-files = ["LICENSE"]
13
+ requires-python = ">=3.11,<3.15"
14
+ keywords = ["mcp", "mudm", "spatial-data", "spatial-transcriptomics", "connectomics", "llm", "agents", "duckdb", "parquet"]
15
+ classifiers = [
16
+ "Development Status :: 4 - Beta",
17
+ "Intended Audience :: Science/Research",
18
+ "Topic :: Scientific/Engineering :: Bio-Informatics",
19
+ "Programming Language :: Python :: 3",
20
+ "Programming Language :: Python :: 3.11",
21
+ "Programming Language :: Python :: 3.12",
22
+ "Programming Language :: Python :: 3.13",
23
+ "Programming Language :: Python :: 3.14",
24
+ ]
25
+ dependencies = [
26
+ "mcp>=2.3,<3", # official Model Context Protocol Python SDK (stdio server)
27
+ "duckdb>=1.4,<3", # local query engine over downloaded tiles
28
+ "pyarrow>=16.0", # read muDM Parquet locally; zero-copy into NumPy/Torch
29
+ "httpx>=0.27", # fetch the static catalog + tiles (download / optional range reads)
30
+ "numpy>=2.4.6",
31
+ "brotli>=1.2.0", # the catalog's JSON is served brotli-encoded
32
+ "certifi>=2024.2.2",
33
+ ]
34
+
35
+ [project.urls]
36
+ Homepage = "https://ngrf.mudm.io"
37
+ Repository = "https://github.com/NovagenResearch/mudm-mcp"
38
+ Issues = "https://github.com/NovagenResearch/mudm-mcp/issues"
39
+
40
+ [project.scripts]
41
+ mudm-mcp = "mudm_mcp.server:main"
42
+
43
+ [dependency-groups]
44
+ dev = ["pytest>=8"]
45
+
46
+ [tool.setuptools.packages.find]
47
+ where = ["src"]
48
+
49
+ [tool.pytest.ini_options]
50
+ testpaths = ["tests/"]
51
+ pythonpath = ["src/"]
52
+ markers = ["live: needs network access to ngrf.mudm.io"]
53
+ addopts = "-m 'not live'"
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,4 @@
1
+ """muDM MCP — a local MCP server (client of the static muDM catalog) for agent/LLM discovery,
2
+ access, and local analysis of muDM spatial datasets."""
3
+
4
+ __version__ = "0.1.0"
@@ -0,0 +1,236 @@
1
+ """Read the static muDM catalog: a CLIENT of `pyramids.json` (mudm.PyramidJSON) + per-dataset
2
+ `tilejson.json` (mudm.TileModel) + derived `croissant.json`. No backend — fetches static files over
3
+ http(s) (cached to a local dir) or reads a local build dir (`file://` / plain path) for dev/tests.
4
+ """
5
+ from __future__ import annotations
6
+
7
+ import difflib
8
+ import json
9
+ import os
10
+ import time
11
+ from pathlib import Path
12
+ from urllib.parse import urlparse
13
+
14
+ DEFAULT_JSON_TTL_S = 24 * 3600.0
15
+
16
+
17
+ def default_cache_dir(host: str) -> Path:
18
+ return Path.home() / ".cache" / "mudm-mcp" / (host or "local")
19
+
20
+
21
+ class DatasetNotFound(KeyError):
22
+ """Unknown dataset id; ``suggestions`` holds up to 5 close catalog ids."""
23
+
24
+ def __init__(self, dataset_id: str, known_ids: list[str]):
25
+ super().__init__(f"dataset {dataset_id!r} not found in pyramids.json")
26
+ self.dataset_id = dataset_id
27
+ self.suggestions = difflib.get_close_matches(dataset_id, known_ids, n=5, cutoff=0.5)
28
+
29
+
30
+ def _zoom_of(part: str) -> int:
31
+ """'zoom=9/cells_part_000.mu.parquet' -> 9."""
32
+ return int(part.split("/", 1)[0].split("=", 1)[1])
33
+
34
+
35
+ class CatalogClient:
36
+ """Resolve + cache the catalog artifacts. ``base_url`` may be an http(s) origin (default
37
+ ``https://ngrf.mudm.io``) or a local directory (a `build_site --out` dir, as `file://…` or a path).
38
+
39
+ Cached catalog JSON is refetched once it is older than ``json_ttl_s`` (default 24 h), so a long-lived
40
+ install follows catalog updates; if the refetch fails (offline) the stale copy is served."""
41
+
42
+ def __init__(self, base_url: str, cache_dir: str | Path | None = None,
43
+ json_ttl_s: float = DEFAULT_JSON_TTL_S):
44
+ self.base_url = str(base_url).rstrip("/")
45
+ self.json_ttl_s = float(json_ttl_s)
46
+ parsed = urlparse(self.base_url)
47
+ self.is_http = parsed.scheme in ("http", "https")
48
+ if self.is_http:
49
+ self.cache_dir = Path(cache_dir) if cache_dir else default_cache_dir(parsed.netloc)
50
+ self._local_root: Path | None = None
51
+ else: # file:// or a plain local path
52
+ self._local_root = Path(parsed.path if parsed.scheme == "file" else self.base_url)
53
+ self.cache_dir = Path(cache_dir) if cache_dir else self._local_root
54
+
55
+ # --- paths ---
56
+ @staticmethod
57
+ def _rel(parts) -> str:
58
+ return "/".join(str(p).strip("/") for p in parts if str(p).strip("/"))
59
+
60
+ def url(self, *parts: str) -> str:
61
+ """Absolute URL (http origin) or local path (local origin) of a site-relative path."""
62
+ rel = self._rel(parts)
63
+ return f"{self.base_url}/{rel}" if self.is_http else str(self._local_root / rel)
64
+
65
+ # --- fetch / cache ---
66
+ def fetch(self, *parts: str) -> bytes:
67
+ """Bytes of a catalog-relative path (e.g. ``fetch("datasets", id, "tilejson.json")``)."""
68
+ rel = self._rel(parts)
69
+ if not self.is_http:
70
+ return (self._local_root / rel).read_bytes()
71
+ cached = self.cache_dir / rel
72
+ have = cached.is_file()
73
+ if have and (time.time() - cached.stat().st_mtime) < self.json_ttl_s:
74
+ return cached.read_bytes()
75
+ import httpx # lazy: file:// dev/tests need no network/httpx
76
+ try:
77
+ resp = httpx.get(f"{self.base_url}/{rel}", follow_redirects=True, timeout=60.0)
78
+ resp.raise_for_status()
79
+ except httpx.HTTPError:
80
+ if have: # expired but unreachable: a stale catalog beats no catalog
81
+ return cached.read_bytes()
82
+ raise
83
+ cached.parent.mkdir(parents=True, exist_ok=True)
84
+ tmp = cached.with_name(cached.name + ".part")
85
+ tmp.write_bytes(resp.content)
86
+ os.replace(tmp, cached)
87
+ return resp.content
88
+
89
+ def fetch_json(self, *parts: str) -> dict:
90
+ return json.loads(self.fetch(*parts))
91
+
92
+ def _try_fetch_json(self, *parts: str) -> dict | None:
93
+ """``fetch_json`` that returns None when the file does not exist (local miss / HTTP 4xx)."""
94
+ try:
95
+ return self.fetch_json(*parts)
96
+ except FileNotFoundError:
97
+ return None
98
+ except Exception as exc: # httpx.HTTPStatusError without importing httpx eagerly
99
+ status = getattr(getattr(exc, "response", None), "status_code", None)
100
+ if status is not None and 400 <= status < 500: # S3/CloudFront answer 403 for a missing key
101
+ return None
102
+ raise
103
+
104
+ def download_file(self, rel_parts, out_path) -> int:
105
+ """Stream a (possibly large) leaf file to ``out_path`` (http GET or local copy); returns bytes
106
+ written. Atomic: the data goes to ``<out_path>.part`` and is renamed only when complete, so an
107
+ interrupted download never leaves a truncated file under the final name. Unlike ``fetch`` this
108
+ does not hold the body in memory or use the JSON cache (parquet leaves are served uncompressed)."""
109
+ rel = self._rel(rel_parts)
110
+ out = Path(out_path)
111
+ out.parent.mkdir(parents=True, exist_ok=True)
112
+ tmp = out.with_name(out.name + ".part")
113
+ try:
114
+ if not self.is_http:
115
+ with open(self._local_root / rel, "rb") as src, open(tmp, "wb") as fh:
116
+ while chunk := src.read(1 << 20):
117
+ fh.write(chunk)
118
+ else:
119
+ import httpx # lazy
120
+ with httpx.stream("GET", f"{self.base_url}/{rel}", follow_redirects=True,
121
+ timeout=120.0) as resp:
122
+ resp.raise_for_status()
123
+ with open(tmp, "wb") as fh:
124
+ for chunk in resp.iter_bytes():
125
+ fh.write(chunk)
126
+ size = tmp.stat().st_size
127
+ os.replace(tmp, out)
128
+ except BaseException:
129
+ tmp.unlink(missing_ok=True)
130
+ raise
131
+ return size
132
+
133
+ # --- sizes ---
134
+ def part_size(self, base: str, part: str) -> int | None:
135
+ """Size in bytes of one leaf (local ``stat``; HTTP ``HEAD``, else a 1-byte ranged GET). None if
136
+ it cannot be determined."""
137
+ rel = self._rel([base, part])
138
+ if not self.is_http:
139
+ p = self._local_root / rel
140
+ return p.stat().st_size if p.is_file() else None
141
+ import httpx # lazy
142
+ url = f"{self.base_url}/{rel}"
143
+ try:
144
+ r = httpx.head(url, follow_redirects=True, timeout=30.0)
145
+ if r.status_code == 200 and "content-length" in r.headers:
146
+ return int(r.headers["content-length"])
147
+ r = httpx.get(url, headers={"Range": "bytes=0-0"}, follow_redirects=True, timeout=30.0)
148
+ total = r.headers.get("content-range", "").rsplit("/", 1)[-1]
149
+ return int(total) if total.isdigit() else None
150
+ except httpx.HTTPError:
151
+ return None
152
+
153
+ def estimate_bytes(self, base: str, parts: list[str], sample: int = 4) -> int | None:
154
+ """Total size of ``parts``: exact when there are at most ``sample`` of them, else the mean of
155
+ ``sample`` evenly spaced parts times the count. None if no size could be read."""
156
+ if not parts:
157
+ return 0
158
+ if len(parts) <= sample:
159
+ sizes = [self.part_size(base, p) for p in parts]
160
+ return None if any(s is None for s in sizes) else sum(sizes)
161
+ step = len(parts) / sample
162
+ sizes = [s for s in (self.part_size(base, parts[int(i * step)]) for i in range(sample)) if s is not None]
163
+ return round(sum(sizes) / len(sizes) * len(parts)) if sizes else None
164
+
165
+ # --- catalog accessors ---
166
+ def pyramids(self) -> dict:
167
+ """The unified 2D+3D catalog (mudm.PyramidJSON)."""
168
+ return self.fetch_json("pyramids.json")
169
+
170
+ def entries(self) -> list[dict]:
171
+ return self.pyramids().get("pyramids", [])
172
+
173
+ def entry(self, dataset_id: str) -> dict:
174
+ """The catalog entry (PyramidEntry: id/label/kind + OPEN facet metadata) for one dataset."""
175
+ entries = self.entries()
176
+ for e in entries:
177
+ if e.get("id") == dataset_id:
178
+ return e
179
+ raise DatasetNotFound(dataset_id, [str(e.get("id")) for e in entries])
180
+
181
+ def descriptor(self, dataset_id: str) -> dict:
182
+ """The per-dataset muDM TileModel descriptor (assets[] + TileLayer schema + multiscale coords)."""
183
+ return self.fetch_json("datasets", dataset_id, "tilejson.json")
184
+
185
+ def croissant(self, dataset_id: str) -> dict:
186
+ """The derived Croissant JSON-LD (schema.org/Dataset + cr:)."""
187
+ return self.fetch_json("datasets", dataset_id, "croissant.json")
188
+
189
+ def viewer_descriptor(self, dataset_id: str) -> tuple[str, dict] | None:
190
+ """The VIEWER's descriptor ``(<site-relative dir>, tilejson)`` that sits next to the tiles
191
+ (``tiles/<id>/`` for 3D, ``tiles2d/<id>/`` for 2D). Its asset hrefs are relative to that dir. Used
192
+ as a fallback when the agent descriptor (``datasets/<id>/tilejson.json``) omits an asset."""
193
+ entry = self.entry(dataset_id)
194
+ feats = (entry.get("features") or "").strip("/")
195
+ dirs = [feats.rsplit("/", 1)[0]] if "/" in feats else []
196
+ dirs += [d for d in (f"tiles/{dataset_id}", f"tiles2d/{dataset_id}") if d not in dirs]
197
+ for d in dirs:
198
+ doc = self._try_fetch_json(d, "tilejson.json")
199
+ if doc is not None:
200
+ return d, doc
201
+ return None
202
+
203
+ @staticmethod
204
+ def _asset(descriptor: dict, role: str) -> dict | None:
205
+ return next((a for a in descriptor.get("assets") or [] if a.get("role") == role), None)
206
+
207
+ def features(self, dataset_id: str) -> dict | None:
208
+ """The download index of a dataset's ``features.parquet``:
209
+ ``{base, parts, zooms, full_res_zoom, source}`` — ``base`` is the site-relative directory holding
210
+ the ``zoom=<z>/…`` leaves, ``source`` is ``"agent"`` or ``"viewer"`` (which descriptor carried it).
211
+ None when neither descriptor lists parquet parts."""
212
+ entry = self.entry(dataset_id)
213
+ entry_base = (entry.get("features") or "").strip("/")
214
+ asset = self._asset(self.descriptor(dataset_id), "features")
215
+ source, viewer_dir = "agent", None
216
+ if not (asset and asset.get("parts")):
217
+ found = self.viewer_descriptor(dataset_id)
218
+ asset = self._asset(found[1], "features") if found else None
219
+ if not (asset and asset.get("parts")):
220
+ return None
221
+ source, viewer_dir = "viewer", found[0]
222
+ href = str(asset.get("href") or "")
223
+ if entry_base:
224
+ base = entry_base
225
+ elif viewer_dir is not None:
226
+ base = self._rel([viewer_dir, href or "features.parquet"])
227
+ elif href.startswith(self.base_url + "/"):
228
+ base = href[len(self.base_url) + 1:].strip("/")
229
+ else:
230
+ raise ValueError(f"{dataset_id}: cannot locate features.parquet (href {asset.get('href')!r} is "
231
+ f"outside {self.base_url})")
232
+ parts = list(asset["parts"])
233
+ zooms = sorted(int(z) for z in (asset.get("zooms") or {_zoom_of(p) for p in parts}))
234
+ full = asset.get("full_res_zoom")
235
+ return {"base": base, "parts": parts, "zooms": zooms,
236
+ "full_res_zoom": int(full) if full is not None else zooms[-1], "source": source}
@@ -0,0 +1,68 @@
1
+ """query_facets — SQL over a dataset's facet store (a per-cell / per-neuron attribute table published
2
+ as ONE parquet file next to the tiles), read in place over HTTP range requests: no download.
3
+
4
+ Facet stores by assay: Xenium ``expression`` (long: cell_id, gene, count) · CODEX / CosMx / MERSCOPE /
5
+ MIBI ``markers`` (wide: one column per marker or gene) · connectomes and reference anatomy ``properties``
6
+ (wide: one row per neuron / structure with cell_type, synapse counts, …). Visium has none.
7
+ """
8
+ from __future__ import annotations
9
+
10
+ from .catalog_client import CatalogClient
11
+ from .sandbox import open_sandbox, run_select
12
+ from .tools import facet_assets
13
+
14
+ MAX_SCHEMA_COLUMNS = 60
15
+
16
+
17
+ def _names(cols: list[str], cap: int = MAX_SCHEMA_COLUMNS) -> str:
18
+ more = f", … (+{len(cols) - cap} more)" if len(cols) > cap else ""
19
+ return ", ".join(cols[:cap]) + more
20
+
21
+
22
+ def query_facets(client: CatalogClient, dataset_id: str, sql: str, facet: str | None = None,
23
+ max_rows: int = 200) -> dict:
24
+ """Run ONE read-only SELECT against a dataset's facet store, exposed as the view ``facets`` (the
25
+ store's own name, e.g. ``expression`` or ``properties``, works as an alias).
26
+
27
+ ``facet`` picks the store by name when a dataset has several. Returns the bounded result of
28
+ ``run_select`` plus ``facet``, ``source`` (the file's URL) and ``schema`` — the file's REAL columns,
29
+ which can differ from what the descriptor advertises (``missing_columns`` lists advertised-but-absent
30
+ ones)."""
31
+ import duckdb # lazy
32
+
33
+ client.entry(dataset_id) # unknown id -> DatasetNotFound with suggestions
34
+ stores = facet_assets(client, dataset_id)
35
+ if not stores:
36
+ raise ValueError(f"{dataset_id} has no facet store (Visium datasets have none). "
37
+ "describe_dataset lists a dataset's facets; download + query reads the features.")
38
+ names = [s["name"] for s in stores]
39
+ if facet:
40
+ store = next((s for s in stores if s["name"] == facet), None)
41
+ if store is None:
42
+ raise ValueError(f"{dataset_id} has no facet store named {facet!r}; available: {', '.join(names)}")
43
+ elif len(stores) > 1:
44
+ raise ValueError(f"{dataset_id} has several facet stores; pass facet= one of: {', '.join(names)}")
45
+ else:
46
+ store = stores[0]
47
+
48
+ con = open_sandbox("facets", store["url"])
49
+ try:
50
+ # Agents often write FROM <store name> (e.g. FROM expression) because that is the name
51
+ # describe_dataset shows: accept it as an alias of the `facets` view instead of failing.
52
+ if store["name"].isidentifier() and store["name"].lower() != "facets":
53
+ con.execute(f'CREATE VIEW "{store["name"]}" AS SELECT * FROM facets')
54
+ schema = [{"name": r[0], "type": r[1]} for r in con.execute("DESCRIBE facets").fetchall()]
55
+ cols = [c["name"] for c in schema]
56
+ try:
57
+ result = run_select(con, sql, max_rows=max_rows)
58
+ except (duckdb.BinderException, duckdb.CatalogException) as exc:
59
+ first = str(exc).split("\n", 1)[0]
60
+ raise ValueError(f"{first} The view is named `facets`. Available columns: {_names(cols)}") from exc
61
+ finally:
62
+ con.close()
63
+ out = {**result, "facet": store["name"], "layout": store["layout"], "source": store["url"],
64
+ "n_columns": len(schema), "schema": schema[:MAX_SCHEMA_COLUMNS]}
65
+ missing = [c for c in store["columns"] if c not in cols]
66
+ if missing:
67
+ out["missing_columns"] = missing[:MAX_SCHEMA_COLUMNS]
68
+ return out