aind-ibl-ephys-alignment-preprocessing 0.8.0__tar.gz → 0.8.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/CHANGELOG.md +12 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/CONTRIBUTING.md +5 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/PKG-INFO +2 -2
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/pyproject.toml +2 -2
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/discovery.py +29 -16
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/ephys.py +5 -1
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/histology.py +3 -4
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/stages.py +13 -18
- aind_ibl_ephys_alignment_preprocessing-0.8.2/tests/fixtures/README.md +17 -0
- aind_ibl_ephys_alignment_preprocessing-0.8.2/tests/fixtures/nested_image_sources.json +20 -0
- aind_ibl_ephys_alignment_preprocessing-0.8.2/tests/fixtures/transformed_image_sources.json +87 -0
- aind_ibl_ephys_alignment_preprocessing-0.8.2/tests/test_image_discovery.py +32 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/tests/test_registration_override.py +0 -2
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/tests/test_sorting_detection.py +29 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/tests/test_stages.py +26 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/uv.lock +5 -5
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/.copier-answers.yml +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/.github/ISSUE_TEMPLATE/user-story.md +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/.github/dependabot.yml +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/.github/workflows/bump-call.yml +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/.github/workflows/ci-call.yml +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/.github/workflows/docs-ci.yml +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/.github/workflows/publish-call.yml +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/.gitignore +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/.python-version +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/.readthedocs.yaml +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/AGENTS.md +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/CLAUDE.md +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/LICENSE +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/README.md +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/docs/external_asset_references.md +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/docs/source/_static/dark-logo.svg +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/docs/source/_static/favicon.ico +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/docs/source/_static/light-logo.svg +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/docs/source/api.rst +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/docs/source/conf.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/docs/source/data_contracts.md +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/docs/source/index.md +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/scripts/run_linters_and_checks.sh +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/__init__.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/_async/__init__.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/_async/concurrency.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/_async/ephys.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/_async/histology.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/_async/pipeline.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/_async/probes.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/_constants.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/_timing.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/coverage.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/datapackage.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/datapackage_schema.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/manifest_builder.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/pipeline.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/probes.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/py.typed +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/registration_frame.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/schemas/aind-ibl-ephys-alignment-datapackage/3.1.0/datapackage.schema.json +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/schemas/aind-ibl-ephys-alignment-datapackage/3.2.0/datapackage.schema.json +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/schemas/aind-ibl-ephys-alignment-datapackage/4.0.0/datapackage.schema.json +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/schemas/aind-ibl-ephys-alignment-datapackage/4.1.0/datapackage.schema.json +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/scripts/__init__.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/scripts/build_manifest.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/scripts/export_datapackage_schema.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/scripts/run.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/types.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/src/aind_ibl_ephys_alignment_preprocessing/validation.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/tests/__init__.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/tests/sidecar_fixtures.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/tests/test_ants_domain.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/tests/test_coverage.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/tests/test_datapackage.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/tests/test_datapackage_schema.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/tests/test_example.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/tests/test_geometry_sidecar.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/tests/test_manifest_builder.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/tests/test_registration_frame.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/tests/test_resample.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/tests/test_sync_async_parity.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/tests/test_template_to_ccf_override.py +0 -0
- {aind_ibl_ephys_alignment_preprocessing-0.8.0 → aind_ibl_ephys_alignment_preprocessing-0.8.2}/tests/test_timing.py +0 -0
|
@@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## v0.8.2 (2026-10-26)
|
|
9
|
+
|
|
10
|
+
### Fix
|
|
11
|
+
|
|
12
|
+
- **deps**: require aind-zarr-utils 0.18 for level-correct origins
|
|
13
|
+
|
|
14
|
+
## v0.8.1 (2026-10-24)
|
|
15
|
+
|
|
16
|
+
### Fix
|
|
17
|
+
|
|
18
|
+
- discover mounted inputs from paths and metadata
|
|
19
|
+
|
|
8
20
|
## v0.8.0 (2026-10-23)
|
|
9
21
|
|
|
10
22
|
### BREAKING CHANGE
|
|
@@ -96,6 +96,11 @@ src/aind_ibl_ephys_alignment_preprocessing/
|
|
|
96
96
|
- **Path resolution**: `PipelineConfig` is a frozen Pydantic v2 model with a
|
|
97
97
|
`@model_validator(mode="after")` that resolves all relative paths against
|
|
98
98
|
`data_root`.
|
|
99
|
+
- **Discovery**: Image sources identify the asset directory before
|
|
100
|
+
`image_tile_fusing/OMEZarr`, including nested S3 prefixes; asset names need no
|
|
101
|
+
modality prefix. Mounted sortings match the manifest's exact pin first, with
|
|
102
|
+
input-recording metadata as a fallback for renamed captures. Raw extraction
|
|
103
|
+
uses the sorting's input metadata to distinguish main and surface recordings.
|
|
99
104
|
- **Frozen data structures**: All dataclasses and Pydantic models are frozen.
|
|
100
105
|
Use `dataclasses.replace()` or `.model_copy(update=...)` to create modified
|
|
101
106
|
copies.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: aind-ibl-ephys-alignment-preprocessing
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.2
|
|
4
4
|
Summary: Library to prepare histology and ephys for the IBL ephys alignment GUI
|
|
5
5
|
Project-URL: Repository, https://github.com/AllenNeuralDynamics/aind-ibl-ephys-alignment-preprocessing/
|
|
6
6
|
Project-URL: Issues, https://github.com/AllenNeuralDynamics/aind-ibl-ephys-alignment-preprocessing/issues
|
|
@@ -17,7 +17,7 @@ Requires-Dist: aind-code-ocean-pipeline-utils>=0.4
|
|
|
17
17
|
Requires-Dist: aind-ephys-ibl-gui-conversion>=0.5.0
|
|
18
18
|
Requires-Dist: aind-registration-utils>=0.4
|
|
19
19
|
Requires-Dist: aind-s3-cache>=0.1
|
|
20
|
-
Requires-Dist: aind-zarr-utils
|
|
20
|
+
Requires-Dist: aind-zarr-utils<0.19,>=0.18
|
|
21
21
|
Requires-Dist: antspyx>=0.3.0
|
|
22
22
|
Requires-Dist: boto3
|
|
23
23
|
Requires-Dist: filelock
|
|
@@ -13,7 +13,7 @@ authors = [
|
|
|
13
13
|
]
|
|
14
14
|
classifiers = ["Programming Language :: Python :: 3"]
|
|
15
15
|
readme = "README.md"
|
|
16
|
-
version = "0.8.
|
|
16
|
+
version = "0.8.2"
|
|
17
17
|
|
|
18
18
|
dependencies = [
|
|
19
19
|
"numpy>=1.21.0",
|
|
@@ -28,7 +28,7 @@ dependencies = [
|
|
|
28
28
|
"aind-ephys-ibl-gui-conversion>=0.5.0",
|
|
29
29
|
"aind-registration-utils>=0.4",
|
|
30
30
|
"aind-s3-cache>=0.1",
|
|
31
|
-
"aind-zarr-utils>=0.
|
|
31
|
+
"aind-zarr-utils>=0.18,<0.19",
|
|
32
32
|
"boto3",
|
|
33
33
|
"filelock",
|
|
34
34
|
]
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import logging
|
|
6
|
-
from pathlib import Path
|
|
6
|
+
from pathlib import Path
|
|
7
7
|
from typing import TYPE_CHECKING
|
|
8
8
|
|
|
9
9
|
import numpy as np
|
|
@@ -11,7 +11,6 @@ from aind_s3_cache.json_utils import get_json
|
|
|
11
11
|
from aind_s3_cache.uri_utils import as_pathlike
|
|
12
12
|
from aind_zarr_utils.neuroglancer import get_image_sources
|
|
13
13
|
from aind_zarr_utils.pipeline_transformed import (
|
|
14
|
-
_asset_from_zarr_pathlike,
|
|
15
14
|
alignment_zarr_uri_and_metadata_from_zarr_or_asset_pathlike,
|
|
16
15
|
pipeline_transforms_local_paths,
|
|
17
16
|
)
|
|
@@ -39,6 +38,25 @@ if TYPE_CHECKING:
|
|
|
39
38
|
logger = logging.getLogger(__name__)
|
|
40
39
|
|
|
41
40
|
|
|
41
|
+
def image_asset_path(data_root: Path, source_uri: str) -> Path:
|
|
42
|
+
"""Locate a mounted image asset from its URI and processing layout.
|
|
43
|
+
|
|
44
|
+
S3 prefixes may have several components. Code Ocean mounts the asset under
|
|
45
|
+
its own directory name, without the bucket's enclosing prefix directories.
|
|
46
|
+
Neither acquisition prefixes nor timestamps identify that directory.
|
|
47
|
+
"""
|
|
48
|
+
kind, _, path = as_pathlike(source_uri)
|
|
49
|
+
for parent in path.parents:
|
|
50
|
+
if parent.name == "image_tile_fusing":
|
|
51
|
+
asset = parent.parent
|
|
52
|
+
break
|
|
53
|
+
else:
|
|
54
|
+
raise ValueError(f"Image source has no image_tile_fusing asset layout: {source_uri!r}")
|
|
55
|
+
if kind != "s3" and asset.is_absolute() and Path(asset).is_dir():
|
|
56
|
+
return Path(asset)
|
|
57
|
+
return data_root / asset.name
|
|
58
|
+
|
|
59
|
+
|
|
42
60
|
#: Manifest columns naming one directory per brain, read off whichever row carries them.
|
|
43
61
|
PER_BRAIN_PATH_COLUMNS = ("registration_asset", "template_to_ccf_asset")
|
|
44
62
|
|
|
@@ -215,9 +233,10 @@ def find_asset_info(config: PipelineConfig) -> AssetInfo:
|
|
|
215
233
|
a_zarr_uri = next(iter(sources.values()), None)
|
|
216
234
|
if a_zarr_uri is None:
|
|
217
235
|
raise ValueError("No image sources found in neuroglancer data")
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
236
|
+
asset_paths = {image_asset_path(config.data_root, uri) for uri in sources.values() if uri}
|
|
237
|
+
if len(asset_paths) != 1:
|
|
238
|
+
raise ValueError(f"Neuroglancer image layers reference multiple assets: {sources!r}")
|
|
239
|
+
asset_path = next(iter(asset_paths))
|
|
221
240
|
if not asset_path.exists():
|
|
222
241
|
raise FileNotFoundError(f"Asset path not found: {asset_path}")
|
|
223
242
|
asset_path_str = str(asset_path)
|
|
@@ -268,7 +287,7 @@ def find_asset_info(config: PipelineConfig) -> AssetInfo:
|
|
|
268
287
|
check_registration_template(registration_dir_path, config.template_25, template_to_ccf_dir)
|
|
269
288
|
return AssetInfo(
|
|
270
289
|
asset_path=asset_path,
|
|
271
|
-
asset_uri=_infer_asset_uri(a_zarr_uri
|
|
290
|
+
asset_uri=_infer_asset_uri(a_zarr_uri),
|
|
272
291
|
zarr_volumes=zarr_paths,
|
|
273
292
|
pipeline_registration_chains=pipeline_reg_info,
|
|
274
293
|
registration_dir_path=registration_dir_path,
|
|
@@ -276,24 +295,18 @@ def find_asset_info(config: PipelineConfig) -> AssetInfo:
|
|
|
276
295
|
)
|
|
277
296
|
|
|
278
297
|
|
|
279
|
-
def _infer_asset_uri(source_uri: str
|
|
298
|
+
def _infer_asset_uri(source_uri: str) -> str | None:
|
|
280
299
|
"""Infer the asset-level URI from a source URI inside that asset.
|
|
281
300
|
|
|
282
301
|
Neuroglancer sources often point inside the stitched SmartSPIM asset, for
|
|
283
302
|
example ``s3://bucket/<asset>/image_tile_fusing/...``. The datapackage
|
|
284
303
|
registry should name the asset that contains ``image_atlas_alignment/``,
|
|
285
|
-
not the nested zarr.
|
|
286
|
-
|
|
304
|
+
not the nested zarr. The processing directory marks the asset boundary,
|
|
305
|
+
including any enclosing S3 prefix directories.
|
|
287
306
|
"""
|
|
288
307
|
if "://" not in source_uri:
|
|
289
308
|
return None
|
|
290
|
-
|
|
291
|
-
if not marker:
|
|
292
|
-
return None
|
|
293
|
-
idx = source_uri.find(marker)
|
|
294
|
-
if idx < 0:
|
|
295
|
-
return None
|
|
296
|
-
return source_uri[: idx + len(marker)]
|
|
309
|
+
return source_uri.split("/image_tile_fusing/", 1)[0]
|
|
297
310
|
|
|
298
311
|
|
|
299
312
|
def prepare_result_dirs(mouse_id: str, results_root: Path) -> OutputDirs:
|
|
@@ -483,6 +483,9 @@ def run_ephys_for_recording(
|
|
|
483
483
|
# so raw and sorted need not be siblings. None -> converter falls back to its
|
|
484
484
|
# legacy sibling-split, preserving the monolith/RR behaviour.
|
|
485
485
|
session_folder = find_session_dir(data_root, recording_id)
|
|
486
|
+
if session_folder is None:
|
|
487
|
+
source_recording = read_sorted_input_recording(recording_folder) or recording_id
|
|
488
|
+
session_folder = find_raw_session_dir(data_root, recording_id=source_recording)
|
|
486
489
|
|
|
487
490
|
if row.surface_finding is not None:
|
|
488
491
|
extract_continuous(
|
|
@@ -579,7 +582,8 @@ def run_ephys_for_stream(
|
|
|
579
582
|
)
|
|
580
583
|
# Raw session resolved by content too (ecephys_compressed/ marker), told apart
|
|
581
584
|
# from a surface-finding raw by recording_id. None -> converter sibling-split.
|
|
582
|
-
|
|
585
|
+
source_recording = read_sorted_input_recording(recording_folder) or recording_id
|
|
586
|
+
session_folder = find_raw_session_dir(data_root, recording_id=source_recording)
|
|
583
587
|
if session_folder is None:
|
|
584
588
|
session_folder = find_session_dir(data_root, recording_id)
|
|
585
589
|
|
|
@@ -135,10 +135,9 @@ def _mirror_pipeline_grid(pipeline: sitk.Image, base_before: sitk.Image, base_af
|
|
|
135
135
|
|
|
136
136
|
So the resample is done once, on the base, and the pipeline image inherits
|
|
137
137
|
the result: the same voxels, with its header rescaled by the same per-axis
|
|
138
|
-
factor. Origin is kept,
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
image lands where index ``i * factor`` of the original did.
|
|
138
|
+
factor. Origin is kept, as :func:`resample_to_isotropic` keeps the base's, so
|
|
139
|
+
in both headers index ``i`` of the resampled image lands where index
|
|
140
|
+
``i * factor`` of the original did.
|
|
142
141
|
|
|
143
142
|
Parameters
|
|
144
143
|
----------
|
|
@@ -171,25 +171,20 @@ def _probe_viability(
|
|
|
171
171
|
if not config.skip_ephys and mr.ephys_collection is not None:
|
|
172
172
|
if sorted_dirs is None:
|
|
173
173
|
sorted_dirs = find_sorted_session_dirs(config.data_root)
|
|
174
|
-
#
|
|
175
|
-
#
|
|
176
|
-
#
|
|
177
|
-
|
|
178
|
-
# nothing and the whole session reads as unsorted; that is how 791094 lost all
|
|
179
|
-
# 10 probes of 2025-10-09_14-26-31 while the run still exited 0.
|
|
180
|
-
sorted_dir = sorted_dirs.get(mr.recording_id)
|
|
174
|
+
# Honour the exact sorting pin first. Several sorting variants can name
|
|
175
|
+
# the same input recording. Input metadata is a compatibility fallback
|
|
176
|
+
# for published captures mounted under a different name.
|
|
177
|
+
sorted_dir = find_session_dir(config.data_root, str(mr.sorted_recording))
|
|
181
178
|
if sorted_dir is None:
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
)
|
|
192
|
-
notes.append(f"sorting matched by directory name ({sorted_dir.name}), not by input recording")
|
|
179
|
+
sorted_dir = sorted_dirs.get(mr.recording_id)
|
|
180
|
+
elif mr.recording_id not in sorted_dirs:
|
|
181
|
+
logger.warning(
|
|
182
|
+
"[discover] %s: sorting matched by directory name %s; "
|
|
183
|
+
"its input recording is absent from data_description",
|
|
184
|
+
mr.recording_id,
|
|
185
|
+
sorted_dir,
|
|
186
|
+
)
|
|
187
|
+
notes.append(f"sorting matched by directory name ({sorted_dir.name}), not by input recording")
|
|
193
188
|
if not has_sorting_output(sorted_dir, str(mr.ephys_collection)):
|
|
194
189
|
return False, "no spike-sorting output (bad sorting)", notes
|
|
195
190
|
ok, reason = _track_annotation_present(config.data_root, mr)
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# Image-source fixtures
|
|
2
|
+
|
|
3
|
+
These are the image layers from real Neuroglancer states under
|
|
4
|
+
`/mnt/vast/scratch/ephys/persist/data/light_sheet/NeuroglancerAnnotations`.
|
|
5
|
+
Only layer `name`, `type`, and `source` are retained; annotation points and viewer
|
|
6
|
+
state are omitted.
|
|
7
|
+
|
|
8
|
+
- `nested_image_sources.json`: `717381/717381_Neuroglancer_annotation_GL.json`.
|
|
9
|
+
Includes nested S3 prefixes, protocol wrappers, trailing slashes, and an object
|
|
10
|
+
source with `url`.
|
|
11
|
+
- `transformed_image_sources.json`:
|
|
12
|
+
`837772/837772-probe-annotations-CK-2026-09-29.json`. Includes plain string
|
|
13
|
+
sources and a source object carrying a transform.
|
|
14
|
+
|
|
15
|
+
Tests also remove the modality prefix from these sources to cover the new asset
|
|
16
|
+
naming convention. Copies live in both repositories so their checks run
|
|
17
|
+
independently.
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
{
|
|
2
|
+
"layers": [
|
|
3
|
+
{
|
|
4
|
+
"name": "Ex_561_Em_593.ome.zarr",
|
|
5
|
+
"type": "image",
|
|
6
|
+
"source": {
|
|
7
|
+
"url": "zarr://s3://aind-msma-morphology-data/test_data/SmartSPIM/SmartSPIM_717381_2024-07-03_10-49-01_stitched_2024-08-16_23-15-47/image_tile_fusing/OMEZarr/Ex_561_Em_593.ome.zarr/",
|
|
8
|
+
"subsources": {
|
|
9
|
+
"default": true
|
|
10
|
+
},
|
|
11
|
+
"enableDefaultSubsources": false
|
|
12
|
+
}
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"name": "Ex_639_Em_667.zarr",
|
|
16
|
+
"type": "image",
|
|
17
|
+
"source": "zarr://s3://aind-msma-morphology-data/test_data/SmartSPIM/SmartSPIM_717381_2024-07-03_10-49-01_stitched_2024-08-16_23-15-47/image_tile_fusing/OMEZarr/Ex_639_Em_667.ome.zarr/"
|
|
18
|
+
}
|
|
19
|
+
]
|
|
20
|
+
}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
{
|
|
2
|
+
"layers": [
|
|
3
|
+
{
|
|
4
|
+
"name": "Ex_488_Em_525.zarr",
|
|
5
|
+
"type": "image",
|
|
6
|
+
"source": "s3://aind-open-data/SmartSPIM_837772_2026-08-27_18-29-18_stitched_2026-09-03_13-39-04/image_tile_fusing/OMEZarr/Ex_488_Em_525.zarr"
|
|
7
|
+
},
|
|
8
|
+
{
|
|
9
|
+
"name": "Ex_561_Em_593.zarr",
|
|
10
|
+
"type": "image",
|
|
11
|
+
"source": {
|
|
12
|
+
"url": "s3://aind-open-data/SmartSPIM_837772_2026-08-27_18-29-18_stitched_2026-09-03_13-39-04/image_tile_fusing/OMEZarr/Ex_561_Em_593.zarr",
|
|
13
|
+
"transform": {
|
|
14
|
+
"matrix": [
|
|
15
|
+
[
|
|
16
|
+
1,
|
|
17
|
+
0,
|
|
18
|
+
0,
|
|
19
|
+
0,
|
|
20
|
+
0,
|
|
21
|
+
0
|
|
22
|
+
],
|
|
23
|
+
[
|
|
24
|
+
0,
|
|
25
|
+
1,
|
|
26
|
+
0,
|
|
27
|
+
0,
|
|
28
|
+
0,
|
|
29
|
+
0
|
|
30
|
+
],
|
|
31
|
+
[
|
|
32
|
+
0,
|
|
33
|
+
0,
|
|
34
|
+
1,
|
|
35
|
+
0,
|
|
36
|
+
0,
|
|
37
|
+
2
|
|
38
|
+
],
|
|
39
|
+
[
|
|
40
|
+
0,
|
|
41
|
+
0,
|
|
42
|
+
0,
|
|
43
|
+
1,
|
|
44
|
+
0,
|
|
45
|
+
-4
|
|
46
|
+
],
|
|
47
|
+
[
|
|
48
|
+
0,
|
|
49
|
+
0,
|
|
50
|
+
0,
|
|
51
|
+
0,
|
|
52
|
+
1,
|
|
53
|
+
0
|
|
54
|
+
]
|
|
55
|
+
],
|
|
56
|
+
"outputDimensions": {
|
|
57
|
+
"t": [
|
|
58
|
+
0.001,
|
|
59
|
+
"s"
|
|
60
|
+
],
|
|
61
|
+
"c'": [
|
|
62
|
+
1,
|
|
63
|
+
""
|
|
64
|
+
],
|
|
65
|
+
"z": [
|
|
66
|
+
2e-06,
|
|
67
|
+
"m"
|
|
68
|
+
],
|
|
69
|
+
"y": [
|
|
70
|
+
1.8e-06,
|
|
71
|
+
"m"
|
|
72
|
+
],
|
|
73
|
+
"x": [
|
|
74
|
+
1.8e-06,
|
|
75
|
+
"m"
|
|
76
|
+
]
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
"name": "Ex_639_Em_667.zarr",
|
|
83
|
+
"type": "image",
|
|
84
|
+
"source": "s3://aind-open-data/SmartSPIM_837772_2026-08-27_18-29-18_stitched_2026-09-03_13-39-04/image_tile_fusing/OMEZarr/Ex_639_Em_667.zarr"
|
|
85
|
+
}
|
|
86
|
+
]
|
|
87
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""Image assets are identified by source location and layout, not modality names."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
import pytest
|
|
7
|
+
from aind_zarr_utils.neuroglancer import get_image_sources
|
|
8
|
+
|
|
9
|
+
from aind_ibl_ephys_alignment_preprocessing.discovery import image_asset_path
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@pytest.mark.parametrize("fixture", ["nested_image_sources.json", "transformed_image_sources.json"])
|
|
13
|
+
@pytest.mark.parametrize("prefix", ["SmartSPIM_", ""])
|
|
14
|
+
def test_real_image_sources_resolve_to_the_asset_mount(tmp_path, fixture, prefix):
|
|
15
|
+
text = (Path(__file__).parent / "fixtures" / fixture).read_text()
|
|
16
|
+
state = json.loads(text.replace("SmartSPIM_", prefix))
|
|
17
|
+
sources = get_image_sources(state, remove_zarr_protocol=True)
|
|
18
|
+
for uri in sources.values():
|
|
19
|
+
name = uri.split("/image_tile_fusing/")[0].rsplit("/", 1)[1]
|
|
20
|
+
assert image_asset_path(tmp_path, uri) == tmp_path / name
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def test_local_image_source_preserves_its_existing_asset_root(tmp_path):
|
|
24
|
+
asset = tmp_path / "elsewhere" / "my-brain"
|
|
25
|
+
asset.mkdir(parents=True)
|
|
26
|
+
source = asset / "image_tile_fusing" / "OMEZarr" / "channel.ome.zarr" / "0"
|
|
27
|
+
assert image_asset_path(tmp_path / "data", str(source)) == asset
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def test_bad_image_source_reports_the_uri(tmp_path):
|
|
31
|
+
with pytest.raises(ValueError, match="s3://bucket/unrecognised/channel.zarr"):
|
|
32
|
+
image_asset_path(tmp_path, "s3://bucket/unrecognised/channel.zarr")
|
|
@@ -415,8 +415,6 @@ def _patch_zarr_utils(monkeypatch, asset: Path, default_channel: str) -> dict:
|
|
|
415
415
|
|
|
416
416
|
monkeypatch.setattr(discovery, "get_json", lambda *a, **k: {})
|
|
417
417
|
monkeypatch.setattr(discovery, "get_image_sources", lambda *a, **k: {"layer": str(default_zarr)})
|
|
418
|
-
monkeypatch.setattr(discovery, "as_pathlike", lambda uri: (None, None, str(uri)))
|
|
419
|
-
monkeypatch.setattr(discovery, "_asset_from_zarr_pathlike", lambda p: asset.name)
|
|
420
418
|
monkeypatch.setattr(
|
|
421
419
|
discovery,
|
|
422
420
|
"alignment_zarr_uri_and_metadata_from_zarr_or_asset_pathlike",
|
|
@@ -399,6 +399,35 @@ def test_find_raw_session_dir_ambiguous_without_match_raises(tmp_path):
|
|
|
399
399
|
find_raw_session_dir(tmp_path, recording_id="ecephys_does_not_match")
|
|
400
400
|
|
|
401
401
|
|
|
402
|
+
def test_stream_extraction_uses_sorting_source_to_distinguish_surface_raw(tmp_path, monkeypatch):
|
|
403
|
+
from types import SimpleNamespace
|
|
404
|
+
|
|
405
|
+
import aind_ephys_ibl_gui_conversion.ephys as converter
|
|
406
|
+
|
|
407
|
+
from aind_ibl_ephys_alignment_preprocessing.ephys import run_ephys_for_stream
|
|
408
|
+
|
|
409
|
+
_mount_sorted(tmp_path / "chosen-sorting", input_recording="actual-input")
|
|
410
|
+
raw = _mount_raw(tmp_path / "main-raw", name="actual-input")
|
|
411
|
+
_mount_raw(tmp_path / "surface-raw", name="surface-input")
|
|
412
|
+
seen = []
|
|
413
|
+
|
|
414
|
+
def extract(*args, **kwargs):
|
|
415
|
+
seen.append(kwargs["session_folder"])
|
|
416
|
+
|
|
417
|
+
monkeypatch.setattr(converter, "extract_continuous", extract)
|
|
418
|
+
monkeypatch.setattr(converter, "extract_spikes", extract)
|
|
419
|
+
outputs = SimpleNamespace(tracks_root=tmp_path / "results" / "mouse" / "tracks")
|
|
420
|
+
run_ephys_for_stream(
|
|
421
|
+
"chosen-sorting",
|
|
422
|
+
"output-label",
|
|
423
|
+
"ProbeA",
|
|
424
|
+
Path("surface-raw"),
|
|
425
|
+
outputs,
|
|
426
|
+
tmp_path,
|
|
427
|
+
)
|
|
428
|
+
assert seen == [raw, raw]
|
|
429
|
+
|
|
430
|
+
|
|
402
431
|
def _mount_nwb_output(sorted_root, recording_name="ecephys_771432_2025-03-07_18-22-06"):
|
|
403
432
|
"""Add zarr-backed NWB output to a sorted asset, as the 2026-07 sorts do.
|
|
404
433
|
|
|
@@ -461,6 +461,32 @@ def test_probe_viability_notes_a_name_matched_sorting(tmp_path: Path, tracks_pre
|
|
|
461
461
|
assert any("matched by directory name" in note for note in notes)
|
|
462
462
|
|
|
463
463
|
|
|
464
|
+
@pytest.mark.parametrize(
|
|
465
|
+
"pin",
|
|
466
|
+
[
|
|
467
|
+
"ecephys_863973_2026-08-27_14-52-47_sorted-opto_2026-10-10_12-12-16",
|
|
468
|
+
"my-chosen-sorting",
|
|
469
|
+
],
|
|
470
|
+
)
|
|
471
|
+
def test_probe_viability_honours_the_exact_sorting_pin(tmp_path, tracks_present, pin):
|
|
472
|
+
config = _config(tmp_path)
|
|
473
|
+
row = ManifestRow(
|
|
474
|
+
mouseid="863973",
|
|
475
|
+
sorted_recording=pin,
|
|
476
|
+
probe_file="ng",
|
|
477
|
+
probe_id="Track1",
|
|
478
|
+
probe_name="ProbeA",
|
|
479
|
+
)
|
|
480
|
+
# Another variant of this recording has no ProbeA output. A metadata-first
|
|
481
|
+
# lookup would choose it and incorrectly discard this manifest row.
|
|
482
|
+
wrong = config.data_root / "other-sorting"
|
|
483
|
+
_mount_sorted_for_coverage(wrong, input_recording=row.recording_id, collection="ProbeB")
|
|
484
|
+
chosen = config.data_root / pin
|
|
485
|
+
_mount_sorted_for_coverage(chosen, input_recording=row.recording_id, collection="ProbeA")
|
|
486
|
+
ok, reason, _ = stages._probe_viability(config, row, {row.recording_id: wrong})
|
|
487
|
+
assert ok and reason is None
|
|
488
|
+
|
|
489
|
+
|
|
464
490
|
class _Row:
|
|
465
491
|
"""Minimal ManifestRow stand-in for the track-annotation check."""
|
|
466
492
|
|
|
@@ -76,7 +76,7 @@ wheels = [
|
|
|
76
76
|
|
|
77
77
|
[[package]]
|
|
78
78
|
name = "aind-ibl-ephys-alignment-preprocessing"
|
|
79
|
-
version = "0.8.
|
|
79
|
+
version = "0.8.2"
|
|
80
80
|
source = { editable = "." }
|
|
81
81
|
dependencies = [
|
|
82
82
|
{ name = "aind-anatomical-utils" },
|
|
@@ -146,7 +146,7 @@ requires-dist = [
|
|
|
146
146
|
{ name = "aind-ephys-ibl-gui-conversion", specifier = ">=0.5.0" },
|
|
147
147
|
{ name = "aind-registration-utils", specifier = ">=0.4" },
|
|
148
148
|
{ name = "aind-s3-cache", specifier = ">=0.1" },
|
|
149
|
-
{ name = "aind-zarr-utils", specifier = ">=0.
|
|
149
|
+
{ name = "aind-zarr-utils", specifier = ">=0.18,<0.19" },
|
|
150
150
|
{ name = "antspyx", specifier = ">=0.3.0" },
|
|
151
151
|
{ name = "boto3" },
|
|
152
152
|
{ name = "filelock" },
|
|
@@ -253,7 +253,7 @@ wheels = [
|
|
|
253
253
|
|
|
254
254
|
[[package]]
|
|
255
255
|
name = "aind-zarr-utils"
|
|
256
|
-
version = "0.
|
|
256
|
+
version = "0.18.0"
|
|
257
257
|
source = { registry = "https://pypi.org/simple" }
|
|
258
258
|
dependencies = [
|
|
259
259
|
{ name = "aind-anatomical-utils" },
|
|
@@ -266,9 +266,9 @@ dependencies = [
|
|
|
266
266
|
{ name = "s3fs" },
|
|
267
267
|
{ name = "simpleitk" },
|
|
268
268
|
]
|
|
269
|
-
sdist = { url = "https://files.pythonhosted.org/packages/
|
|
269
|
+
sdist = { url = "https://files.pythonhosted.org/packages/75/90/216520a6d6330030c373088219c086096657ebe779ca0dbee80d56e0936a/aind_zarr_utils-0.18.0.tar.gz", hash = "sha256:898db68f942377279e25d36c4724a1fcadf5e7aeda5836b1be1780e56286d8a9", size = 566717, upload-time = "2026-10-26T00:20:02.363Z" }
|
|
270
270
|
wheels = [
|
|
271
|
-
{ url = "https://files.pythonhosted.org/packages/
|
|
271
|
+
{ url = "https://files.pythonhosted.org/packages/9e/a0/5c15ccadb7f133f2b2dccf3c151ac360d01c2733f4edab23f626fff75d96/aind_zarr_utils-0.18.0-py3-none-any.whl", hash = "sha256:4518cdccc085ace3f46b6a9d767f3a736fec2366abab6fa355ff7dc9cab6e56e", size = 67937, upload-time = "2026-10-26T00:20:01.265Z" },
|
|
272
272
|
]
|
|
273
273
|
|
|
274
274
|
[[package]]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|