MNutils 1.2.0__py3-none-any.whl

mnutils/testing/synthetic.py

@@ -0,0 +1,373 @@
+"""Synthetic GE exam fixtures for docs pages and tests -- no real scanner data required.
+
+See docs/diary/2026-08-19-synthetic-exam-fixtures.md for why this exists, what it does and doesn't
+fake, and where the spectra/anatomy come from. `build_fake_exam(dataset)` writes the requested
+dataset under a process-local temp directory (via `tempfile.mkdtemp`) the first time it's called,
+caches each dataset's path for the rest of the process, and registers an `atexit` cleanup so
+nothing survives past the process that created it.
+
+`_TEMPLATE_ATTRIBUTION` in `_spectra.py` covers the one piece of real (downloaded, cached) data
+used here -- a CC BY-NC 4.0 T1 template.
+"""
+
+import atexit
+import functools
+import shutil
+import tempfile
+from collections.abc import Callable
+from pathlib import Path
+
+import h5py
+import nibabel as nib
+import numpy as np
+import scipy.io
+import xmris
+from numpy.typing import ArrayLike
+
+from . import _spectra
+
+_RNG_SEED = 20260819
+
+
+def _mat_header(
+    *,
+    nucleus: float,
+    protocol: str,
+    description: str,
+    exam_number: int,
+    bandwidth: float,
+    centre_freq_x1e7: float,
+    sctime: float = 60_000_000.0,
+    fov_mm: float,
+) -> dict:
+    """Build the nested dict `RawMRISeries._get_header_value` walks (`h.rdb_hdr.*` etc.)."""
+    return {
+        "rdb_hdr": {
+            "data_collect_type": 2.0,
+            "run_int": 1.0,
+            "nframes": 8.0,
+            "user0": bandwidth,
+            "ps_mps_freq": centre_freq_x1e7,
+            "te": 30000.0,
+            "fov": fov_mm,
+            "user25": 0.0,
+            "user14": 0.0,
+            "user3": 0.0,
+            "ps_mps_r1": 100.0,
+            "ps_mps_r2": 100.0,
+            "ps_mps_tg": 100.0,
+            "recon": 1.0,
+            "scan_date": "01/01/25",
+            "scan_time": "12:00",
+        },
+        "series": {"prtcl": protocol, "se_desc": description},
+        "exam": {"ex_no": exam_number},
+        "image": {
+            "specnuc": nucleus,
+            "mr_flip": 90.0,
+            "nex": 1.0,
+            "tr": 2_000_000.0,
+            "slthick": 5.0,
+            "sctime": sctime,
+        },
+        "mrconfig": {"fieldStrength": 30_000.0},
+    }
+
+
+# Deuterium at ~3T. Kept in one place so the .mat header, the ppm axis GESeries derives from it,
+# and the peaks xmris simulates all agree on what "4.7 ppm" means.
+_NUCLEUS = 2.0
+_BANDWIDTH_HZ = 5000.0
+_CENTRE_FREQ_X1E7 = 1.9612e8  # -> 80.0 MHz via the `centre_frequency` property
+_CENTRE_FREQ_MHZ = _CENTRE_FREQ_X1E7 / 1e7
+_CARRIER_PPM = 4.68  # matches GESeries.RawMRISeries.carrier_ppm for nucleus in {1, 2}
+
+
+def _rng(*salt: object) -> np.random.Generator:
+    return np.random.default_rng(abs(hash((_RNG_SEED, *salt))) % (2**32))
+
+
+def _write_own_nii(path: Path, data: np.ndarray, affine: np.ndarray) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    nib.save(nib.Nifti1Image(data.astype(np.float32), affine), path)
+
+
+def _write_mat(path: Path, *, spec: np.ndarray, fov_mm: float, seed: int, **header_kwargs) -> None:
+    recon = {
+        "h": _mat_header(fov_mm=fov_mm, **header_kwargs),
+        "bb": _rng(seed, "bb").standard_normal((4, 4)),
+        "spec": spec,
+    }
+    path.parent.mkdir(parents=True, exist_ok=True)
+    scipy.io.savemat(path, recon)
+
+
+def _write_mrs_mat(
+    path: Path,
+    *,
+    series_id: int,
+    averages: int,
+    npts: int,
+    seed: int,
+    intensities: ArrayLike | None = None,
+    fov_mm: float = 240.0,
+    **header_kwargs,
+) -> None:
+    """Write the `.mat` spectrum and, from the same simulated peaks, the raw FID `.h5` cache.
+
+    Every real MRS acquisition has both, so every synthetic one does too -- `simulate_fid`
+    returns the raw time-domain FID `xr.DataArray`, which we convert both ways from a single
+    simulation instead of writing an independently-random raw-FID array.
+    """
+    fid_da = _spectra.simulate_fid(
+        npts=npts,
+        n_specs=averages,
+        bandwidth=_BANDWIDTH_HZ,
+        seed=seed,
+        intensities=intensities,
+    )
+    fid = np.asarray(fid_da.values, dtype=complex)
+    spec = np.asarray(xmris.to_spectrum(fid_da).values, dtype=complex)
+    _write_mat(
+        path,
+        spec=spec,
+        fov_mm=fov_mm,
+        seed=seed,
+        nucleus=_NUCLEUS,
+        bandwidth=_BANDWIDTH_HZ,
+        centre_freq_x1e7=_CENTRE_FREQ_X1E7,
+        **header_kwargs,
+    )
+    # In mrs data the fids are chopped. IE every other fid is multiplied by -1
+    # Fake this too
+    fid[1::2, :] *= -1
+    # And the data is complex conjugated
+    fid = np.conjugate(fid)
+
+    _write_raw_fids_h5(path.parent, series_id, fid)
+
+
+def _write_mrsi_mat(
+    path: Path,
+    *,
+    npts: int,
+    grid: tuple[int, int, int],
+    fov_mm: float,
+    seed: int,
+    intensity_map: np.ndarray | None = None,
+    **header_kwargs,
+) -> None:
+    if intensity_map is None:
+        intensity_map = np.full(grid, 0.5, dtype=np.float32)
+    spec = _spectra.simulate_grid(
+        npts=npts,
+        grid=grid,
+        bandwidth=_BANDWIDTH_HZ,
+        centre_freq_mhz=_CENTRE_FREQ_MHZ,
+        carrier_ppm=_CARRIER_PPM,
+        intensity_map=intensity_map,
+        seed=seed,
+    )
+    _write_mat(
+        path,
+        spec=spec,
+        fov_mm=fov_mm,
+        seed=seed,
+        nucleus=_NUCLEUS,
+        bandwidth=_BANDWIDTH_HZ,
+        centre_freq_x1e7=_CENTRE_FREQ_X1E7,
+        **header_kwargs,
+    )
+
+
+def _write_raw_fids_h5(series_folder: Path, series_id: int, fids: np.ndarray) -> None:
+    """A dummy `ScanArchive*.h5` (only its existence matters) plus a pre-cached `fids` dataset.
+
+    `load_raw_fids` reads the cache directly when present and never opens the archive -- see the
+    diary entry.
+    """
+    series_folder.mkdir(parents=True, exist_ok=True)
+    (series_folder / f"ScanArchive_{series_id}.h5").touch()
+    with h5py.File(series_folder / f"Series{series_id}_raw_fids.h5", "w") as hf:
+        hf.create_dataset("fids", data=fids)
+
+
+def _write_raw_fid_files(series_folder: Path, series_id: int, npts: int, ntime: int = 150) -> None:
+    """Random-noise raw FIDs for series with no `.mat`-backed spectrum to derive real ones from."""
+    fids = _rng(series_folder, "fids").standard_normal((npts, ntime)) + 1j * _rng(
+        series_folder, "fids-imag"
+    ).standard_normal((npts, ntime))
+    _write_raw_fids_h5(series_folder, series_id, fids)
+
+
+def _build_brain_mrs_mrsi_exam(root: Path) -> None:
+    data = root / "data"
+    exam = data / "ExamHeVo18anon"
+    grid = (16, 16, 16)
+
+    # The real T1 template, cropped to the head -- see _spectra._TEMPLATE_ATTRIBUTION.
+    _spectra.write_template_t1(data / "002_3D_Ax_T1_BRAVO" / "2_3D_Ax_T1_BRAVO.nii.gz")
+
+    # Series 8's own NIfTI seeds create_MRSI_nii/create_MRSI_affine *and* doubles as the
+    # per-grid-cell intensity map below -- real brain signal in, brain-shaped spectra out.
+    own_affine, intensity_map = _spectra.template_grid_intensity(grid)
+    _write_own_nii(
+        data / "008_MRSI_pseudo_S64_X10_Y10_Z10_T1_C1" / "8_MRSI_pseudo.nii.gz",
+        intensity_map * 200,
+        own_affine,
+    )
+    # create_MRSI_affine() rescales x/y from the header's fov_mm/mtx[0] against own_affine's x
+    # voxel size -- derive fov_mm from that same affine so the two agree (a mismatched hardcoded
+    # value here inflates the MRSI grid relative to the real brain extent it's drawn over).
+    fov_mm = own_affine[0, 0] * grid[0]
+    (data / "014_localizer").mkdir(parents=True, exist_ok=True)  # DICOM-only gap filler
+
+    _write_mrs_mat(
+        exam / "Series6" / "ScanArchive_Series6.mat",
+        series_id=6,
+        averages=64,
+        npts=2048,
+        seed=6,
+        protocol="MRS_unloc",
+        description="006_MRS_unloc",
+        exam_number=1801,
+    )
+
+    # Create increasing intensities for the washin series
+    # Defines start values [HDO, Glucose, Glx, Baseline] and their stop values
+    intensities = np.linspace(start=[1.0, 0.4, 0.0, 1.0], stop=[1.2, 3.0, 0.6, 1.0], num=250)
+
+    _write_mrs_mat(
+        exam / "Series7" / "ScanArchive_Series7.mat",
+        series_id=7,
+        averages=250,
+        intensities=intensities,
+        npts=2048,
+        seed=7,
+        protocol="MRS_washin",
+        description="007_MRS_washin",
+        exam_number=1801,
+        sctime=300_000_000.0,
+    )
+
+    _write_mrsi_mat(
+        exam / "Series8" / "ScanArchive_Series8.mat",
+        npts=700,
+        grid=grid,
+        fov_mm=fov_mm,
+        seed=8000,
+        intensity_map=intensity_map,
+        protocol="MRSI_pseudo",
+        description="008_MRSI_pseudo_S64_X10_Y10_Z10_T1_C1",
+        exam_number=1801,
+    )
+
+    for series_id, npts in ((9, 700), (11, 700), (12, 1678)):
+        _write_raw_fid_files(exam / f"Series{series_id}", series_id, npts=npts)
+
+    (exam / "Series13").mkdir(parents=True, exist_ok=True)  # exam data, no DICOM
+
+
+def _build_brain_extraction_exam(root: Path) -> None:
+    data = root / "data"
+    (data / "Series0010_BS_prescan").mkdir(parents=True, exist_ok=True)  # new-style naming
+    _spectra.write_template_t1(data / "002_3D_Ax_T1_BRAVO" / "2_3D_Ax_T1_BRAVO_BODY.nii.gz")
+
+
+def _build_mrsi_missing_series_exam(root: Path) -> None:
+    exam = root / "Exam4873anon"
+    _write_mrsi_mat(
+        exam / "Series7" / "ScanArchive_Series7.mat",
+        npts=700,
+        grid=(16, 16, 16),
+        fov_mm=300.0,
+        seed=7700,
+        protocol="MRSI_pseudo",
+        description="007_MRSI_pseudo",
+        exam_number=4873,
+    )
+    (exam / "Series5").mkdir(parents=True, exist_ok=True)  # no .mat -- FileNotFoundError case
+    (root / "010_axial_localizer").mkdir(parents=True, exist_ok=True)  # old-style naming
+
+
+def _build_nist_phantom_exam(root: Path) -> None:
+    data = root / "data"
+    exam = data / "ExamNISTanon"
+    grid = (16, 16, 16)
+    fov_mm = 300.0  # -> 14 mm in-plane voxels, comfortably resolves the 8-sphere ring
+    z_extent_mm = 300.0
+    rings = _spectra.DEFAULT_SPHERE_RINGS
+
+    _spectra.sphere_phantom_image(
+        data / "002_3D_Ax_T1_BRAVO" / "2_3D_Ax_T1_BRAVO.nii.gz",
+        shape=(128, 128, 100),
+        voxel_mm=1,
+        rings=rings,
+        seed=2,
+    )
+
+    intensity_map, voxel_xy = _spectra.sphere_grid_intensity(
+        grid, fov_mm=fov_mm, z_extent_mm=z_extent_mm, rings=rings
+    )
+    own_affine = _spectra.grid_own_affine(grid, (voxel_xy, voxel_xy, z_extent_mm / grid[2]))
+    _write_own_nii(
+        data / "005_MRSI_pseudo_S64_X12_Y12_Z3_T1_C1" / "5_MRSI_pseudo.nii.gz",
+        intensity_map * 200,
+        own_affine,
+    )
+
+    _write_mrsi_mat(
+        exam / "Series5" / "ScanArchive_Series5.mat",
+        npts=700,
+        grid=grid,
+        fov_mm=fov_mm,
+        seed=5500,
+        intensity_map=intensity_map * 500,
+        protocol="MRSI_pseudo",
+        description="005_MRSI_pseudo_S64_X12_Y12_Z3_T1_C1",
+        exam_number=408,
+    )
+
+
+def _write_legacy_mat(path: Path) -> None:
+    """A genuine MATLAB v7.3 (HDF5) file -- `mat73`'s fallback path, not `scipy.io.loadmat`."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with h5py.File(path, "w") as hf:
+        dataset = hf.create_dataset("pyr", data=_rng(path, "pyr").standard_normal(64))
+        dataset.attrs["MATLAB_class"] = np.bytes_(b"double")
+
+
+_DATASETS: dict[str, tuple[Callable[[Path], None], str]] = {
+    "brain_mrs_mrsi_exam": (_build_brain_mrs_mrsi_exam, "brain_mrs_mrsi_exam"),
+    "brain_extraction_exam": (_build_brain_extraction_exam, "brain_extraction_exam"),
+    "mrsi_missing_series_exam": (_build_mrsi_missing_series_exam, "mrsi_missing_series_exam"),
+    "nist_phantom_exam": (_build_nist_phantom_exam, "nist_phantom_exam"),
+    "legacy_matlab73_example": (_write_legacy_mat, "legacy_matlab73_example.mat"),
+}
+
+
+@functools.cache
+def _fake_exam_root() -> Path:
+    root = Path(tempfile.mkdtemp(prefix="mnutils_fake_exam_"))
+    atexit.register(shutil.rmtree, root, ignore_errors=True)
+    return root
+
+
+@functools.cache
+def build_fake_exam(dataset: str) -> Path:
+    """Build (on first call) and return the path to one synthetic dataset.
+
+    `dataset` is one of `_DATASETS`' keys -- the same names and shapes docs pages and tests
+    previously read from `tests/datasets/`, entirely fabricated (see the diary entry for what's
+    simulated vs. downloaded). Every dataset lives under one shared `tempfile.mkdtemp` directory
+    (created on first call to any dataset) removed by an `atexit` hook when the process exits, so
+    nothing accumulates across runs. Each dataset is built at most once per process.
+    """
+    if dataset not in _DATASETS:
+        raise ValueError(
+            f"Unknown fake-exam dataset {dataset!r}; choose one of {sorted(_DATASETS)}"
+        )
+    builder, filename = _DATASETS[dataset]
+    path = _fake_exam_root() / filename
+    builder(path)
+    return path

mnutils/utils/__init__.py

@@ -0,0 +1,25 @@
+"""Loading, file discovery, NIfTI and spectral helpers."""
+
+from . import data_loaders, file_helpers, images, nifti, spectra
+from .data_loaders import load_mat_file
+from .file_helpers import (
+    get_exam_folder,
+    get_h5_data_from_series,
+    get_mat_data_from_series,
+)
+from .spectra import calculate_ppm_axis
+
+__all__ = [
+    # modules
+    "data_loaders",
+    "file_helpers",
+    "images",
+    "nifti",
+    "spectra",
+    # frequently used helpers
+    "calculate_ppm_axis",
+    "get_exam_folder",
+    "get_h5_data_from_series",
+    "get_mat_data_from_series",
+    "load_mat_file",
+]

mnutils/utils/data_loaders.py

@@ -0,0 +1,218 @@
+import getpass
+from datetime import datetime
+from pathlib import Path
+
+import h5py
+import numpy as np
+from loguru import logger
+
+from . import file_helpers
+
+
+def load_mat_file(file_path: Path | str) -> dict:
+    """Load a .mat file and return its contents as a dictionary.
+
+    Transparently falls back from `scipy.io.loadmat` (plain v5/v7 files) to
+    `mat73` (v7.3, HDF5-based files) when the former can't read the file.
+
+    Parameters
+    ----------
+    file_path : Path or str
+        Path to the .mat file.
+
+    Returns
+    -------
+    dict
+        Contents of the .mat file.
+
+    Raises
+    ------
+    FileNotFoundError
+        If `file_path` does not exist or is not a file.
+    ValueError
+        If `file_path` does not have a `.mat` extension.
+    RuntimeError
+        If the file can't be loaded by either scipy or mat73.
+    """
+    file_path = Path(file_path)
+    if not file_path.is_file():
+        raise FileNotFoundError(
+            f"The specified path does not exist or is not a file: '{file_path}'"
+        )
+    if file_path.suffix.lower() != ".mat":
+        raise ValueError(f"The file does not have a .mat extension: '{file_path}'")
+
+    try:
+        # Try loading with scipy.io.loadmat first
+        return _loadmat(file_path)
+    except (NotImplementedError, ValueError):
+        # If it fails, try loading with mat73 (for v7.3 .mat files)
+        try:
+            import mat73
+
+            return mat73.loadmat(file_path)
+        except ImportError as e:
+            raise RuntimeError(
+                "mat73 is required to read MATLAB v7.3 files but could not be imported."
+            ) from e
+        except OSError as e:
+            logger.error(f"Failed to load .mat file with both scipy and mat73: {e}")
+            raise OSError(f"Failed to open .mat file with mat73: {e}")
+        except (TypeError, ValueError) as e:
+            raise RuntimeError(f"Unexpected error loading .mat file with mat73: {e}")
+    except (OSError, TypeError) as e:
+        logger.error(f"Failed to load .mat file with scipy: {e}.")
+        raise RuntimeError(f"Unexpected error loading .mat file with scipy: {e}")
+
+
+def load_raw_fids(
+    DATA_FOLDER: Path | str,
+    series_id: int,
+    load_most_recent: bool = True,
+    force_override: bool = False,
+) -> tuple[np.ndarray, Path]:
+    """Load raw FIDs for a series, reading from a cached HDF5 file when available.
+
+    Parameters
+    ----------
+    DATA_FOLDER : Path or str
+        Path to the exam's data folder.
+    series_id : int
+        Series number to load raw FIDs for.
+    load_most_recent : bool
+        If True, return the most recently created cached `*_raw_fids.h5` file
+        instead of re-reading the scan archive.
+    force_override : bool
+        If True, delete and regenerate an existing output HDF5 file rather than
+        returning its cached contents.
+
+    Returns
+    -------
+    tuple[np.ndarray, Path]
+        The complex raw FIDs array and the path to the HDF5 file it was
+        loaded from (or written to).
+    """
+    scan_archive_file = file_helpers.get_h5_data_from_series(DATA_FOLDER, series_id)
+    series_folder = file_helpers.get_exam_folder(DATA_FOLDER) / f"Series{series_id}"
+
+    # Find if there is already an h5 file with raw fids
+    if load_most_recent:
+        existing_files = list(series_folder.glob(f"Series{series_id}_raw_fids.h5"))
+        if existing_files:
+            # Get the most recent file based on creation time
+            output_h5_file = max(existing_files, key=lambda f: f.stat().st_ctime)
+            logger.debug(f"Found existing raw FIDs file: {output_h5_file}. Loading data from it.")
+            with h5py.File(output_h5_file, "r") as hf:
+                data = np.array(hf["fids"])
+            return data, output_h5_file
+        else:
+            logger.debug(
+                f"No existing raw FIDs file found in {series_folder}. Will create a new one."
+            )
+
+    output_h5_file = series_folder / f"Series{series_id}_raw_fids.h5"
+
+    logger.debug(f"Loading raw FIDs from {scan_archive_file} into {output_h5_file}")
+
+    if output_h5_file.exists():
+        if not force_override:
+            logger.warning(f"Output file {output_h5_file} already exists. Returning data from it.")
+            with h5py.File(output_h5_file, "r") as hf:
+                data = np.array(hf["fids"])
+            return data, output_h5_file
+        else:
+            logger.warning(
+                f"Output file {output_h5_file} already exists. "
+                "Overriding as force_override is True."
+            )
+            output_h5_file.unlink()
+
+    from .. import matlab
+
+    matengine = matlab.connect_to_matlab()
+    matlab.setup_util_path(matengine)
+    matlab.add_matlablatest_path(matengine)
+    matengine.manage_paths("ORCHESTRASDK", True, nargout=0)
+    data = matengine.read_archive(str(scan_archive_file), 0, nargout=1)
+    matengine.quit()
+    logger.debug(
+        f"Raw data loaded from MATLAB engine, type: {type(data)}. Matlabengine session closed."
+    )
+    data = np.asarray(data)
+    # Validate data shape and type to be complex numpy array
+    if not isinstance(data, np.ndarray) or not np.iscomplexobj(data):
+        raise ValueError(f"Loaded data is not a complex numpy array. Got type: {type(data)}")
+
+    with h5py.File(output_h5_file, "w") as hf:
+        hf.create_dataset("fids", data=data)
+        hf.attrs["file_created_date"] = datetime.now().isoformat().replace("+00:00", "Z")
+        try:
+            hf.attrs["file_created_by"] = getpass.getuser()
+        except (OSError, KeyError):
+            hf.attrs["file_created_by"] = "unknown_user"
+
+    # Check if file exists
+    if not output_h5_file.exists():
+        raise FileNotFoundError(f"Output file {output_h5_file} was not created successfully.")
+
+    return data, output_h5_file
+
+
+def _loadmat(filename: str | Path) -> dict:
+    """
+    Load a MATLAB .mat file and convert its contents into Python data structures.
+
+    This function wraps around `scipy.io.loadmat` to address issues with
+    recovering Python dictionaries from MATLAB .mat files. It ensures that
+    all entries, including those stored as MATLAB objects, are properly
+    converted to Python-native types by invoking the `_check_keys` function.
+
+    Args:
+        filename (str): The path to the .mat file to be loaded.
+
+    Returns
+    -------
+        dict: A dictionary containing the contents of the .mat file, with
+        MATLAB objects converted to Python-native types.
+    """
+    import scipy.io
+
+    data = scipy.io.loadmat(filename, struct_as_record=False, squeeze_me=True)
+    return _check_keys(data)
+
+
+def _check_keys(data: dict) -> dict:
+    """Recursively convert MATLAB structs within a dictionary to Python dictionaries.
+
+    Iterates through the keys of the given dictionary and checks if any of the values
+    are MATLAB structs (from `scipy.io.matlab.mat_struct`). If a MATLAB struct is
+    found, it is converted to a Python dictionary using the `_todict` function.
+
+    Parameters
+    ----------
+    data : dict
+        The input dictionary to process.
+
+    Returns
+    -------
+    dict
+        The processed dictionary with MATLAB structs converted to Python dictionaries.
+    """
+    from scipy.io.matlab import mat_struct
+
+    for key in data:
+        if isinstance(data[key], mat_struct):
+            data[key] = _todict(data[key], mat_struct)
+    return data
+
+
+def _todict(matobj, mat_struct_type: type) -> dict:
+    """Recursively construct nested dictionaries from a MATLAB struct object."""
+    out: dict = {}
+    for field_name in matobj._fieldnames:
+        elem = matobj.__dict__[field_name]
+        if isinstance(elem, mat_struct_type):
+            out[field_name] = _todict(elem, mat_struct_type)
+        else:
+            out[field_name] = elem
+    return out