nltools 0.6.0.dev0__py3-none-any.whl

nltools/data/designmatrix/diagnostics.py

@@ -0,0 +1,248 @@
+"""Collinearity diagnostics for DesignMatrix: column correlations, VIF, and cleanup."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+from .utils import _copy_with, _get_data_columns, _is_generated_intercept
+
+if TYPE_CHECKING:
+    from nltools.data import Adjacency
+    from nltools.data.designmatrix import DesignMatrix
+
+
+def _corr(
+    dm: DesignMatrix,
+    *,
+    metric: str = "pearson",
+    columns: list[str] | None = None,
+) -> Adjacency:
+    """Correlation between DesignMatrix columns as an Adjacency.
+
+    Returns the column-by-column correlation matrix wrapped in an nltools
+    ``Adjacency`` (``matrix_type='similarity'``) so it composes with the rest
+    of the similarity-matrix tooling (``.plot()``, MDS, etc.). The Adjacency
+    stores only the off-diagonal entries — self-correlation isn't a meaningful
+    edge — so the unit diagonal is implicit; ``DesignMatrix.plot(method='corr')``
+    restores it for display.
+
+    Args:
+        dm (DesignMatrix): DesignMatrix instance.
+        metric (str): ``'pearson'`` (default) or ``'spearman'``. Spearman is
+            computed as Pearson on column ranks.
+        columns (list[str] | None): Subset of columns to correlate.
+            Defaults to all columns.
+
+    Returns:
+        Adjacency: Similarity matrix whose ``labels`` are the included column
+            names.
+
+    Raises:
+        ValueError: If ``metric`` is unknown or fewer than 2 columns remain.
+
+    Note:
+        Constant columns (e.g. the ``.nl_poly_0`` intercept) have zero variance and
+        yield NaN correlations.
+    """
+    from nltools.data import Adjacency
+
+    valid_metrics = ("pearson", "spearman")
+    if metric not in valid_metrics:
+        raise ValueError(f"metric must be one of {valid_metrics}, got {metric!r}.")
+
+    cols = list(columns) if columns is not None else list(dm.columns)
+    if len(cols) < 2:
+        raise ValueError(
+            f"Correlation requires at least 2 columns; got {len(cols)} ({cols})."
+        )
+
+    X = dm.data.select(cols).to_numpy().astype(float)
+
+    if metric == "spearman":
+        # Spearman == Pearson on per-column ranks. Ranking column-wise (rather
+        # than scipy.stats.spearmanr) keeps a uniform n-by-n result even for
+        # the 2-column case, where spearmanr collapses to a scalar.
+        from scipy.stats import rankdata
+
+        X = np.column_stack([rankdata(X[:, i]) for i in range(X.shape[1])])
+
+    # Constant columns (zero variance) make corrcoef divide by zero -> NaN.
+    # That NaN is the intended, documented signal; silence the low-level
+    # numpy warning so it isn't surfaced as noise.
+    with np.errstate(invalid="ignore", divide="ignore"):
+        mat = np.corrcoef(X, rowvar=False)
+    return Adjacency(
+        np.asarray(mat, dtype=float), matrix_type="similarity", labels=cols
+    )
+
+
+def _vif(dm: DesignMatrix, exclude_confounds: bool = True) -> np.ndarray | None:
+    """Compute the variance inflation factor for each column.
+
+    Uses diagonal elements of inverted correlation matrix
+    (same method as Matlab and R).
+
+    Args:
+        dm (DesignMatrix): DesignMatrix instance.
+        exclude_confounds (bool): Skip nuisance/confound columns. Default: True.
+
+    Returns:
+        np.ndarray | None: VIF values for each included column, or None if the
+            correlation matrix is singular (perfect collinearity detected).
+
+    Raises:
+        ValueError: If the DesignMatrix has only 1 column.
+    """
+    import polars.selectors as cs
+
+    if dm.shape[1] <= 1:
+        raise ValueError(
+            "Can't compute VIF with only 1 column! "
+            "VIF measures multicollinearity and requires at least 2 columns. "
+            f"Your DesignMatrix has shape {dm.shape}."
+        )
+
+    # Determine which columns to include (using polars selectors for declarative filtering)
+    if exclude_confounds and dm.confounds:
+        # Use polars selector: "select all columns except confound terms"
+        subset_df = dm.data.select(cs.exclude(dm.confounds))
+    elif exclude_confounds:
+        # No confounds to exclude, use all columns
+        subset_df = dm.data
+    else:
+        # Always exclude generated intercepts even when exclude_confounds=False:
+        # an all-ones column has zero variance, so it makes the correlation
+        # matrix singular and VIF undefined.
+        cols_to_use = [c for c in dm.columns if not _is_generated_intercept(c)]
+        subset_df = dm.data.select(cols_to_use)
+
+    # Edge case: single column has VIF = 1 (no multicollinearity)
+    if subset_df.shape[1] == 1:
+        return np.array([1.0])
+
+    # Convert to numpy for correlation matrix and linear algebra
+    # NECESSARY: Polars doesn't have correlation matrix or matrix inversion
+    data_array = subset_df.to_numpy()
+
+    # Compute correlation matrix
+    corr_matrix = np.corrcoef(data_array, rowvar=False)
+
+    # Compute VIF = diagonal of inverse correlation matrix
+    try:
+        inv_corr = np.linalg.inv(corr_matrix)
+        return np.diag(inv_corr)
+    except np.linalg.LinAlgError:
+        # Matrix is singular - perfect collinearity detected
+        # Return None and warn user (matches old behavior)
+        print(
+            "ERROR: Cannot compute VIF! Design Matrix is singular because it has "
+            "some perfectly correlated or duplicated columns. Using .clean() may help."
+        )
+        return None
+
+
+def _clean(
+    dm: DesignMatrix,
+    *,
+    fill_na: int | float | None = 0,
+    exclude_confounds: bool = False,
+    thresh: float = 0.95,
+    progress_bar: bool = False,
+) -> DesignMatrix:
+    """Remove highly correlated columns.
+
+    Removes columns with correlation >= threshold. Keeps first instance
+    of correlated pair, drops duplicates.
+
+    Args:
+        dm (DesignMatrix): DesignMatrix instance.
+        fill_na (int | float | None): Fill NaN values before checking correlations.
+            Default: 0.
+        exclude_confounds (bool): Skip nuisance/confound columns from correlation check.
+            Default: False.
+        thresh (float): Correlation threshold (drop if abs(r) >= thresh).
+            Default: 0.95.
+        progress_bar (bool): Print dropped column names. Default: False.
+
+    Returns:
+        DesignMatrix: Cleaned matrix with highly correlated columns removed
+    """
+    # Check for duplicate column names
+    if len(dm.columns) != len(set(dm.columns)):
+        raise ValueError(
+            "Duplicate column names detected. Using .clean() with duplicate "
+            "columns is not supported as it can produce unexpected results."
+        )
+
+    # Start with a copy
+    result = dm
+
+    # Fill NaN if requested
+    if fill_na is not None:
+        result = result.fillna(fill_na)
+
+    # Determine which columns to check for correlation
+    if exclude_confounds:
+        cols_to_check = _get_data_columns(result, exclude_confounds=True)
+    else:
+        cols_to_check = list(result.columns)
+
+    if len(cols_to_check) <= 1:
+        if progress_bar:
+            print("Only 1 column to check...skipping")
+        return result
+
+    # Compute pairwise correlations and identify columns to drop
+    keep = []
+    remove = []
+
+    # Convert to numpy for pairwise correlation computation
+    # NECESSARY: More efficient than Polars for this operation
+    subset_df = result.data.select(cols_to_check)
+    data_array = subset_df.to_numpy()
+
+    # Check each pair of columns
+    for i in range(len(cols_to_check)):
+        col_i = cols_to_check[i]
+        col_i_data = data_array[:, i]
+
+        for j in range(i + 1, len(cols_to_check)):
+            col_j = cols_to_check[j]
+            col_j_data = data_array[:, j]
+
+            # Skip if already marked for removal or keeping
+            if col_j in keep or col_j in remove:
+                continue
+
+            # Check for constant arrays (avoid correlation warnings)
+            if np.var(col_i_data) == 0 or np.var(col_j_data) == 0:
+                r = 0.0
+            else:
+                # Compute correlation
+                r = np.abs(np.corrcoef(col_i_data, col_j_data)[0, 1])
+
+            # Mark for removal if correlation exceeds threshold
+            if r >= thresh and col_i not in keep and col_i not in remove:
+                if progress_bar:
+                    print(
+                        f"{col_i} and {col_j} correlated at {r:.2f} which is >= "
+                        f"threshold of {thresh}. Dropping {col_j}"
+                    )
+                keep.append(col_i)
+                remove.append(col_j)
+
+    # Drop correlated columns
+    if remove:
+        # Drop from DataFrame
+        new_df = result.data.drop(remove)
+
+        # Update confounds metadata
+        new_confounds = [p for p in result.confounds if p not in remove]
+
+        # Return cleaned matrix
+        return _copy_with(result, new_df, confounds=new_confounds)
+    if progress_bar:
+        print("Dropping columns not needed...skipping")
+    return result

nltools/data/designmatrix/io.py

@@ -0,0 +1,356 @@
+"""Read and write DesignMatrix objects.
+
+Loads BIDS events and tabular confound files into the frame a `DesignMatrix`
+wraps, exports NumPy arrays, and round-trips through TSV/CSV or HDF5
+(which also preserves the metadata). A private pandas adapter serves nilearn.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import numpy as np
+import polars as pl
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+    from nltools.data.designmatrix import DesignMatrix
+
+
+def _events_to_convolved_dm(
+    events: pl.DataFrame | pd.DataFrame,
+    *,
+    run_length: int,
+    sampling_freq: float,
+    hrf_model: str,
+) -> pl.DataFrame:
+    """Convert a BIDS events table straight to HRF-convolved regressors.
+
+    `make_first_level_design_matrix` convolves the events at nilearn's own
+    oversampling and only then samples onto the frame times, so onsets that
+    fall between TRs keep their timing. Going through `events_to_dm` first
+    would quantize them onto the TR grid before convolution, and the result
+    would no longer match a nilearn `FirstLevelModel` on the same events.
+
+    Args:
+        events (pl.DataFrame | pd.DataFrame): Events table with BIDS columns
+            `onset`, `duration`, `trial_type` (required); `modulation` is
+            passed through if present.
+        run_length (int): Number of TRs the run contains.
+        sampling_freq (float): Sampling frequency in Hz (= 1/TR).
+        hrf_model (str): An HRF model name from `_KERNELS`.
+
+    Returns:
+        pl.DataFrame: One column per unique `trial_type`, convolved and named
+            `<trial_type>_c0`.
+    """
+    import pandas as pd
+    from nilearn.glm.first_level import make_first_level_design_matrix
+
+    from .regressors import _KERNELS
+
+    if isinstance(events, pl.DataFrame):
+        events = pd.DataFrame(events.to_dict(as_series=False))
+
+    kernel = _KERNELS[hrf_model]
+    frame_times = np.arange(run_length) / sampling_freq
+    dm = make_first_level_design_matrix(
+        frame_times,
+        events=events,
+        hrf_model=kernel,
+        drift_model=None,
+    )
+    if "constant" in dm.columns:
+        dm = dm.drop(columns=["constant"])
+    # nilearn suffixes a column with the name of the function that convolved
+    # it when the model is a callable; nltools names every convolved column
+    # `<col>_c0` regardless of kernel, so strip it back off.
+    suffix = "" if isinstance(kernel, str) else f"_{kernel.__name__}"
+    return pl.DataFrame(
+        {f"{str(c).removesuffix(suffix)}_c0": dm[c].to_numpy() for c in dm.columns}
+    )
+
+
+def _separator_for_path(path: str | Path) -> str:
+    """Return the delimiter a text DesignMatrix file uses, from its extension.
+
+    The single source of truth for both `write` and `_load_from_file`, so a
+    file nltools writes is always a file nltools can read back. ``.csv`` means
+    comma; every other extension means tab, matching the BIDS convention for
+    ``.tsv`` and keeping the historical default for ``.txt`` and friends.
+
+    Args:
+        path (str | Path): File path whose extension decides the delimiter.
+
+    Returns:
+        str: ``','`` for `.csv`, ``'\\t'`` otherwise.
+    """
+    return "," if Path(path).suffix.lower() == ".csv" else "\t"
+
+
+def _read_delimited(path: Path, sep: str) -> pl.DataFrame:
+    """Read a delimited text file, rejecting a separator its extension belies.
+
+    Args:
+        path (Path): File to read.
+        sep (str): Delimiter the extension implies.
+
+    Returns:
+        pl.DataFrame: The parsed table.
+
+    Raises:
+        ValueError: If the file parses as a single column whose name still
+            holds the other delimiter — the file's separator does not match
+            its extension.
+    """
+    raw = pl.read_csv(
+        path,
+        separator=sep,
+        null_values=["n/a", "N/A", "NA", ""],
+        infer_schema_length=10_000,
+    )
+    alternate = "," if sep == "\t" else "\t"
+    if raw.width == 1 and alternate in raw.columns[0]:
+        shown = {",": "','", "\t": "tab"}
+        expected = ".tsv" if alternate == "\t" else ".csv"
+        raise ValueError(
+            f"{path.name} parsed as a single column with the {shown[sep]} "
+            f"separator its extension implies, but its header contains "
+            f"{shown[alternate]}. The file's separator does not match its "
+            f"extension: rename it to {expected}, or rewrite it with "
+            f"DesignMatrix.write(name, sep=...) using the delimiter the "
+            f"extension implies."
+        )
+    return raw
+
+
+def _load_from_file(
+    path: str | Path,
+    *,
+    run_length: int | str,
+    sampling_freq: float,
+    hrf_model: str | None = None,
+) -> tuple[pl.DataFrame, bool]:
+    """Read a TSV/CSV into the frame a DesignMatrix wraps.
+
+    Dispatches on column inspection: when `onset` and `duration` are both
+    present the file is a BIDS events table and becomes an experimental design
+    — HRF-convolved by nilearn when `hrf_model` names a model, raw boxcars via
+    `events_to_dm` when it is `None` — otherwise it is a tabular file
+    (confounds / nuisance regressors) read as-is.
+
+    ``run_length='infer'`` is accepted only for the tabular path; events
+    files must provide an explicit integer (they have a variable row count
+    per run, unlike confounds which are 1 row per TR).
+
+    Args:
+        path (str | Path): Path to a `.tsv` or `.csv` file.
+        run_length (int | str): Number of TRs, or ``'infer'`` for tabular inputs.
+        sampling_freq (float): Sampling frequency in Hz (= 1/TR).
+        hrf_model (str | None): HRF model name to convolve an events table
+            with, or ``None`` for raw boxcars. Ignored for tabular files.
+
+    Returns:
+        tuple[pl.DataFrame, bool]: `(frame, is_events)` — `is_events` signals to
+            the caller that the columns are experimental regressors rather than
+            nuisance.
+    """
+    from nltools.io.events import events_to_dm
+
+    p = Path(path)
+    raw = _read_delimited(p, _separator_for_path(p))
+
+    is_events = "onset" in raw.columns and "duration" in raw.columns
+
+    if is_events:
+        if run_length == "infer":
+            raise ValueError(
+                "run_length='infer' is not valid for BIDS events files "
+                "(the row count is the number of events, not the number "
+                "of TRs). Pass an explicit integer run_length."
+            )
+        if hrf_model is None:
+            data_df = events_to_dm(
+                raw,
+                run_length=int(run_length),
+                sampling_freq=sampling_freq,
+            )
+        else:
+            data_df = _events_to_convolved_dm(
+                raw,
+                run_length=int(run_length),
+                sampling_freq=sampling_freq,
+                hrf_model=hrf_model,
+            )
+        return data_df, True
+
+    if run_length != "infer":
+        rl = int(run_length)
+        if raw.height != rl:
+            raise ValueError(
+                f"Tabular file {p.name} has {raw.height} rows but "
+                f"run_length={rl}. Pass run_length='infer' to accept "
+                f"whatever the file contains."
+            )
+    return raw, False
+
+
+def _to_pandas(dm: DesignMatrix):
+    """Build the pandas table required by nilearn's GLM boundary."""
+    import pandas as pd
+
+    return pd.DataFrame(dm.data.to_dict(as_series=False), index=range(dm.shape[0]))
+
+
+def _to_numpy(dm: DesignMatrix) -> np.ndarray:
+    """Convert a DesignMatrix to a NumPy array.
+
+    Returns the data columns as a 2D array (rows x columns), preserving the
+    DataFrame's column order.
+
+    Args:
+        dm (DesignMatrix): DesignMatrix instance.
+
+    Returns:
+        np.ndarray: 2D array with shape ``(n_samples, n_columns)``.
+
+    Examples:
+        ```python
+        dm = DesignMatrix({"a": [1, 2, 3], "b": [4, 5, 6]}, sampling_freq=1)
+        arr = to_numpy(dm)
+        arr.shape  # → (3, 2)
+        ```
+    """
+    # np.asarray(dm) routes through DesignMatrix.__array__, which knows how to
+    # honor the recorded length of a column-less matrix (polars itself would
+    # report (0, 0)).
+    return np.asarray(dm)
+
+
+def _write(dm: DesignMatrix, file_name: str, sep: str | None = None) -> None:
+    """Write DesignMatrix to file.
+
+    Supports TSV, CSV, and HDF5 formats. The format is automatically
+    determined by file extension.
+
+    Args:
+        dm (DesignMatrix): DesignMatrix instance.
+        file_name (str): Output file path with a `.tsv`, `.csv`, `.h5`, or
+            `.hdf5` extension.
+        sep (str | None): Column separator for text files. Defaults to the
+            delimiter the extension implies (comma for `.csv`, tab otherwise),
+            so the file reads back correctly; pass a value to override.
+            Ignored for HDF5.
+
+    Examples:
+        ```python
+        dm = DesignMatrix(np.random.randn(100, 3), sampling_freq=1)
+        write(dm, "design_matrix.tsv")  # tab separated (BIDS compatible)
+        write(dm, "design_matrix.csv")  # comma separated
+        write(dm, "design_matrix.h5")   # HDF5, metadata preserved
+        ```
+
+    Note:
+        TSV format is recommended for BIDS compatibility. Text formats carry
+        the data only — HDF5 additionally preserves ``sampling_freq``,
+        ``.convolved``, ``.confounds``, ``.multi``, and the row count of a
+        column-less matrix, so ``DesignMatrix(path)`` restores the object.
+    """
+    from pathlib import Path
+
+    from nltools.io.h5 import _is_h5_path
+
+    if isinstance(file_name, Path):
+        file_name = str(file_name)
+
+    if _is_h5_path(file_name):
+        _write_h5(dm, file_name)
+    else:
+        if dm.shape[1] == 0:
+            raise ValueError(
+                "Text export requires at least one column; use HDF5 to preserve observations."
+            )
+        # Write as delimited text file. The separator follows the extension by
+        # default so `write` and the file constructor cannot disagree.
+        dm.data.write_csv(
+            file_name, separator=_separator_for_path(file_name) if sep is None else sep
+        )
+
+
+def _write_h5(dm: DesignMatrix, file_name: str) -> None:
+    """Write DesignMatrix to HDF5 file with metadata.
+
+    The frame is stored as Arrow IPC bytes (via the shared
+    `nltools.io.h5` helpers) so every dtype round-trips exactly — an integer
+    spike indicator comes back an integer rather than being floated by a
+    detour through a homogeneous numpy array.
+
+    Args:
+        dm (DesignMatrix): DesignMatrix instance.
+        file_name (str): Output HDF5 file path.
+    """
+    import h5py
+
+    from nltools.io.h5 import _write_polars_frame
+
+    with h5py.File(file_name, "w") as f:
+        _write_polars_frame(f, "data", dm.data, "gzip")
+
+        meta = f.create_group("metadata")
+        if dm.sampling_freq is not None:
+            meta.attrs["sampling_freq"] = dm.sampling_freq
+        meta.attrs["convolved"] = np.array(
+            dm.convolved, dtype=h5py.string_dtype("utf-8")
+        )
+        meta.attrs["confounds"] = np.array(
+            dm.confounds, dtype=h5py.string_dtype("utf-8")
+        )
+        meta.attrs["multi"] = dm.multi
+        meta.attrs["run_count"] = dm._run_count
+        # A column-less matrix still describes a specific number of
+        # timepoints, and polars cannot carry that in the frame itself.
+        if dm._n_rows is not None:
+            meta.attrs["n_rows"] = dm._n_rows
+        meta.attrs["obj_type"] = "design_matrix"
+
+
+def _read_h5(file_name: str | Path) -> tuple[pl.DataFrame, dict]:
+    """Read a DesignMatrix HDF5 file written by `_write_h5`.
+
+    Args:
+        file_name (str | Path): Path to the HDF5 file.
+
+    Returns:
+        tuple[pl.DataFrame, dict]: `(frame, metadata)`, where metadata holds
+            ``sampling_freq``, ``convolved``, ``confounds``, ``multi``, and
+            ``n_rows`` — absent keys meaning the file didn't record them.
+    """
+    import h5py
+
+    from nltools.io.h5 import _read_polars_frame
+
+    def _decode(values) -> list[str]:
+        return [v.decode() if isinstance(v, bytes) else str(v) for v in values]
+
+    with h5py.File(file_name, "r") as f:
+        data = _read_polars_frame(f, "data")
+
+        metadata: dict = {}
+        if "metadata" in f:
+            attrs = f["metadata"].attrs
+            if "sampling_freq" in attrs:
+                metadata["sampling_freq"] = float(attrs["sampling_freq"])
+            if "convolved" in attrs:
+                metadata["convolved"] = _decode(attrs["convolved"])
+            if "confounds" in attrs:
+                metadata["confounds"] = _decode(attrs["confounds"])
+            if "multi" in attrs:
+                metadata["multi"] = bool(attrs["multi"])
+            if "run_count" in attrs:
+                metadata["run_count"] = int(attrs["run_count"])
+            if "n_rows" in attrs:
+                metadata["n_rows"] = int(attrs["n_rows"])
+
+    return data, metadata