dfc-kit 1.0.0__py3-none-any.whl

dfckit/commands/states.py

@@ -0,0 +1,266 @@
+"""Command handlers for fitting, decoding, summarizing, and scoring state models."""
+
+from __future__ import annotations
+
+import argparse
+
+from ..artifacts import (
+    StatePredictions,
+    load_fitted_model,
+    load_state_predictions,
+    save_fitted_model,
+    save_state_predictions,
+    write_state_metrics,
+    write_state_model_scores,
+)
+from ..artifacts.state_scoring import _state_model_specification
+from ..states.hmm import GaussianHMMStateModel
+from ..states.kmeans import KMeansStateModel
+from ..states.metrics import summarize_state_assignments
+from ..states.streaming import (
+    fit_kmeans_store_materialized,
+    fit_minibatch_kmeans_store,
+    predict_kmeans_store,
+    score_kmeans_store,
+)
+from ..states.streaming_hmm import (
+    fit_gaussian_hmm_store,
+    predict_gaussian_hmm_store,
+    score_gaussian_hmm_store,
+)
+from ..storage import FeatureStore
+
+
+def fit_states(namespace: argparse.Namespace) -> dict[str, object]:
+    """Fit one state model from a disk-backed feature store."""
+    store = FeatureStore.open(namespace.store)
+    if namespace.output.exists() or namespace.output.is_symlink():
+        raise FileExistsError(f"model artifact path already exists: {namespace.output}")
+    subjects = None if namespace.subject is None else tuple(namespace.subject)
+    if subjects is not None:
+        missing = sorted(set(subjects) - set(store.subjects))
+        if missing:
+            raise ValueError(f"requested fit subjects are absent from the store: {missing}")
+
+    if namespace.method == "kmeans":
+        if namespace.fitting_mode == "streaming":
+            if namespace.algorithm != "minibatch":
+                raise ValueError("streaming KMeans requires --algorithm minibatch")
+            fit = fit_minibatch_kmeans_store(
+                store,
+                n_states=namespace.n_states,
+                seed=namespace.seed,
+                n_init=10 if namespace.n_init is None else namespace.n_init,
+                max_iter=namespace.max_iter,
+                batch_size=namespace.batch_size,
+                standardize_features=namespace.standardize_features,
+                reassignment_ratio=namespace.reassignment_ratio,
+                init_sample_size=namespace.init_sample_size,
+                n_pca_components=namespace.n_pca_components,
+                pca_batch_size=namespace.pca_batch_size,
+                subjects=subjects,
+            )
+        else:
+            if namespace.init_sample_size is not None:
+                raise ValueError("materialized KMeans does not use --init-sample-size")
+            fit = fit_kmeans_store_materialized(
+                store,
+                n_states=namespace.n_states,
+                seed=namespace.seed,
+                n_init=10 if namespace.n_init is None else namespace.n_init,
+                max_iter=namespace.max_iter,
+                algorithm=namespace.algorithm,
+                batch_size=namespace.batch_size,
+                standardize_features=namespace.standardize_features,
+                reassignment_ratio=namespace.reassignment_ratio,
+                n_pca_components=namespace.n_pca_components,
+                subjects=subjects,
+            )
+        model = fit.model
+        fit_sequence_count = len(fit.assignments.sequences)
+        artifact = save_fitted_model(model, namespace.output)
+        return {
+            "model_kind": "kmeans-state",
+            "method": namespace.method,
+            "fitting_mode": namespace.fitting_mode,
+            "output": str(artifact),
+            "fit_subjects": list(model.fit_subjects),
+            "fit_sample_count": model.fit_sample_count,
+            "fit_sequence_count": fit_sequence_count,
+            "n_states": model.n_states,
+            "seed": model.seed,
+            "converged": None,
+            "log_likelihood": None,
+            "inertia": model.inertia,
+            "init_sample_size": model.init_sample_size,
+            "n_pca_components": model.n_pca_components,
+            "pca_batch_size": model.pca_batch_size,
+        }
+
+    if namespace.n_pca_components is None:
+        raise ValueError("--n-pca-components is required when --method hmm")
+    fit = fit_gaussian_hmm_store(
+        store,
+        n_states=namespace.n_states,
+        seed=namespace.seed,
+        n_pca_components=namespace.n_pca_components,
+        covariance_type=namespace.covariance_type,
+        n_init=1 if namespace.n_init is None else namespace.n_init,
+        n_iter=namespace.n_iter,
+        tol=namespace.tol,
+        pca_batch_size=namespace.pca_batch_size,
+        subjects=subjects,
+        minimum_sequence_length=namespace.minimum_sequence_length,
+    )
+    model = fit.model
+    artifact = save_fitted_model(model, namespace.output)
+    return {
+        "model_kind": "gaussian-hmm-state",
+        "method": namespace.method,
+        "output": str(artifact),
+        "fit_subjects": list(model.fit_subjects),
+        "fit_sample_count": model.fit_sample_count,
+        "fit_sequence_count": model.fit_sequence_count,
+        "n_states": model.n_states,
+        "seed": model.seed,
+        "converged": model.converged,
+        "log_likelihood": model.log_likelihood,
+        "n_pca_components": model.n_pca_components,
+        "omitted_short_sequence_count": model.omitted_short_sequence_count,
+        "pca_batch_size": model.pca_batch_size,
+    }
+
+
+def selected_store_subjects(
+    store: FeatureStore,
+    requested: list[str] | None,
+) -> tuple[str, ...] | None:
+    """Validate and retain an optional store-subject selection."""
+    if requested is None:
+        return None
+    missing = sorted(set(requested) - set(store.subjects))
+    if missing:
+        raise ValueError(f"requested prediction subjects are absent from the store: {missing}")
+    return tuple(requested)
+
+
+def predict_states(namespace: argparse.Namespace) -> dict[str, object]:
+    """Decode a feature store with a fitted state model."""
+    store = FeatureStore.open(namespace.store)
+    model = load_fitted_model(namespace.model)
+    subjects = selected_store_subjects(store, namespace.subject)
+    if isinstance(model, KMeansStateModel):
+        assignments = predict_kmeans_store(
+            model,
+            store,
+            subjects=subjects,
+            allow_fit_subjects=namespace.allow_fit_subjects,
+        )
+        predictions = StatePredictions(
+            assignments=assignments,
+            model_kind="kmeans-state",
+            model_seed=model.seed,
+        )
+    elif isinstance(model, GaussianHMMStateModel):
+        result = predict_gaussian_hmm_store(
+            model,
+            store,
+            subjects=subjects,
+            allow_fit_subjects=namespace.allow_fit_subjects,
+        )
+        predictions = StatePredictions(
+            assignments=result.assignments,
+            model_kind="gaussian-hmm-state",
+            model_seed=model.seed,
+            posterior_probabilities=result.posterior_probabilities,
+            log_likelihood=result.log_likelihood,
+        )
+    else:
+        raise TypeError("predict-states requires a KMeans or Gaussian HMM model artifact")
+    artifact = save_state_predictions(predictions, namespace.output)
+    return {
+        "model_kind": predictions.model_kind,
+        "output": str(artifact),
+        "subjects": list(predictions.subjects),
+        "n_sequences": predictions.n_sequences,
+        "n_samples": predictions.n_samples,
+        "allow_fit_subjects": bool(namespace.allow_fit_subjects),
+        "log_likelihood": predictions.log_likelihood,
+    }
+
+
+def summarize_states(namespace: argparse.Namespace) -> dict[str, object]:
+    """Write acquisition-level state metrics."""
+    predictions = load_state_predictions(namespace.predictions)
+    output = write_state_metrics(predictions, namespace.output)
+    metrics = summarize_state_assignments(predictions.assignments)
+    return {
+        "output": str(output),
+        "model_kind": predictions.model_kind,
+        "n_states": predictions.assignments.n_states,
+        "n_runs": len(metrics),
+        "n_sequences": predictions.n_sequences,
+        "n_samples": predictions.n_samples,
+        "subjects": list(predictions.subjects),
+    }
+
+
+def score_states(namespace: argparse.Namespace) -> dict[str, object]:
+    """Score a fitted state model on selected acquisitions."""
+    if namespace.output.exists() or namespace.output.is_symlink():
+        raise FileExistsError(f"state-model score output path already exists: {namespace.output}")
+    store = FeatureStore.open(namespace.store)
+    model = load_fitted_model(namespace.model)
+    subjects = selected_store_subjects(store, namespace.subject)
+    if isinstance(model, KMeansStateModel):
+        scores = score_kmeans_store(
+            model,
+            store,
+            subjects=subjects,
+            allow_fit_subjects=namespace.allow_fit_subjects,
+        )
+        model_kind = "kmeans-state"
+        minimum_sequence_length = 1
+    elif isinstance(model, GaussianHMMStateModel):
+        scores = score_gaussian_hmm_store(
+            model,
+            store,
+            subjects=subjects,
+            allow_fit_subjects=namespace.allow_fit_subjects,
+        )
+        model_kind = "gaussian-hmm-state"
+        minimum_sequence_length = model.minimum_sequence_length
+    else:
+        raise TypeError("score-states requires a KMeans or Gaussian HMM model artifact")
+    scored_subjects = tuple(dict.fromkeys(score.subject for score in scores))
+    selected_subject_set = set(scored_subjects)
+    omitted_short_sequence_count = sum(
+        identity[0] in selected_subject_set and count < minimum_sequence_length
+        for identity, count in store.sequence_sample_counts
+    )
+    output = write_state_model_scores(
+        scores,
+        namespace.output,
+        model_kind=model_kind,
+        model_seed=model.seed,
+        n_states=model.n_states,
+        fit_subjects=model.fit_subjects,
+        source_contract=store.source_contract,
+        sample_interval_seconds=store.sample_interval_seconds,
+        minimum_sequence_length=minimum_sequence_length,
+        omitted_short_sequence_count=omitted_short_sequence_count,
+        allow_fit_subjects=namespace.allow_fit_subjects,
+        model_specification=_state_model_specification(model),
+    )
+    return {
+        "output": str(output),
+        "model_kind": model_kind,
+        "n_states": model.n_states,
+        "subjects": list(scored_subjects),
+        "n_runs": len(scores),
+        "n_sequences": sum(score.n_sequences for score in scores),
+        "n_samples": sum(score.n_samples for score in scores),
+        "minimum_sequence_length": minimum_sequence_length,
+        "omitted_short_sequence_count": omitted_short_sequence_count,
+        "allow_fit_subjects": bool(namespace.allow_fit_subjects),
+    }

dfckit/connectivity/__init__.py

@@ -0,0 +1,68 @@
+"""Connectivity estimators and graph summaries of connectivity matrices."""
+
+from .correlation import edge_index, fisher_z_edges, weighted_correlation
+from .instantaneous import (
+    ETS,
+    MTD,
+    InstantaneousEdgeResult,
+)
+from .leida import (
+    LEiDA,
+    LEiDAResult,
+)
+from .lowrank import (
+    LowRankCovariance,
+    LowRankCovarianceResult,
+    StandardizedPCA,
+    bidirectional_heldout_r2,
+    effective_rank,
+    eigen_concentration,
+    fit_standardized_pca,
+    heldout_reconstruction_r2,
+    mean_projector_basis,
+    subspace_distance,
+    subspace_similarity,
+)
+from .partition import (
+    FixedPartitionGraph,
+    FixedPartitionGraphResult,
+    PartitionEdgeSummary,
+    achieved_density,
+    fixed_partition_modularity,
+    participation_coefficient,
+    partition_edge_summary,
+    positive_proportional_adjacency,
+)
+from .windows import SlidingWindowFC, WindowFCResult
+
+__all__ = [
+    "ETS",
+    "MTD",
+    "FixedPartitionGraph",
+    "FixedPartitionGraphResult",
+    "InstantaneousEdgeResult",
+    "LEiDA",
+    "LEiDAResult",
+    "LowRankCovariance",
+    "LowRankCovarianceResult",
+    "PartitionEdgeSummary",
+    "SlidingWindowFC",
+    "StandardizedPCA",
+    "WindowFCResult",
+    "achieved_density",
+    "bidirectional_heldout_r2",
+    "edge_index",
+    "effective_rank",
+    "eigen_concentration",
+    "fisher_z_edges",
+    "fit_standardized_pca",
+    "fixed_partition_modularity",
+    "heldout_reconstruction_r2",
+    "mean_projector_basis",
+    "participation_coefficient",
+    "partition_edge_summary",
+    "positive_proportional_adjacency",
+    "subspace_distance",
+    "subspace_similarity",
+    "weighted_correlation",
+]

dfckit/connectivity/_edge_products.py

@@ -0,0 +1,130 @@
+"""Shared instantaneous edge-product kernels.
+
+ETS and MTD differ in how they construct their ROI samples. Once those
+samples have been constructed, both methods use the same upper-triangular
+outer-product operation.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+
+import numpy as np
+from numpy.typing import ArrayLike, NDArray
+
+from .._arrays import readonly_copy as _readonly
+from .._validation import validated_roi_indices
+from .correlation import edge_index
+
+
+def _validated_samples(samples: ArrayLike) -> NDArray[np.float64]:
+    values = np.asarray(samples, dtype=float)
+    if values.ndim != 2 or values.shape[1] < 2:
+        raise ValueError("samples must be a frames-by-ROI array")
+    if not np.isfinite(values).all():
+        raise ValueError("samples contain non-finite values")
+    return values
+
+
+def edge_products(samples: ArrayLike) -> NDArray[np.float64]:
+    """Return all upper-triangular instantaneous ROI edge products.
+
+    The rows may be segment-standardized signals (ETS) or standardized
+    temporal derivatives (MTD); the kernel intentionally has no method-specific
+    temporal semantics.
+    """
+    values = _validated_samples(samples)
+    left, right = edge_index(values.shape[1])
+    products = values[:, left] * values[:, right]
+    if not np.isfinite(products).all():
+        raise ValueError("edge products contain non-finite values")
+    return _readonly(products)
+
+
+def edge_rss(samples: ArrayLike) -> NDArray[np.float64]:
+    """Compute all-edge root-sum-square amplitude without materializing edges."""
+    values = _validated_samples(samples)
+    squared = values * values
+    squared_rss = ((squared.sum(axis=1) ** 2 - (squared * squared).sum(axis=1)) / 2).clip(min=0.0)
+    return _readonly(np.sqrt(squared_rss))
+
+
+def cross_block_products(
+    samples: ArrayLike,
+    left: Iterable[int],
+    right: Iterable[int],
+) -> NDArray[np.float64]:
+    """Return the mean product across two disjoint ROI blocks per row."""
+    values = _validated_samples(samples)
+    left_nodes = validated_roi_indices(left, n_rois=values.shape[1], label="left block", minimum=1)
+    right_nodes = validated_roi_indices(
+        right, n_rois=values.shape[1], label="right block", minimum=1
+    )
+    overlap = sorted(set(left_nodes.tolist()).intersection(right_nodes.tolist()))
+    if overlap:
+        raise ValueError(f"cross-block products require disjoint blocks; overlap={overlap}")
+    products = (values[:, left_nodes, None] * values[:, None, right_nodes]).mean(axis=(1, 2))
+    return _readonly(products)
+
+
+def within_block_products(
+    samples: ArrayLike,
+    nodes: Iterable[int],
+) -> NDArray[np.float64]:
+    """Return the off-diagonal mean product within one ROI block per row."""
+    values = _validated_samples(samples)
+    selected = validated_roi_indices(nodes, n_rois=values.shape[1], label="within block", minimum=2)
+    left, right = np.triu_indices(len(selected), k=1)
+    products = (values[:, selected[left]] * values[:, selected[right]]).mean(axis=1)
+    return _readonly(products)
+
+
+def _edge_block_mask(
+    result: object,
+    left_nodes: NDArray[np.int64],
+    right_nodes: NDArray[np.int64],
+) -> NDArray[np.bool_]:
+    edge_i = np.asarray(result.edge_i, dtype=np.int64)
+    edge_j = np.asarray(result.edge_j, dtype=np.int64)
+    return np.isin(edge_i, left_nodes) & np.isin(edge_j, right_nodes) | (
+        np.isin(edge_i, right_nodes) & np.isin(edge_j, left_nodes)
+    )
+
+
+def cross_block_edge_summary(
+    result: object,
+    left: Iterable[int],
+    right: Iterable[int],
+) -> NDArray[np.float64]:
+    """Summarize cross-block products from a common edge result."""
+    n_rois = len(result.roi_names)
+    left_nodes = validated_roi_indices(left, n_rois=n_rois, label="left block", minimum=1)
+    right_nodes = validated_roi_indices(right, n_rois=n_rois, label="right block", minimum=1)
+    overlap = sorted(set(left_nodes.tolist()).intersection(right_nodes.tolist()))
+    if overlap:
+        raise ValueError(f"cross-block products require disjoint blocks; overlap={overlap}")
+    if result.features is not None:
+        selected = _edge_block_mask(result, left_nodes, right_nodes)
+        features = np.asarray(result.features, dtype=float)
+        if not np.any(selected):
+            raise ValueError("cross-block product summary selected no edges")
+        return _readonly(features[:, selected].mean(axis=1))
+    raise ValueError("cross-block summary requires materialized edge features")
+
+
+def within_block_edge_summary(
+    result: object,
+    nodes: Iterable[int],
+) -> NDArray[np.float64]:
+    """Summarize within-block products from a common edge result."""
+    n_rois = len(result.roi_names)
+    selected_nodes = validated_roi_indices(nodes, n_rois=n_rois, label="within block", minimum=2)
+    if result.features is not None:
+        edge_i = np.asarray(result.edge_i, dtype=np.int64)
+        edge_j = np.asarray(result.edge_j, dtype=np.int64)
+        selected = np.isin(edge_i, selected_nodes) & np.isin(edge_j, selected_nodes)
+        features = np.asarray(result.features, dtype=float)
+        if not np.any(selected):
+            raise ValueError("within-block product summary selected no edges")
+        return _readonly(features[:, selected].mean(axis=1))
+    raise ValueError("within-block summary requires materialized edge features")

dfckit/connectivity/correlation.py

@@ -0,0 +1,61 @@
+"""Correlation matrices and edge-vector utilities."""
+
+from __future__ import annotations
+
+import numpy as np
+from numpy.typing import ArrayLike, NDArray
+
+
+def edge_index(n_rois: int) -> tuple[NDArray[np.int64], NDArray[np.int64]]:
+    if n_rois < 2:
+        raise ValueError("at least two ROIs are required")
+    left, right = np.triu_indices(n_rois, k=1)
+    return left.astype(np.int64), right.astype(np.int64)
+
+
+def weighted_correlation(
+    values: ArrayLike,
+    weights: ArrayLike | None = None,
+) -> NDArray[np.float64]:
+    """Compute a finite weighted Pearson correlation matrix."""
+    data = np.asarray(values, dtype=float)
+    if data.ndim != 2 or data.shape[0] < 2 or data.shape[1] < 2:
+        raise ValueError("values must have at least two frames and two ROIs")
+    if not np.isfinite(data).all():
+        raise ValueError("values contain non-finite samples")
+
+    if weights is None:
+        weight = np.ones(data.shape[0], dtype=float)
+    else:
+        weight = np.asarray(weights, dtype=float)
+    if weight.ndim != 1 or len(weight) != len(data):
+        raise ValueError("weights must be one-dimensional and match the frame count")
+    if not np.isfinite(weight).all() or np.any(weight < 0) or weight.sum() <= 0:
+        raise ValueError("weights must be finite, non-negative, and have positive sum")
+
+    mean = np.average(data, axis=0, weights=weight)
+    centered = data - mean
+    covariance = (centered * weight[:, None]).T @ centered / weight.sum()
+    variance = np.diag(covariance)
+    if np.any(~np.isfinite(variance)) or np.any(variance <= 1e-12):
+        raise ValueError("correlation is undefined for a constant or invalid ROI")
+    scale = np.sqrt(variance)
+    correlation = covariance / np.outer(scale, scale)
+    np.fill_diagonal(correlation, 1.0)
+    if not np.isfinite(correlation).all():
+        raise ValueError("correlation matrix contains non-finite values")
+    return np.clip(correlation, -1.0, 1.0)
+
+
+def fisher_z_edges(
+    correlation: ArrayLike,
+) -> tuple[NDArray[np.float64], NDArray[np.int64], NDArray[np.int64]]:
+    """Return finite upper-triangular Fisher-z edges and their node indices."""
+    matrix = np.asarray(correlation, dtype=float)
+    if matrix.ndim != 2 or matrix.shape[0] != matrix.shape[1]:
+        raise ValueError("correlation must be a square matrix")
+    if not np.isfinite(matrix).all():
+        raise ValueError("correlation contains non-finite values")
+    left, right = edge_index(matrix.shape[0])
+    edges = np.arctanh(np.clip(matrix[left, right], -0.999999, 0.999999))
+    return edges, left, right