PsychiatryNLPKit 0.1.0__py3-none-any.whl

PsychiatryNLPKit/__init__.py

@@ -0,0 +1,34 @@
+"""Computational linguistics toolkit for psychosis risk assessment and thought disorder analysis.
+
+Provides analysis functions grounded in clinical research on language markers of
+psychosis risk, including syntax metrics, semantic coherence, perplexity, graph-based
+network measures, and semantic density estimation.  All functions support both English
+and French where applicable.
+
+Quick start::
+
+    import PsychiatryNLPKit as pnlp
+
+    pnlp.configure_logging()
+    data = pnlp.data.TextData(sections=[...], lang="en")
+    results = pnlp.analysis.sentence_length(data.pos_tags)
+
+Batch analysis::
+
+    result = pnlp.BatchAnalyzer(text_data).run()
+"""
+
+from . import analysis, data, model
+from .analysis import AnalysisResult, BatchAnalyzer
+from .config import configure_logging, device, hf_token
+
+__all__: list[str] = [
+    "AnalysisResult",
+    "BatchAnalyzer",
+    "analysis",
+    "configure_logging",
+    "data",
+    "device",
+    "hf_token",
+    "model",
+]

PsychiatryNLPKit/analysis/Density.py

@@ -0,0 +1,164 @@
+"""Semantic density analysis for PsychiatryNLPKit.
+
+Measures dimensional properties of semantic spaces derived from token embeddings.
+Schizophrenia patients can show high or low semantic density, and lower intrinsic
+dimensionality indicates more redundant speech.
+"""
+
+import logging
+
+import numpy as np
+import torch
+from skdim.id import MLE
+from sklearn.decomposition import PCA
+
+logger = logging.getLogger(__name__)
+
+
+def pca_density_metrics(
+    token_embedding_vectors: dict[str, torch.Tensor],
+    sections: list[str] | None = None,
+) -> dict[str, dict[str, float]]:
+    """PCA-based density metrics per section.
+
+    PCA is applied in **token space** (features=tokens, i.e. ``X.T``) so metrics
+    reflect how many token directions are needed to explain semantic variance.
+    During statistical analysis, total paragraph length must be controlled.
+
+    Notes:
+        Theoretical basis - While healthy controls tend to have moderately
+        compressible semantic space, schizophrenia patients can have high or low
+        semantic density (Palominos et al., 2025).
+
+    Args:
+        token_embedding_vectors: Dict mapping section names to token-level
+            embedding tensors (from ``TextData.token_embedding_vectors``).
+        sections: Sections to process. ``None`` processes all sections in the dict.
+
+    Returns:
+        Dict mapping section names to a metric dict with keys
+        ``"Ncomp_90"``, ``"Pcomp_90"``, and ``"ExVar_2"``. Empty sections
+        receive ``float("nan")`` for all metrics.
+
+    References:
+        Palominos, C., Stein, F., Kircher, T., Ayesa-Arriola, R., Palaniyappan,
+        L., Homan, P., Sommer, I. E., & Hinzen, W. (2025). Lexical meaning is
+        lower dimensional in psychosis. Scientific Reports, 16(1), 859.
+        https://doi.org/10.1038/s41598-025-30443-1
+    """
+    if sections is None:
+        sections = list(token_embedding_vectors.keys())
+
+    results: dict[str, dict[str, float]] = {}
+
+    for sec in sections:
+        if sec not in token_embedding_vectors:
+            logger.warning("Section %s not found in embeddings, skipping", sec)
+            continue
+
+        vectors = token_embedding_vectors[sec]
+        n_tokens = vectors.shape[0]
+
+        # Handle edge case: fewer than 2 tokens.
+        if n_tokens < 2:
+            results[sec] = {
+                "Ncomp_90": float("nan"),
+                "Pcomp_90": float("nan"),
+                "ExVar_2": float("nan"),
+            }
+            continue
+
+        # Move to CPU for sklearn.
+        X = vectors.detach().cpu().numpy()
+
+        # PCA in **token space**: transpose so tokens are samples, embedding dims are features.
+        n_components = min(n_tokens, X.shape[1])
+        pca = PCA(n_components=n_components)
+        pca.fit(X.T)
+
+        explained_variance_ratio = np.asarray(
+            pca.explained_variance_ratio_, dtype=float
+        )
+        cumulative_variance = np.cumsum(explained_variance_ratio)
+
+        ncomp_90 = int(np.searchsorted(cumulative_variance, 0.9, side="left") + 1)
+        pcomp_90 = ncomp_90 / n_tokens
+        exvar_2 = float(
+            explained_variance_ratio[: min(2, len(explained_variance_ratio))].sum()
+        )
+
+        results[sec] = {
+            "Ncomp_90": float(ncomp_90),
+            "Pcomp_90": float(pcomp_90),
+            "ExVar_2": exvar_2,
+        }
+
+    return results
+
+
+def intrinsic_dimensionality_density(
+    token_embedding_vectors: dict[str, torch.Tensor],
+    sections: list[str] | None = None,
+    k: int | None = None,
+) -> dict[str, dict[str, float]]:
+    """Estimate intrinsic dimensionality using MLE (Levina & Bickel, 2004).
+
+    Intrinsic dimensionality quantifies the local geometric complexity of the
+    semantic space formed by token embeddings.  Lower values indicate more
+    redundant speech.
+
+    Notes:
+        Theoretical basis - Lower intrinsic dimensionality indicates more
+        redundant speech (Palominos et al., 2025).
+
+    Args:
+        token_embedding_vectors: Dict mapping section names to token-level
+            embedding tensors (from ``TextData.token_embedding_vectors``).
+        sections: Sections to process. ``None`` processes all sections in the dict.
+        k: Number of neighbors for MLE estimator. Defaults to
+            ``min(10, n_samples - 1)``.
+
+    Returns:
+        Dict mapping section names to a metric dict with key ``"ID_MLE"``.
+        Empty sections receive ``float("nan")``.
+
+    References:
+        Palominos, C., Stein, F., Kircher, T., Ayesa-Arriola, R., Palaniyappan,
+        L., Homan, P., Sommer, I. E., & Hinzen, W. (2025). Lexical meaning is
+        lower dimensional in psychosis. Scientific Reports, 16(1), 859.
+        https://doi.org/10.1038/s41598-025-30443-1
+    """
+    if sections is None:
+        sections = list(token_embedding_vectors.keys())
+
+    results: dict[str, dict[str, float]] = {}
+
+    for sec in sections:
+        if sec not in token_embedding_vectors:
+            logger.warning("Section %s not found in embeddings, skipping", sec)
+            continue
+
+        vectors = token_embedding_vectors[sec]
+        n_tokens = vectors.shape[0]
+
+        # Handle edge case: fewer than 2 tokens.
+        if n_tokens < 2:
+            results[sec] = {"ID_MLE": float("nan")}
+            continue
+
+        # Move to CPU for skdim.
+        X = vectors.detach().cpu().numpy()
+
+        k_param = k if k is not None else min(10, n_tokens - 1)
+        id_estimator = MLE(K=k_param)
+        intrinsic_dim = float(id_estimator.fit_transform(X))
+
+        results[sec] = {"ID_MLE": intrinsic_dim}
+
+    return results
+
+
+__all__: list[str] = [
+    "pca_density_metrics",
+    "intrinsic_dimensionality_density",
+]

PsychiatryNLPKit/analysis/Graph.py

@@ -0,0 +1,302 @@
+"""Graph-based network analysis for PsychiatryNLPKit.
+
+Constructs structural word graphs from text and computes network metrics
+associated with thought disorder.  Schizophrenia patients show smaller
+connected components and lower average shortest path length.
+"""
+
+import logging
+
+import networkx as nx
+import numpy as np
+
+logger = logging.getLogger(__name__)
+
+
+def _z_score(value: float, null_distribution: list[float]) -> float:
+    """Compute z-score of *value* against a null distribution."""
+    if not null_distribution:
+        return 0.0
+    mean = np.mean(null_distribution)
+    std = np.std(null_distribution)
+    if std == 0:
+        return 0.0
+    return float((value - mean) / std)
+
+
+def _largest_component_size(
+    graph: nx.Graph | nx.DiGraph, strongly: bool = False
+) -> int:
+    """Size of largest (strongly) connected component."""
+    if graph.number_of_nodes() == 0:
+        return 0
+    if strongly and isinstance(graph, nx.DiGraph):
+        return max((len(c) for c in nx.strongly_connected_components(graph)), default=0)
+    if isinstance(graph, nx.DiGraph):
+        return max((len(c) for c in nx.weakly_connected_components(graph)), default=0)
+    return max((len(c) for c in nx.connected_components(graph)), default=0)
+
+
+def _diameter_and_aspl(
+    graph: nx.Graph | nx.DiGraph, weight: str | None = None
+) -> tuple[float, float]:
+    """Diameter and average shortest path length of largest component.
+
+    For directed graphs, converts to undirected for distance calculations
+    to ensure symmetric reachability.
+    """
+    if graph.number_of_nodes() == 0:
+        return 0.0, 0.0
+
+    if isinstance(graph, nx.DiGraph):
+        components = list(nx.weakly_connected_components(graph))
+    else:
+        components = list(nx.connected_components(graph))
+
+    if not components:
+        return 0.0, 0.0
+
+    largest = max(components, key=len)
+    subgraph = graph.subgraph(largest)
+
+    # Convert directed graphs to undirected for distance calculations.
+    distance_graph = (
+        subgraph.to_undirected() if isinstance(subgraph, nx.DiGraph) else subgraph
+    )
+
+    try:
+        diam = nx.diameter(distance_graph, weight=weight)
+        aspl = nx.average_shortest_path_length(distance_graph, weight=weight)
+    except nx.NetworkXError:
+        return 0.0, 0.0
+
+    return float(diam), float(aspl)
+
+
+def _degree_variance(graph: nx.Graph | nx.DiGraph, weight: str | None = None) -> float:
+    """Variance of node degrees (in+out for directed graphs)."""
+    if graph.number_of_nodes() == 0:
+        return 0.0
+
+    if isinstance(graph, nx.DiGraph):
+        digraph: nx.DiGraph = graph
+        degrees = np.fromiter(
+            (
+                digraph.in_degree(n, weight=weight)
+                + digraph.out_degree(n, weight=weight)
+                for n in digraph.nodes()
+            ),
+            dtype=float,
+            count=digraph.number_of_nodes(),
+        )
+    else:
+        degrees = np.fromiter(
+            (graph.degree(n, weight=weight) for n in graph.nodes()),
+            dtype=float,
+            count=graph.number_of_nodes(),
+        )
+
+    return float(degrees.var())
+
+
+def _generate_random_graphs(
+    n_nodes: int,
+    n_edges: int,
+    n_graphs: int = 1000,
+) -> list[nx.Graph | nx.DiGraph]:
+    """Generate random graphs with same node/edge counts (configuration model)."""
+    if n_nodes == 0 or n_edges == 0:
+        return []
+
+    graphs: list[nx.Graph | nx.DiGraph] = []
+    for i in range(n_graphs):
+        G = nx.gnm_random_graph(n_nodes, n_edges, directed=True, seed=i)
+        graphs.append(G)
+    return graphs
+
+
+def structural_graph(
+    content_words: dict[str, list[str]],
+    directed: bool = True,
+    weighted: bool = True,
+    sections: list[str] | None = None,
+    n_random_graphs: int = 1000,
+) -> dict[str, dict[str, float]]:
+    """Constructs a structural graph from lemmatized content words per section
+    and immediately computes all network metrics.
+
+    Notes:
+        Theoretical basis - Graphs from schizophrenia patients show fewer nodes
+        and edges, lower average degrees, and smaller connected components. The
+        graphs also have lower average shortest path length and network diameter
+        per fixed word length (Nikzad et al., 2022).
+
+    Args:
+        content_words: Dict mapping section names to lists of lemmatized content
+            words.
+        directed: Whether the constructed graph is directed (word transitions).
+            Defaults to ``True``.
+        weighted: Whether to assign edge weights based on repetition count. Edge
+            weight equals consecutive occurrence count; distance = 1/weight for
+            path calculations.
+        sections: Sections to process. ``None`` processes all sections in
+            *content_words*.
+        n_random_graphs: Number of random graphs to generate for z-score
+            computation. Defaults to 1000. **Performance note:** this incurs a
+            non negligible runtime cost. For each section, ``n_random_graphs``
+            random graphs are generated and analyzed. With N sections, total cost
+            is O(N * n_random_graphs). For large datasets, consider reducing this
+            parameter or pre-computing null distributions for common node/edge
+            counts.
+
+    Returns:
+        Dict mapping section names to a metric dict with keys: nodes_count,
+        edges_count, average_degree, density, diameter,
+        average_shortest_path_length, largest_connected_component,
+        largest_strongly_connected_component, lcc/n, lsc/n,
+        edge_weight_repetition_index, lcc_z_score, lsc_z_score,
+        aspl_z_score, degree_distribution_z_score.
+
+    References:
+        Nikzad, A. H., Cong, Y., Berretta, S., Hänsel, K., Cho, S., Pradhan, S.,
+        Behbehani, L., DeSouza, D. D., Liberman, M. Y., & Tang, S. X. (2022). Who
+        does what to whom? graph representations of action-predication in speech
+        relate to psychopathological dimensions of psychosis. Schizophrenia,
+        8(1), 58. https://doi.org/10.1038/s41537-022-00263-7
+    """
+    if sections is None:
+        sections = list(content_words.keys())
+
+    results: dict[str, dict[str, float]] = {}
+
+    for sec in sections:
+        if sec not in content_words:
+            logger.warning("Section %s not found in content_words, skipping", sec)
+            continue
+
+        graph_class = nx.DiGraph if directed else nx.Graph
+        graph = graph_class()
+
+        # Add nodes.
+        for word in content_words[sec]:
+            if not graph.has_node(word):
+                graph.add_node(word)
+
+        # Add edges with weights based on consecutive occurrence counts.
+        words = content_words[sec]
+        for i in range(len(words) - 1):
+            w1, w2 = words[i], words[i + 1]
+            if graph.has_edge(w1, w2):
+                if weighted:
+                    graph[w1][w2]["weight"] += 1
+                    graph[w1][w2]["distance"] = 1.0 / graph[w1][w2]["weight"]
+            else:
+                if weighted:
+                    graph.add_edge(w1, w2, weight=1, distance=1.0)
+                else:
+                    graph.add_edge(w1, w2)
+
+        # --- Compute all metrics for this section ---
+        nodes_count = graph.number_of_nodes()
+        edges_count = graph.number_of_edges()
+
+        section_metrics: dict[str, float] = {
+            "nodes_count": float(nodes_count),
+            "edges_count": float(edges_count),
+        }
+
+        if nodes_count == 0:
+            section_metrics.update(
+                {
+                    "average_degree": 0.0,
+                    "density": 0.0,
+                    "diameter": 0.0,
+                    "average_shortest_path_length": 0.0,
+                    "largest_connected_component": 0.0,
+                    "largest_strongly_connected_component": 0.0,
+                    "lcc/n": 0.0,
+                    "lsc/n": 0.0,
+                    "edge_weight_repetition_index": 0.0,
+                    "lcc_z_score": 0.0,
+                    "lsc_z_score": 0.0,
+                    "aspl_z_score": 0.0,
+                    "degree_distribution_z_score": 0.0,
+                }
+            )
+            results[sec] = section_metrics
+            continue
+
+        # Degree and density.
+        weighted_degree_sum = sum(
+            degree for _, degree in graph.degree(weight="weight" if weighted else None)
+        )
+        section_metrics["average_degree"] = float(weighted_degree_sum / nodes_count)
+        section_metrics["density"] = float(nx.density(graph))
+
+        # Diameter and ASPL on largest component (undirected for directed graphs).
+        diameter, aspl = _diameter_and_aspl(
+            graph, weight="distance" if weighted else None
+        )
+        section_metrics["diameter"] = diameter
+        section_metrics["average_shortest_path_length"] = aspl
+
+        # Connectivity metrics.
+        lcc = _largest_component_size(graph)
+        lsc = _largest_component_size(graph, strongly=True)
+        section_metrics["largest_connected_component"] = float(lcc)
+        section_metrics["largest_strongly_connected_component"] = float(lsc)
+        section_metrics["lcc/n"] = float(lcc / nodes_count) if nodes_count else 0.0
+        section_metrics["lsc/n"] = float(lsc / nodes_count) if nodes_count else 0.0
+
+        # Edge weight repetition index.
+        total_edge_weight = 0.0
+        repeated_weight = 0.0
+        observed_weights: list[float] = []
+        for _, _, attrs in graph.edges(data=True):
+            edge_weight = float(attrs.get("weight", 1.0))
+            observed_weights.append(edge_weight)
+            total_edge_weight += edge_weight
+            repeated_weight += max(0.0, edge_weight - 1.0)
+        section_metrics["edge_weight_repetition_index"] = (
+            repeated_weight / total_edge_weight if total_edge_weight > 0 else 0.0
+        )
+
+        # Z-scores against random graphs with SAME edge weights.
+        observed_degree_var = _degree_variance(
+            graph, weight="weight" if weighted else None
+        )
+        random_lcc, random_lsc, random_aspl, random_deg_var = [], [], [], []
+
+        for G in _generate_random_graphs(nodes_count, edges_count, n_random_graphs):
+            # Assign observed edge weights to random graph for fair comparison.
+            if weighted and observed_weights:
+                G = G.to_directed() if graph.is_directed() else G
+                for (u, v), random_weight in zip(
+                    G.edges(), observed_weights, strict=True
+                ):
+                    G[u][v]["weight"] = random_weight
+                    G[u][v]["distance"] = 1.0 / random_weight
+
+            random_lcc.append(float(_largest_component_size(G)))
+            random_lsc.append(float(_largest_component_size(G, strongly=True)))
+            _, r_aspl = _diameter_and_aspl(G, weight="distance" if weighted else None)
+            random_aspl.append(r_aspl)
+            random_deg_var.append(
+                _degree_variance(G, weight="weight" if weighted else None)
+            )
+
+        section_metrics["lcc_z_score"] = float(_z_score(float(lcc), random_lcc))
+        section_metrics["lsc_z_score"] = float(_z_score(float(lsc), random_lsc))
+        section_metrics["aspl_z_score"] = float(_z_score(aspl, random_aspl))
+        section_metrics["degree_distribution_z_score"] = float(
+            _z_score(observed_degree_var, random_deg_var)
+        )
+
+        results[sec] = section_metrics
+
+    return results
+
+
+__all__: list[str] = [
+    "structural_graph",
+]