pysmartcor 1.0.0__py3-none-any.whl

pysmartcor/LICENSES/semopy.txt

@@ -0,0 +1,30 @@
+smartcor's `_polycorr.polychoric_corr` and `_polycorr.polyserial_corr_mle`
+are derived from semopy 2.3.11 (`semopy/polycorr.py`), distributed under the
+MIT licence below. The vendored functions are modified to replace the removed
+`scipy.stats.mvn.mvnun` helper with `scipy.stats.multivariate_normal.cdf` and
+to correct a scale-vs-variance bug in the polyserial MLE (semopy passes the
+variance to `scipy.stats.norm.logpdf`'s `scale` argument, which expects an SD).
+
+----- semopy LICENSE (verbatim) -----
+
+MIT License
+
+Copyright (c) 2021 Georgy Andreyevich Meshcheryakov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pysmartcor/__init__.py

@@ -0,0 +1,54 @@
+"""smartcor: correlation methods matched to variable types.
+
+Detects each variable's type (continuous, count, binary, ordinal,
+categorical) and picks a defensible correlation method for every pair,
+with the reasoning attached. Fourteen methods: Pearson, Spearman, Kendall,
+point-biserial, phi, tetrachoric, polychoric, polyserial, Cramer's V,
+Theil's U, Tschuprow's T, Yule's Q, Goodman-Kruskal gamma, and
+rank-biserial.
+
+Each pairwise result carries the full inference payload (estimate, test
+statistic, p-value, confidence interval, citation for the CI source) in a
+20-column tidy DataFrame matching the R package's ``tidy.smartcor()``
+output.
+
+Quick start::
+
+    from pysmartcor import smart_cor, smart_cormat
+
+    res = smart_cor(x, y, verbose=False)
+    res.estimate   # point estimate
+    res.ci_lower, res.ci_upper
+    res.tidy()     # 20-column DataFrame
+
+    mat = smart_cormat(df, verbose=False)
+    mat.correlations
+    mat.to_long()  # tidy long-format DataFrame
+
+For visualization, install with ``pip install smartcor[viz]`` and use::
+
+    from pysmartcor.viz import cor_heatmap, method_heatmap
+"""
+
+from pysmartcor.compare import compare_methods
+from pysmartcor.core import smart_cor, smart_cor_df, smart_cormat
+from pysmartcor.detect_type import detect_type
+from pysmartcor.methods import available_methods
+from pysmartcor.normality import NormalityTestResult, test_bivariate_normality
+from pysmartcor.results import SmartCorMatResult, SmartCorResult, TIDY_COLUMNS
+
+__version__ = "1.0.0"
+
+__all__ = [
+    "detect_type",
+    "smart_cor",
+    "smart_cormat",
+    "smart_cor_df",
+    "available_methods",
+    "compare_methods",
+    "test_bivariate_normality",
+    "SmartCorResult",
+    "SmartCorMatResult",
+    "NormalityTestResult",
+    "TIDY_COLUMNS",
+]

pysmartcor/_polycorr.py

@@ -0,0 +1,418 @@
+"""Polychoric and polyserial correlation: self-contained MLE implementations.
+
+The polychoric MLE here is a near-verbatim port of ``semopy.polycorr`` from
+semopy 2.3.11 (MIT-licensed, copyright (c) 2021 Georgy A. Meshcheryakov --
+see ``smartcor/LICENSES/semopy.txt``). We vendor it for two reasons:
+
+1. semopy 2.3.11 imports ``scipy.stats.mvn`` and calls ``mvn.mvnun(...)`` to
+   evaluate bivariate-normal rectangle probabilities. ``scipy.stats.mvn`` was
+   removed in scipy 1.14, so semopy's polychoric/polyserial routines raise
+   ``AttributeError`` on any recent scipy install. We replace those calls
+   with ``scipy.stats.multivariate_normal.cdf`` via inclusion-exclusion.
+
+2. semopy's polyserial MLE has a unit bug: the standardisation step is
+   ``z = (x - mean) / variance`` (should divide by SD), and
+   ``norm.logpdf(x, mean, variance)`` passes the variance to scipy's
+   ``scale`` argument (which expects an SD). This biases the rho estimate.
+   The vendored ``polyserial_corr_mle`` below fixes both.
+
+``polyserial_corr_mle`` is the public polyserial path; the two-step
+Olsson-Drasgow-Dorans estimator (``polyserial_corr``) is kept as a
+closed-form fallback for when the optimiser fails.
+
+``polychoric_se`` and ``polyserial_se`` give the asymptotic standard error
+of rho from a central-difference Hessian of the joint negative
+log-likelihood, evaluated at the estimate and inverted. This reproduces the
+SE that R's ``polycor`` derives from the ML variance matrix.
+
+References
+----------
+* Olsson, U. (1979). Maximum likelihood estimation of the polychoric correlation
+  coefficient. *Psychometrika*, 44(4), 443-460.
+* Drasgow, F. (1986). Polychoric and polyserial correlations. In *Encyclopedia
+  of Statistical Sciences*, vol. 7, 68-74.
+* Olsson, U., Drasgow, F., & Dorans, N. J. (1982). The polyserial correlation
+  coefficient. *Psychometrika*, 47(3), 337-347.
+"""
+
+from __future__ import annotations
+
+from itertools import chain
+from typing import Sequence
+
+import numpy as np
+import pandas as pd
+from scipy.optimize import minimize_scalar
+from scipy.stats import multivariate_normal, norm
+
+
+__all__ = [
+    "estimate_intervals",
+    "polychoric_corr",
+    "polychoric_se",
+    "polyserial_corr",
+    "polyserial_corr_mle",
+    "polyserial_se",
+    "bivariate_cdf",
+    "univariate_cdf",
+]
+
+
+def bivariate_cdf(
+    lower: Sequence[float],
+    upper: Sequence[float],
+    corr: float,
+    means: Sequence[float] = (0.0, 0.0),
+    var: Sequence[float] = (1.0, 1.0),
+) -> float:
+    """``P(lower < (X, Y) < upper)`` for ``(X, Y) ~ N(means, Sigma)``.
+
+    Replaces ``scipy.stats.mvn.mvnun`` (removed in scipy 1.14). Uses the
+    inclusion-exclusion identity on the 2D CDF::
+
+        P(rect) = F(u1, u2) - F(l1, u2) - F(u1, l2) + F(l1, l2)
+
+    ``-inf`` / ``+inf`` bounds are supported natively by scipy's CDF.
+    """
+    cov = np.array([[float(var[0]), float(corr)], [float(corr), float(var[1])]])
+    mu = np.asarray(means, dtype=float)
+    # cap the correlation strictly inside (-1, 1); the optimiser sometimes probes the boundary
+    eps = 1e-9
+    if abs(cov[0, 1]) >= np.sqrt(cov[0, 0] * cov[1, 1]) - eps:
+        cov[0, 1] = np.sign(cov[0, 1]) * (np.sqrt(cov[0, 0] * cov[1, 1]) - eps)
+        cov[1, 0] = cov[0, 1]
+    rv = multivariate_normal(mean=mu, cov=cov, allow_singular=True)
+    u = np.asarray(upper, dtype=float)
+    l = np.asarray(lower, dtype=float)
+    return float(
+        rv.cdf([u[0], u[1]])
+        - rv.cdf([l[0], u[1]])
+        - rv.cdf([u[0], l[1]])
+        + rv.cdf([l[0], l[1]])
+    )
+
+
+def univariate_cdf(
+    lower: float, upper: float, mean: float = 0.0, var: float = 1.0
+) -> float:
+    """``P(lower < X < upper)`` for ``X ~ N(mean, var)`` via ``norm.cdf``."""
+    sd = np.sqrt(var)
+    return float(norm.cdf(upper, mean, sd) - norm.cdf(lower, mean, sd))
+
+
+def estimate_intervals(x, inf: float = 10.0):
+    """Estimate latent-variable thresholds for an ordinal variable.
+
+    Returns ``(thresholds, indices)`` where ``thresholds`` is the list
+    ``[-inf, tau_1, ..., tau_{K-1}, +inf]`` and ``indices[i]`` is the
+    1-based category index of observation ``x[i]``. NA entries get NaN
+    indices.
+    """
+    x = np.asarray(x)
+    x_f = x[~pd.isna(x)]
+    u, counts = np.unique(x_f, return_counts=True)
+    sz = len(x_f)
+    if sz == 0 or len(u) < 2:
+        return [-inf, inf], [np.nan] * len(x)
+    cumcounts = np.cumsum(counts[:-1])
+    cat_to_idx = {v: i + 1 for i, v in enumerate(u)}
+    indices = [np.nan if pd.isna(v) else cat_to_idx.get(v, np.nan) for v in x]
+    thresholds = list(chain([-inf], (norm.ppf(n / sz) for n in cumcounts), [inf]))
+    return thresholds, indices
+
+
+def polychoric_corr(x, y, *, x_ints=None, y_ints=None, tol: float = 1e-7) -> float:
+    """Polychoric correlation between two ordinal variables (Olsson 1979).
+
+    Two-step MLE: estimate the marginal thresholds from each variable's
+    empirical frequencies, then maximise the bivariate-normal log-likelihood
+    over the contingency table.
+
+    Adapted from ``semopy.polycorr.polychoric_corr`` (MIT), with
+    :func:`bivariate_cdf` used for current SciPy compatibility.
+    """
+    if x_ints is None:
+        x_ints = estimate_intervals(x)
+    if y_ints is None:
+        y_ints = estimate_intervals(y)
+    x_thr, x_inds = x_ints
+    y_thr, y_inds = y_ints
+    p, m = len(x_thr) - 1, len(y_thr) - 1
+    if p < 2 or m < 2:
+        return float("nan")
+    n = np.zeros((p, m))
+    for a, b in zip(x_inds, y_inds):
+        if not (np.isnan(a) or np.isnan(b)):
+            n[int(a) - 1, int(b) - 1] += 1
+
+    def calc_neg_loglik(r: float) -> float:
+        total = 0.0
+        for i in range(p):
+            for j in range(m):
+                nij = n[i, j]
+                if nij == 0:
+                    continue
+                pij = bivariate_cdf(
+                    [x_thr[i], y_thr[j]],
+                    [x_thr[i + 1], y_thr[j + 1]],
+                    r,
+                )
+                if pij <= 0:
+                    return 1e20
+                total -= np.log(pij) * nij
+        return total
+
+    res = minimize_scalar(
+        calc_neg_loglik, bounds=(-0.999, 0.999), method="bounded",
+        options={"xatol": tol},
+    )
+    return float(res.x)
+
+
+def polyserial_corr(x_continuous, x_ordinal) -> float:
+    """Two-step polyserial estimator (Olsson, Drasgow & Dorans 1982).
+
+    Closed form::
+
+        rho_hat = r_pb * sd(y_int) / sum_k phi(tau_k)
+
+    where ``r_pb`` is the Pearson correlation of the continuous variable
+    with the integer-coded ordinal, ``sd(y_int)`` is the population SD of
+    the integer codes, and ``tau_k`` are the cumulative z-thresholds
+    estimated from the ordinal's empirical marginal. No optimiser required.
+
+    R's ``polycor::polyserial(ML=FALSE)`` computes
+    ``sqrt((n-1)/n) * sd(y) * r / sum(dnorm(tau))``, which is the same
+    population-SD formula, so the two agree exactly when the ordinal codes
+    are equally spaced.
+    """
+    x = np.asarray(x_continuous, dtype=float)
+    y_raw = np.asarray(x_ordinal)
+    mask = ~(np.isnan(x) | pd.isna(y_raw))
+    x = x[mask]
+    y_raw = y_raw[mask]
+    cats, y = np.unique(y_raw, return_inverse=True)
+    K = len(cats)
+    if K < 2 or len(x) < 3:
+        return float("nan")
+    n = len(x)
+    counts = np.bincount(y, minlength=K)
+    cum = np.cumsum(counts[:-1]) / n
+    tau = norm.ppf(cum)
+    phi_sum = float(norm.pdf(tau).sum())
+    if phi_sum <= 0:
+        return float("nan")
+    r_pb = float(np.corrcoef(x, y)[0, 1])
+    s_y = float(np.std(y, ddof=0))
+    if s_y <= 0:
+        return float("nan")
+    rho = r_pb * s_y / phi_sum
+    return float(np.clip(rho, -0.999, 0.999))
+
+
+def polyserial_corr_mle(
+    x_continuous, x_ordinal, *, y_ints=None, tol: float = 1e-7
+) -> float:
+    """Polyserial MLE (Drasgow 1986), vendored from semopy with the SD fix.
+
+    Two-step: estimate the ordinal's thresholds from marginal frequencies,
+    then maximise the conditional log-likelihood
+    ``sum_i log f(x_i) + log P(y_i = c_i | x_i)`` over rho.
+
+    Notes
+    -----
+    The vendored semopy implementation standardised by the variance instead
+    of the SD and passed the variance to ``norm.logpdf``'s ``scale`` argument.
+    Both are fixed here.
+    """
+    x = np.asarray(x_continuous, dtype=float)
+    mask = ~np.isnan(x)
+    x = x[mask]
+    y_raw = np.asarray(x_ordinal)[mask]
+    if y_ints is None:
+        y_ints = estimate_intervals(y_raw)
+    thr, inds = y_ints
+    if len(thr) < 3 or len(x) < 3:
+        return float("nan")
+
+    x_mean = float(np.mean(x))
+    x_sd = float(np.std(x, ddof=0))
+    if x_sd <= 0:
+        return float("nan")
+    x_z = (x - x_mean) / x_sd
+    x_logpdf = norm.logpdf(x, x_mean, x_sd)
+
+    def transform_tau(tau, rho, z):
+        return (tau - rho * z) / np.sqrt(max(1 - rho ** 2, 1e-12))
+
+    def sub_pr(k, rho, z):
+        i = transform_tau(thr[k], rho, z)
+        j = transform_tau(thr[k - 1], rho, z)
+        return univariate_cdf(j, i)
+
+    def neg_loglik(rho: float) -> float:
+        total = 0.0
+        for z, ind, pdf in zip(x_z, inds, x_logpdf):
+            if np.isnan(ind):
+                continue
+            pr = sub_pr(int(ind), rho, z)
+            if pr <= 0:
+                return 1e20
+            total -= pdf + np.log(pr)
+        return total
+
+    res = minimize_scalar(
+        neg_loglik, bounds=(-0.999, 0.999), method="bounded",
+        options={"xatol": tol},
+    )
+    return float(res.x)
+
+
+def polychoric_se(x, y, rho: float) -> float:
+    """Asymptotic SE of a polychoric (or tetrachoric) correlation.
+
+    Central-difference Hessian of the joint negative log-likelihood in
+    ``(rho, row thresholds, column thresholds)``, evaluated at the estimate,
+    inverted; the SE is the square root of the rho diagonal entry. This is
+    the same variance matrix R's ``polycor::polychor(std.err=TRUE)`` uses
+    (Olsson 1979), reproduced numerically.
+
+    Parameters
+    ----------
+    x, y : array-like
+        The ordinal variables the estimate was computed from.
+    rho : float
+        The polychoric point estimate.
+
+    Returns
+    -------
+    float
+        The standard error, or ``NaN`` when the Hessian is singular or the
+        inputs are degenerate.
+    """
+    if np.isnan(rho) or abs(rho) >= 0.999:
+        return float("nan")
+    x_thr, x_inds = estimate_intervals(x)
+    y_thr, y_inds = estimate_intervals(y)
+    p, m = len(x_thr) - 1, len(y_thr) - 1
+    if p < 2 or m < 2:
+        return float("nan")
+    n = np.zeros((p, m))
+    for a, b in zip(x_inds, y_inds):
+        if not (np.isnan(a) or np.isnan(b)):
+            n[int(a) - 1, int(b) - 1] += 1
+    a0 = np.asarray(x_thr[1:-1], dtype=float)
+    b0 = np.asarray(y_thr[1:-1], dtype=float)
+
+    def nll(theta: np.ndarray) -> float:
+        r = theta[0]
+        a = theta[1:p]
+        b = theta[p:]
+        if abs(r) >= 0.9995 or np.any(np.diff(a) <= 0) or np.any(np.diff(b) <= 0):
+            return 1e12
+        probs = _bvn_cell_probs(
+            np.concatenate([[-np.inf], a, [np.inf]]),
+            np.concatenate([[-np.inf], b, [np.inf]]),
+            r,
+        )
+        if np.any(probs[n > 0] <= 0):
+            return 1e12
+        return float(-np.sum(n[n > 0] * np.log(probs[n > 0])))
+
+    return _se_from_hessian(nll, np.concatenate([[rho], a0, b0]))
+
+
+def polyserial_se(x_continuous, x_ordinal, rho: float) -> float:
+    """Asymptotic SE of a polyserial correlation.
+
+    Central-difference Hessian of the negative log-likelihood in
+    ``(rho, thresholds)`` with the continuous variable standardised, matching
+    the parameterisation R's ``polycor::polyserial(std.err=TRUE)`` inverts
+    (Olsson, Drasgow & Dorans 1982). Returns ``NaN`` on failure.
+    """
+    if np.isnan(rho) or abs(rho) >= 0.999:
+        return float("nan")
+    xc = np.asarray(x_continuous, dtype=float)
+    thr, inds = estimate_intervals(np.asarray(x_ordinal))
+    tau0 = np.asarray(thr[1:-1], dtype=float)
+    if len(tau0) < 1:
+        return float("nan")
+    k_idx = np.asarray(inds, dtype=float)
+    ok = ~(np.isnan(k_idx) | np.isnan(xc))
+    xc = xc[ok]
+    k_idx = k_idx[ok].astype(int)
+    if len(xc) < 4:
+        return float("nan")
+    sd = xc.std(ddof=0)
+    if sd <= 0:
+        return float("nan")
+    z = (xc - xc.mean()) / sd
+
+    def nll(theta: np.ndarray) -> float:
+        r = theta[0]
+        tau = theta[1:]
+        if abs(r) >= 0.9995 or np.any(np.diff(tau) <= 0):
+            return 1e12
+        denom = np.sqrt(max(1 - r ** 2, 1e-12))
+        tf = np.concatenate([[-np.inf], tau, [np.inf]])
+        pr = norm.cdf((tf[k_idx] - r * z) / denom) - norm.cdf(
+            (tf[k_idx - 1] - r * z) / denom
+        )
+        if np.any(pr <= 0):
+            return 1e12
+        return float(-np.sum(norm.logpdf(z) + np.log(pr)))
+
+    return _se_from_hessian(nll, np.concatenate([[rho], tau0]))
+
+
+def _bvn_cell_probs(x_thr: np.ndarray, y_thr: np.ndarray, rho: float) -> np.ndarray:
+    """Rectangle probabilities for every cell of a threshold grid.
+
+    One vectorised bivariate-normal CDF call over the full grid of threshold
+    crossings, then the inclusion-exclusion differences. Infinite thresholds
+    are clipped to +/-12 (beyond any representable tail mass).
+    """
+    xt = np.clip(np.asarray(x_thr, dtype=float), -12.0, 12.0)
+    yt = np.clip(np.asarray(y_thr, dtype=float), -12.0, 12.0)
+    gx, gy = np.meshgrid(xt, yt, indexing="ij")
+    pts = np.column_stack([gx.ravel(), gy.ravel()])
+    r = float(np.clip(rho, -1 + 1e-9, 1 - 1e-9))
+    rv = multivariate_normal(mean=[0.0, 0.0], cov=[[1.0, r], [r, 1.0]])
+    cdf = np.asarray(rv.cdf(pts)).reshape(len(xt), len(yt))
+    return cdf[1:, 1:] - cdf[:-1, 1:] - cdf[1:, :-1] + cdf[:-1, :-1]
+
+
+def _hessian(f, theta: np.ndarray) -> np.ndarray:
+    """Central-difference Hessian of ``f`` at ``theta``."""
+    theta = np.asarray(theta, dtype=float)
+    k = len(theta)
+    h = np.finfo(float).eps ** 0.25 * np.maximum(1.0, np.abs(theta))
+    H = np.empty((k, k))
+    f0 = f(theta)
+    for i in range(k):
+        ei = np.zeros(k)
+        ei[i] = h[i]
+        H[i, i] = (f(theta + ei) - 2 * f0 + f(theta - ei)) / h[i] ** 2
+        for j in range(i + 1, k):
+            ej = np.zeros(k)
+            ej[j] = h[j]
+            H[i, j] = H[j, i] = (
+                f(theta + ei + ej)
+                - f(theta + ei - ej)
+                - f(theta - ei + ej)
+                + f(theta - ei - ej)
+            ) / (4 * h[i] * h[j])
+    return H
+
+
+def _se_from_hessian(nll, theta0: np.ndarray) -> float:
+    """SE of the first parameter from the inverse Hessian of ``nll``."""
+    try:
+        H = _hessian(nll, theta0)
+        if not np.all(np.isfinite(H)):
+            return float("nan")
+        cov = np.linalg.inv(H)
+    except np.linalg.LinAlgError:
+        return float("nan")
+    v = cov[0, 0]
+    return float(np.sqrt(v)) if v > 0 else float("nan")