dit 2.2__tar.gz → 2.3__tar.gz

{dit-2.2 → dit-2.3}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: dit
-Version: 2.2
+Version: 2.3
 Summary: Python package for information theory on discrete random variables.
 Project-URL: Homepage, http://dit.io
 Project-URL: Repository, https://github.com/dit/dit
@@ -10,8 +10,6 @@ License-Expression: BSD-3-Clause
 License-File: LICENSE.txt
 Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: BSD License
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13

{dit-2.2 → dit-2.3}/dit/__init__.py

@@ -10,7 +10,13 @@ Python package for sigma-algebras defined on finite sets.
 
 """
 
-__version__ = "1.5"
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as _pkg_version
+
+try:
+    __version__ = _pkg_version("dit")
+except PackageNotFoundError:  # pragma: no cover
+    __version__ = "0.0.0"
 
 from loguru import logger as _logger
 

{dit-2.2 → dit-2.3}/dit/algorithms/__init__.py

@@ -12,6 +12,17 @@ from .maxentropyfw import *
 from .minimal_sufficient_statistic import *
 from .optimization import *
 from .distribution_optimizers import *
+from .mixture_of_products import fit_mixture_of_products, mixture_of_products_dists
+from .marginal_lifts import fit_marginal_lift_mixture, lift_marginal, marginal_lift_dists
+from .mprojection import (
+    m_projection,
+    m_projection_eps_limit,
+    m_projection_from_subsets,
+    mflat_design_matrix,
+    mflat_mprojection_dists,
+    mflat_subsets_from_dependency,
+    symmetric_smooth,
+)
 from .prune_expand import expanded_samplespace, pruned_samplespace
 from .stats import (
     cdf,
@@ -34,6 +45,11 @@ from .stats import (
     standard_moment,
     variance,
 )
+from .support_spectrum import (
+    spectral_entanglement_bound,
+    support_biadjacency,
+    support_singular_values,
+)
 
 # Don't expose anything yet.
 # from . import pid_broja

dit-2.3/dit/algorithms/marginal_lifts.py

@@ -0,0 +1,182 @@
+"""
+Convex combinations of lifts of a joint's own marginals.
+
+At order :math:`k`, form building blocks
+
+.. math::
+
+    \\bigl\\{ U \\bigr\\} \\cup
+    \\bigl\\{ \\mathrm{Lift}(P_S) : 1 \\le |S| \\le k \\bigr\\}
+
+and fit nonnegative weights summing to one by least squares:
+
+.. math::
+
+    Q^{(k)} = \\arg\\min_{\\alpha \\ge 0,\\ \\sum \\alpha = 1}
+    \\bigl\\| P - \\textstyle\\sum_S \\alpha_S \\mathrm{Lift}(P_S) \\bigr\\|_2^2.
+
+Lifts:
+
+* ``uniform`` — :math:`P_S \\otimes U_{X \\setminus S}`
+* ``product`` — :math:`P_S \\otimes \\prod_{i \\notin S} P_i`
+"""
+
+from collections import defaultdict
+from copy import deepcopy
+from itertools import combinations, product
+
+import numpy as np
+from scipy.optimize import minimize
+
+from ..algorithms.optutil import prepare_dist
+from ..distribution import Distribution
+from ..exceptions import ditException
+
+__all__ = (
+    "lift_marginal",
+    "fit_marginal_lift_mixture",
+    "marginal_lift_dists",
+)
+
+
+def _cartesian(dist):
+    d = prepare_dist(deepcopy(dist))
+    n = d.outcome_length()
+    alph = [tuple(sorted({o[i] for o in d.outcomes})) for i in range(n)]
+    outs = list(product(*alph))
+    pmf_map = {tuple(o): float(p) for o, p in zip(d.outcomes, d.pmf, strict=True)}
+    pmf = np.array([pmf_map.get(o, 0.0) for o in outs], dtype=float)
+    pmf /= pmf.sum()
+    return outs, pmf, alph, d
+
+
+def lift_marginal(outs, pmf, alph, S, mode="uniform"):
+    """
+    Lift a marginal on coordinates ``S`` to a full joint pmf.
+
+    Parameters
+    ----------
+    outs, pmf, alph
+        Dense Cartesian table for the joint.
+    S : tuple of int
+        Variable indices of the marginal.
+    mode : {'uniform', 'product'}
+        How to extend off ``S``.
+    """
+    n = len(alph)
+    S = tuple(S)
+    marg = defaultdict(float)
+    for o, p in zip(outs, pmf, strict=True):
+        marg[tuple(o[i] for i in S)] += p
+    rest = [i for i in range(n) if i not in S]
+    if mode == "uniform":
+        rest_size = int(np.prod([len(alph[i]) for i in rest])) if rest else 1
+        return np.array([marg[tuple(o[i] for i in S)] / rest_size for o in outs], dtype=float)
+    if mode != "product":
+        msg = f"unknown lift mode {mode!r}"
+        raise ditException(msg)
+    ones = []
+    for i in range(n):
+        m1 = defaultdict(float)
+        for o, p in zip(outs, pmf, strict=True):
+            m1[o[i]] += p
+        ones.append(m1)
+    q = np.empty(len(outs), dtype=float)
+    for t, o in enumerate(outs):
+        val = marg[tuple(o[i] for i in S)]
+        for i in rest:
+            val *= ones[i][o[i]]
+        q[t] = val
+    return q / q.sum()
+
+
+def fit_marginal_lift_mixture(dist, order, mode="uniform", n_init=12, seed=0):
+    """
+    Fit a convex combination of lifts of marginals of order at most ``order``.
+
+    Returns
+    -------
+    result : dict
+        Keys ``dist``, ``labels``, ``alpha``, ``L2``.
+    """
+    if order < 0:
+        msg = "order must be nonnegative"
+        raise ditException(msg)
+    outs, pmf, alph, template = _cartesian(dist)
+    n = len(alph)
+    order = min(order, n)
+
+    blocks, labels = [], []
+    blocks.append(np.ones(len(outs)) / len(outs))
+    labels.append(())
+    for k in range(1, order + 1):
+        for S in combinations(range(n), k):
+            blocks.append(lift_marginal(outs, pmf, alph, S, mode=mode))
+            labels.append(S)
+    A = np.column_stack(blocks)
+    nb = A.shape[1]
+
+    def loss(x):
+        return float(np.sum((A @ x - pmf) ** 2))
+
+    cons = {"type": "eq", "fun": lambda x: float(x.sum() - 1.0)}
+    best = None
+    rng = np.random.default_rng(seed)
+    for _ in range(n_init):
+        x0 = rng.dirichlet(np.ones(nb))
+        res = minimize(
+            loss,
+            x0,
+            bounds=[(0.0, None)] * nb,
+            constraints=cons,
+            method="SLSQP",
+            options={"maxiter": 2000, "ftol": 1e-14, "disp": False},
+        )
+        if best is None or res.fun < best[0]:
+            best = (res.fun, res.x)
+
+    alpha = np.maximum(best[1], 0.0)
+    alpha = alpha / alpha.sum()
+    q = A @ alpha
+    q = np.maximum(q, 0.0)
+    q = q / q.sum()
+    qd = Distribution(outs, q, base="linear", validate=False)
+    qd.normalize()
+    if template.get_rv_names() is not None:
+        qd.set_rv_names(template.get_rv_names())
+    return {
+        "dist": qd,
+        "labels": labels,
+        "alpha": alpha,
+        "L2": float(np.sqrt(best[0])),
+    }
+
+
+def marginal_lift_dists(dist, k_max=None, mode="uniform", n_init=12, seed=0):
+    """
+    Ladder of marginal-lift mixtures for orders :math:`0,\\ldots,k_{\\max}`.
+
+    Order 0 is the uniform distribution. Order :math:`n` includes the full
+    joint as a block and recovers :math:`P` exactly.
+    """
+    n = dist.outcome_length()
+    if k_max is None:
+        k_max = n
+    k_max = min(int(k_max), n)
+
+    dists = []
+    metas = []
+    for k in range(0, k_max + 1):
+        if k == 0:
+            outs, _, _, template = _cartesian(dist)
+            q = Distribution(outs, np.ones(len(outs)) / len(outs), base="linear", validate=False)
+            q.normalize()
+            if template.get_rv_names() is not None:
+                q.set_rv_names(template.get_rv_names())
+            dists.append(q)
+            metas.append({"labels": [()], "alpha": np.array([1.0]), "L2": None})
+        else:
+            fit = fit_marginal_lift_mixture(dist, k, mode=mode, n_init=n_init, seed=seed + k)
+            dists.append(fit["dist"])
+            metas.append({key: fit[key] for key in ("labels", "alpha", "L2")})
+    return dists, metas

dit-2.3/dit/algorithms/mixture_of_products.py

@@ -0,0 +1,255 @@
+"""
+Mixtures of fully factorized distributions (latent-class / naive-Bayes).
+
+The family
+
+.. math::
+
+    \\mathcal{F}_k = \\Bigl\\{
+        Q : Q(x) = \\sum_{\\alpha=1}^{k} \\pi_\\alpha
+        \\prod_{i=1}^{n} Q_i(x_i \\mid \\alpha)
+    \\Bigr\\}
+
+is the standard representation underlying Wyner common information: variables
+are independent given a discrete latent of cardinality :math:`k`.  Maximum
+likelihood under :math:`P` is equivalent to the forward-KL projection
+
+.. math::
+
+    Q^{(k)} = \\arg\\min_{Q \\in \\mathcal{F}_k} D(P \\Vert Q)
+
+and is fit by EM.  No support jitter is required: :math:`Q` may place mass
+outside the support of a sparse :math:`P`.
+
+See Rosas et al. (2019) for the shared-randomness / binding interpretation of
+dual total correlation, and Wyner (1975) / Abdallah & Plumbley (2012) for the
+common-information side.
+"""
+
+from copy import deepcopy
+from itertools import product
+
+import numpy as np
+
+from ..distribution import Distribution
+from .optutil import prepare_dist
+
+__all__ = (
+    "fit_mixture_of_products",
+    "mixture_of_products_dists",
+)
+
+
+def _dense_table(dist):
+    """
+    Expand ``dist`` onto its Cartesian sample space.
+
+    Returns
+    -------
+    outcomes : list of tuple
+    pmf : ndarray, shape (n_outcomes,)
+    X : ndarray, shape (n_outcomes, n_vars), integer-coded symbols
+    sizes : list of int
+    """
+    d = prepare_dist(deepcopy(dist))
+    n = d.outcome_length()
+    alphabets = [tuple(sorted({o[i] for o in d.outcomes})) for i in range(n)]
+    outcomes = list(product(*alphabets))
+    pmf_map = {tuple(o): float(p) for o, p in zip(d.outcomes, d.pmf, strict=True)}
+    pmf = np.array([pmf_map.get(o, 0.0) for o in outcomes], dtype=float)
+    total = pmf.sum()
+    if total <= 0:
+        msg = "Distribution has no mass."
+        raise ValueError(msg)
+    pmf /= total
+    sym_index = [{s: j for j, s in enumerate(a)} for a in alphabets]
+    X = np.array([[sym_index[i][o[i]] for i in range(n)] for o in outcomes], dtype=int)
+    sizes = [len(a) for a in alphabets]
+    return outcomes, pmf, X, sizes
+
+
+def _component_logprob(pi, conds, X):
+    """Log joint component densities ``log(π_α ∏_i Q_i(x_i|α))``, shape (k, n_out)."""
+    log_comp = np.log(pi + 1e-300)[:, None]
+    for i, cond in enumerate(conds):
+        log_comp = log_comp + np.log(cond[:, X[:, i]] + 1e-300)
+    return log_comp
+
+
+def _em_once(pmf, X, sizes, k, *, max_iter, tol, rng):
+    """Single EM run. Returns (loglik, q_pmf, pi, conds, I_xv, H_v)."""
+    n_out, n = X.shape
+    pi = rng.dirichlet(np.ones(k))
+    conds = [rng.dirichlet(np.ones(s), size=k) for s in sizes]
+    prev_ll = -np.inf
+
+    for _ in range(max_iter):
+        log_comp = _component_logprob(pi, conds, X)
+        m = log_comp.max(axis=0, keepdims=True)
+        comp = np.exp(log_comp - m)
+        r = comp / (comp.sum(axis=0, keepdims=True) + 1e-300)
+
+        w = r * pmf[None, :]
+        pi = w.sum(axis=1)
+        pi = pi / (pi.sum() + 1e-300)
+
+        for i in range(n):
+            s = sizes[i]
+            c = np.zeros((k, s))
+            for a in range(k):
+                for v in range(s):
+                    c[a, v] = w[a, X[:, i] == v].sum()
+                c[a] /= c[a].sum() + 1e-300
+            conds[i] = c
+
+        log_comp = _component_logprob(pi, conds, X)
+        m = log_comp.max(axis=0)
+        ll = float(np.sum(pmf * (m + np.log(np.exp(log_comp - m).sum(axis=0) + 1e-300))))
+        if abs(ll - prev_ll) < tol:
+            break
+        prev_ll = ll
+
+    log_comp = _component_logprob(pi, conds, X)
+    m = log_comp.max(axis=0)
+    q = np.exp(m) * np.exp(log_comp - m).sum(axis=0)
+    q = q / q.sum()
+
+    # Responsibilities under the data for I(X; V).
+    m = log_comp.max(axis=0, keepdims=True)
+    r = np.exp(log_comp - m)
+    r = r / (r.sum(axis=0, keepdims=True) + 1e-300)
+    p_a = (r * pmf[None, :]).sum(axis=1)
+    p_a = p_a / (p_a.sum() + 1e-300)
+    hv = float(-np.sum(p_a[p_a > 0] * np.log2(p_a[p_a > 0])))
+    hv_x = 0.0
+    for t in range(n_out):
+        if pmf[t] <= 0:
+            continue
+        rt = r[:, t]
+        rt = rt[rt > 0]
+        hv_x += float(pmf[t] * (-np.sum(rt * np.log2(rt))))
+    ixv = hv - hv_x
+
+    return ll, q, pi, conds, float(ixv), hv
+
+
+def fit_mixture_of_products(
+    dist,
+    k,
+    *,
+    n_init=12,
+    max_iter=200,
+    tol=1e-10,
+    seed=0,
+):
+    """
+    Fit a :math:`k`-mixture of product distributions to ``dist`` by EM.
+
+    Parameters
+    ----------
+    dist : Distribution
+        Target joint.
+    k : int
+        Number of mixture components.
+    n_init, max_iter : int
+        Random restarts and EM iteration cap.
+    tol : float
+        Log-likelihood convergence tolerance.
+    seed : int
+        RNG seed for restarts.
+
+    Returns
+    -------
+    result : dict
+        Keys ``dist`` (fitted :class:`Distribution`), ``pi``, ``conds``,
+        ``I_xv`` (:math:`I(X;V)` under data-weighted responsibilities),
+        ``H_v``, ``loglik``.
+    """
+    if k < 1:
+        msg = "k must be >= 1"
+        raise ValueError(msg)
+
+    outcomes, pmf, X, sizes = _dense_table(dist)
+    rng = np.random.default_rng(seed)
+    best = None
+    for _ in range(n_init):
+        run_rng = np.random.default_rng(rng.integers(0, 2**31 - 1))
+        cand = _em_once(pmf, X, sizes, k, max_iter=max_iter, tol=tol, rng=run_rng)
+        if best is None or cand[0] > best[0]:
+            best = cand
+
+    ll, q_pmf, pi, conds, ixv, hv = best
+    q = Distribution(outcomes, q_pmf, base="linear", validate=False)
+    q.normalize()
+    return {
+        "dist": q,
+        "pi": pi,
+        "conds": conds,
+        "I_xv": ixv,
+        "H_v": hv,
+        "loglik": ll,
+    }
+
+
+def mixture_of_products_dists(
+    dist,
+    k_max=None,
+    *,
+    n_init=12,
+    max_iter=200,
+    tol=1e-10,
+    seed=0,
+    early_stop=True,
+    kl_tol=1e-8,
+):
+    """
+    Fit the mixture-of-products ladder :math:`Q^{(1)},\\ldots,Q^{(k_{\\max})}`.
+
+    Parameters
+    ----------
+    dist : Distribution
+    k_max : int or None
+        Maximum number of components.  Default ``min(8, |X|)``.
+    n_init, max_iter, tol, seed
+        Passed to :func:`fit_mixture_of_products` (seed offset by ``k``).
+    early_stop : bool
+        If True, stop once :math:`D(P\\Vert Q^{(k)}) <` ``kl_tol``.
+    kl_tol : float
+        Forward-KL threshold for early stopping.
+
+    Returns
+    -------
+    dists : list of Distribution
+        ``dists[k-1]`` is the MLE in :math:`\\mathcal{F}_k`.
+    meta : list of dict
+        Per-``k`` diagnostics (``I_xv``, ``H_v``, ``loglik``, ``pi``, ``conds``).
+    """
+    from ..divergences import kullback_leibler_divergence as D
+
+    outcomes, pmf, _, _ = _dense_table(dist)
+    p_dense = Distribution(outcomes, pmf, base="linear", validate=False)
+    p_dense.normalize()
+
+    if k_max is None:
+        k_max = min(8, len(outcomes))
+    k_max = max(1, int(k_max))
+
+    dists = []
+    meta = []
+    for k in range(1, k_max + 1):
+        fit = fit_mixture_of_products(
+            dist,
+            k,
+            n_init=n_init,
+            max_iter=max_iter,
+            tol=tol,
+            seed=seed + 17 * k,
+        )
+        dists.append(fit["dist"])
+        entry = {key: fit[key] for key in ("I_xv", "H_v", "loglik", "pi", "conds")}
+        entry["forward_kl"] = float(D(p_dense, fit["dist"]))
+        meta.append(entry)
+        if early_stop and entry["forward_kl"] < kl_tol:
+            break
+
+    return dists, meta