pysp-learn 0.2.0__py3-none-any.whl

pysp/__init__.py

@@ -0,0 +1 @@
+__all__ = ["stats", "utils", "models", "parallel", "src"]

pysp/arithmetic.py

@@ -0,0 +1,103 @@
+"""Backend-dispatched arithmetic helpers used by pysparkplug classes."""
+
+from __future__ import annotations
+
+import numpy as np
+
+from pysp.engines import NUMPY_ENGINE, engine_of
+
+__all__ = [
+    "asarray",
+    "zeros",
+    "empty",
+    "arange",
+    "to_numpy",
+    "log",
+    "exp",
+    "sqrt",
+    "abs",
+    "where",
+    "maximum",
+    "clip",
+    "floor",
+    "isnan",
+    "isinf",
+    "dot",
+    "matmul",
+    "cumsum",
+    "logsumexp",
+    "stack",
+    "bincount",
+    "index_add",
+    "unique",
+    "searchsorted",
+    "gammaln",
+    "digamma",
+    "betaln",
+    "erf",
+    "pi",
+    "maxint",
+    "maxrandint",
+    "one",
+    "zero",
+    "two",
+    "half",
+    "inf",
+    "eps",
+]
+
+
+def _dispatch(name):
+    def fn(*args, **kwargs):
+        engine = engine_of(args, default=NUMPY_ENGINE)
+        return getattr(engine, name)(*args, **kwargs)
+
+    fn.__name__ = name
+    return fn
+
+
+asarray = _dispatch("asarray")
+zeros = _dispatch("zeros")
+empty = _dispatch("empty")
+arange = _dispatch("arange")
+to_numpy = _dispatch("to_numpy")
+
+log = _dispatch("log")
+exp = _dispatch("exp")
+sqrt = _dispatch("sqrt")
+abs = _dispatch("abs")
+where = _dispatch("where")
+maximum = _dispatch("maximum")
+clip = _dispatch("clip")
+floor = _dispatch("floor")
+isnan = _dispatch("isnan")
+isinf = _dispatch("isinf")
+
+sum = _dispatch("sum")
+max = _dispatch("max")
+dot = _dispatch("dot")
+matmul = _dispatch("matmul")
+cumsum = _dispatch("cumsum")
+logsumexp = _dispatch("logsumexp")
+stack = _dispatch("stack")
+
+bincount = _dispatch("bincount")
+index_add = _dispatch("index_add")
+unique = _dispatch("unique")
+searchsorted = _dispatch("searchsorted")
+
+gammaln = _dispatch("gammaln")
+digamma = _dispatch("digamma")
+betaln = _dispatch("betaln")
+erf = _dispatch("erf")
+
+
+pi = np.pi
+maxint = 2**31 - 1
+maxrandint = 2**31 - 1
+one = 1.0
+zero = 0.0
+two = 2.0
+half = 0.5
+inf = float("inf")
+eps = 1.0e-8

pysp/data/__init__.py

@@ -0,0 +1,20 @@
+"""Data adapters and observation representations for pysparkplug.
+
+These are input/representation helpers (pandas DataFrame adapters, graph
+observation encoding, Spark RDD sampling) — not probability distributions,
+so they live outside ``pysp.stats``.
+"""
+
+from pysp.data.dataframe import dataframe_records, seq_encode_dataframe
+from pysp.data.graph_data import GraphDataEncoder, GraphObservation
+from pysp.data.rdd_sampler import sample_rdd, sample_seq_as_rdd, take_sample
+
+__all__ = [
+    "GraphDataEncoder",
+    "GraphObservation",
+    "dataframe_records",
+    "sample_rdd",
+    "sample_seq_as_rdd",
+    "seq_encode_dataframe",
+    "take_sample",
+]

pysp/data/dataframe.py

@@ -0,0 +1,75 @@
+"""Pandas DataFrame adapters for the sequence-encoded stats API."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Any
+
+from pysp.stats.compute.pdist import DataSequenceEncoder, ParameterEstimator, SequenceEncodableProbabilityDistribution
+
+FieldSpec = str | Sequence[Any] | None
+
+
+def _field_source(field: Any) -> Any:
+    if isinstance(field, tuple) and len(field) == 2:
+        return field[1]
+    return field
+
+
+def dataframe_records(df: Any, fields: FieldSpec = None, as_dict: bool = False) -> list[Any]:
+    """Convert DataFrame columns into observation records for ``seq_encode``.
+
+    A single selected field becomes scalar observations. Multiple selected
+    fields become tuple observations in the requested field order, matching the
+    data shape expected by composite distributions. When ``as_dict=True``,
+    each row is returned as a mapping keyed by the selected source field names.
+    """
+    if fields is None:
+        field_list = list(df.columns)
+    elif isinstance(fields, str):
+        field_list = [fields]
+    else:
+        field_list = list(fields)
+
+    source_list = [_field_source(name) for name in field_list]
+    missing = [name for name in source_list if name not in df.columns]
+    if missing:
+        raise KeyError("DataFrame is missing fields: %s" % ", ".join(map(str, missing)))
+
+    if len(field_list) == 0:
+        raise ValueError("fields must select at least one DataFrame column.")
+
+    if as_dict:
+        rows = []
+        for row in df.loc[:, source_list].itertuples(index=False, name=None):
+            rows.append({name: value for name, value in zip(source_list, row)})
+        return rows
+
+    if len(field_list) == 1:
+        return df[source_list[0]].tolist()
+
+    return list(df.loc[:, source_list].itertuples(index=False, name=None))
+
+
+def seq_encode_dataframe(
+    df: Any,
+    fields: FieldSpec = None,
+    encoder: DataSequenceEncoder | None = None,
+    estimator: ParameterEstimator | None = None,
+    model: SequenceEncodableProbabilityDistribution | None = None,
+    num_chunks: int = 1,
+    chunk_size: int | None = None,
+):
+    """Sequence-encode selected DataFrame columns with the ordinary stats API."""
+    from pysp.stats import seq_encode
+    from pysp.stats.combinator.record import RecordDistribution, RecordEstimator
+
+    if fields is None and model is not None and isinstance(model, RecordDistribution):
+        fields = tuple(zip(model.fields, model.sources))
+    elif fields is None and estimator is not None and isinstance(estimator, RecordEstimator):
+        fields = tuple(zip(estimator.fields, estimator.sources))
+    as_dict = isinstance(model, RecordDistribution) or isinstance(estimator, RecordEstimator)
+    records = dataframe_records(df, fields=fields, as_dict=as_dict)
+    return seq_encode(
+        records, encoder=encoder, estimator=estimator, model=model, num_chunks=num_chunks, chunk_size=chunk_size
+    )

pysp/data/graph_data.py

@@ -0,0 +1,257 @@
+"""Shared graph observation encoding helpers for stats graph distributions.
+
+Graph observations may be square binary adjacency matrices, NetworkX-like graph
+objects, ``(adjacency, block_assignments)`` pairs, or mappings with
+``adjacency``/``adj`` and optional ``block_assignments``/``blocks``.
+"""
+
+import math
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from typing import Any
+
+import numpy as np
+
+try:
+    import scipy.sparse as sp
+except Exception:  # pragma: no cover - scipy is a package dependency in normal use.
+    sp = None
+
+from pysp.stats.compute.pdist import DataSequenceEncoder
+
+_EPS = 1.0e-12
+
+
+@dataclass(frozen=True)
+class GraphObservation:
+    """Canonical binary graph observation used by graph encoders."""
+
+    adjacency: np.ndarray
+    block_assignments: np.ndarray | None = None
+
+
+def _clip_prob(p: float) -> float:
+    pp = float(p)
+    if not np.isfinite(pp) or pp < 0.0 or pp > 1.0:
+        raise ValueError("probabilities must be finite values in [0, 1].")
+    return float(np.clip(pp, _EPS, 1.0 - _EPS))
+
+
+def _bernoulli_log_likelihood(successes: float, total: float, p: float) -> float:
+    pp = _clip_prob(p)
+    return float(successes * math.log(pp) + (total - successes) * math.log1p(-pp))
+
+
+def _edge_indices(n: int, directed: bool, self_loops: bool):
+    if directed:
+        for i in range(n):
+            for j in range(n):
+                if self_loops or i != j:
+                    yield i, j
+    else:
+        start = 0 if self_loops else 1
+        for i in range(n):
+            for j in range(i + start, n):
+                yield i, j
+
+
+def _networkx_like_to_adjacency(graph: Any) -> tuple[np.ndarray, np.ndarray | None]:
+    nodes = list(graph.nodes())
+    index = {node: i for i, node in enumerate(nodes)}
+    adj = np.zeros((len(nodes), len(nodes)), dtype=np.float64)
+    directed = bool(graph.is_directed()) if hasattr(graph, "is_directed") else False
+
+    for edge in graph.edges(data=True):
+        if len(edge) == 3:
+            u, v, data = edge
+        else:
+            u, v = edge[:2]
+            data = {}
+        weight = 1.0
+        if isinstance(data, Mapping):
+            weight = data.get("weight", 1.0)
+        adj[index[u], index[v]] = weight
+        if not directed and u != v:
+            adj[index[v], index[u]] = weight
+
+    assignments = []
+    found_assignment = False
+    for node in nodes:
+        value = None
+        try:
+            attrs = graph.nodes[node]
+            if isinstance(attrs, Mapping):
+                if "block" in attrs:
+                    value = attrs["block"]
+                elif "block_assignment" in attrs:
+                    value = attrs["block_assignment"]
+        except Exception:
+            value = None
+        assignments.append(value)
+        found_assignment = found_assignment or value is not None
+
+    if found_assignment:
+        if any(value is None for value in assignments):
+            raise ValueError("all graph nodes must have block labels when any node has one.")
+        return adj, np.asarray(assignments, dtype=np.int64)
+    return adj, None
+
+
+def _edge_list_to_adjacency(edges: Sequence[Any], num_nodes: int, directed: bool) -> np.ndarray:
+    n = int(num_nodes)
+    if n < 0:
+        raise ValueError("num_nodes must be non-negative.")
+    adj = np.zeros((n, n), dtype=np.float64)
+    for edge in edges:
+        if len(edge) < 2:
+            raise ValueError("edge entries must contain at least two node indices.")
+        i = int(edge[0])
+        j = int(edge[1])
+        if i < 0 or i >= n or j < 0 or j >= n:
+            raise ValueError("edge node indices must be in [0, num_nodes).")
+        weight = float(edge[2]) if len(edge) >= 3 else 1.0
+        adj[i, j] = weight
+        if not directed and i != j:
+            adj[j, i] = weight
+    return adj
+
+
+def _as_adjacency(adjacency: Any) -> np.ndarray:
+    if hasattr(adjacency, "nodes") and hasattr(adjacency, "edges"):
+        adjacency, _ = _networkx_like_to_adjacency(adjacency)
+    elif sp is not None and sp.issparse(adjacency):
+        adjacency = adjacency.toarray()
+
+    adj = np.asarray(adjacency, dtype=np.float64)
+    if adj.ndim != 2 or adj.shape[0] != adj.shape[1]:
+        raise ValueError("graph adjacency must be a square matrix.")
+    if np.any(~np.isfinite(adj)):
+        raise ValueError("graph adjacency must be finite.")
+    if np.any((adj != 0.0) & (adj != 1.0)):
+        raise ValueError("graph adjacency must contain binary values 0/1.")
+    return adj
+
+
+def _as_assignments(assignments: Any | None, n: int) -> np.ndarray | None:
+    if assignments is None:
+        return None
+    rv = np.asarray(assignments, dtype=np.int64)
+    if rv.ndim != 1 or rv.shape[0] != n:
+        raise ValueError("block assignments must be a length-%d one-dimensional sequence." % n)
+    if rv.size and rv.min() < 0:
+        raise ValueError("block assignments must be non-negative integers.")
+    return rv
+
+
+def _extract_observation(x: Any, directed: bool = False, fallback_assignments: Any | None = None) -> GraphObservation:
+    if isinstance(x, GraphObservation):
+        adj = _as_adjacency(x.adjacency)
+        assignments = x.block_assignments
+    elif isinstance(x, Mapping):
+        assignments = x.get("block_assignments", x.get("blocks", fallback_assignments))
+        if "adjacency" in x:
+            adj = _as_adjacency(x["adjacency"])
+        elif "adj" in x:
+            adj = _as_adjacency(x["adj"])
+        elif "graph" in x:
+            adj, graph_assignments = _networkx_like_to_adjacency(x["graph"])
+            if assignments is None:
+                assignments = graph_assignments
+            adj = _as_adjacency(adj)
+        elif "edges" in x and "num_nodes" in x:
+            adj = _edge_list_to_adjacency(x["edges"], int(x["num_nodes"]), directed=directed)
+        else:
+            raise ValueError("graph mapping must contain adjacency, adj, graph, or edges+num_nodes.")
+    elif isinstance(x, tuple) and len(x) == 2:
+        adj = _as_adjacency(x[0])
+        assignments = x[1] if x[1] is not None else fallback_assignments
+    elif isinstance(x, list) and len(x) == 2 and not np.isscalar(x[0]):
+        try:
+            adj = _as_adjacency(x[0])
+            assignments = x[1] if x[1] is not None else fallback_assignments
+        except Exception:
+            adj = _as_adjacency(x)
+            assignments = fallback_assignments
+    elif hasattr(x, "nodes") and hasattr(x, "edges"):
+        adj, graph_assignments = _networkx_like_to_adjacency(x)
+        assignments = graph_assignments if graph_assignments is not None else fallback_assignments
+        adj = _as_adjacency(adj)
+    else:
+        adj = _as_adjacency(x)
+        assignments = fallback_assignments
+
+    return GraphObservation(adj, _as_assignments(assignments, adj.shape[0]))
+
+
+def _edge_counts(adj: np.ndarray, directed: bool, self_loops: bool) -> tuple[float, float]:
+    total = 0.0
+    successes = 0.0
+    for i, j in _edge_indices(adj.shape[0], directed=directed, self_loops=self_loops):
+        successes += adj[i, j]
+        total += 1.0
+    return total, successes
+
+
+def _validate_block_probs(block_probs: Any) -> np.ndarray:
+    probs = np.asarray(block_probs, dtype=np.float64)
+    if probs.ndim != 2 or probs.shape[0] != probs.shape[1]:
+        raise ValueError("block_probs must be a square matrix.")
+    if probs.shape[0] == 0:
+        raise ValueError("block_probs must contain at least one block.")
+    if np.any(~np.isfinite(probs)) or np.any(probs < 0.0) or np.any(probs > 1.0):
+        raise ValueError("block probabilities must be finite and in [0, 1].")
+    return probs
+
+
+def _validate_block_indices(assignments: np.ndarray, num_blocks: int) -> None:
+    if assignments.ndim != 1:
+        raise ValueError("block assignments must be a one-dimensional sequence.")
+    if assignments.size and (assignments.min() < 0 or assignments.max() >= num_blocks):
+        raise ValueError("block assignments must index block_probs.")
+
+
+def _normalize_prior(block_prior: Any | None, num_blocks: int) -> np.ndarray:
+    if num_blocks <= 0:
+        raise ValueError("num_blocks must be positive.")
+    if block_prior is None:
+        return np.full(int(num_blocks), 1.0 / float(num_blocks), dtype=np.float64)
+    prior = np.asarray(block_prior, dtype=np.float64)
+    if prior.ndim != 1 or prior.shape[0] != num_blocks:
+        raise ValueError("block_prior must be a length-num_blocks vector.")
+    if np.any(~np.isfinite(prior)) or np.any(prior < 0.0) or prior.sum() <= 0.0:
+        raise ValueError("block_prior must contain non-negative finite values with positive sum.")
+    return prior / prior.sum()
+
+
+class GraphDataEncoder(DataSequenceEncoder):
+    """Encode graph observations as canonical adjacency/assignment objects."""
+
+    def __init__(self, directed: bool = False, fallback_assignments: Any | None = None) -> None:
+        self.directed = bool(directed)
+        self.fallback_assignments = (
+            None
+            if fallback_assignments is None
+            else tuple(int(u) for u in np.asarray(fallback_assignments, dtype=np.int64))
+        )
+
+    def __str__(self) -> str:
+        return "GraphDataEncoder(directed=%s)" % repr(self.directed)
+
+    def __eq__(self, other: object) -> bool:
+        return (
+            isinstance(other, GraphDataEncoder)
+            and self.directed == other.directed
+            and self.fallback_assignments == other.fallback_assignments
+        )
+
+    def seq_encode(self, x: Sequence[Any]) -> tuple[GraphObservation, ...]:
+        fallback = None if self.fallback_assignments is None else np.asarray(self.fallback_assignments, dtype=np.int64)
+        return tuple(_extract_observation(u, directed=self.directed, fallback_assignments=fallback) for u in x)
+
+    def nbytes(self, x: Any) -> int:
+        total = 0
+        for obs in x:
+            total += int(obs.adjacency.nbytes)
+            if obs.block_assignments is not None:
+                total += int(obs.block_assignments.nbytes)
+        return total

pysp/data/rdd_sampler.py

@@ -0,0 +1,46 @@
+try:
+    from pyspark import SparkConf, SparkContext
+except ImportError:
+    SparkContext = SparkConf = None  # pip install pysparkplug[spark]
+import pickle
+from typing import Any
+
+import numpy as np
+from numpy.random import RandomState
+
+from pysp.arithmetic import *
+from pysp.arithmetic import maxrandint
+
+
+def take_sample(rdd: Any, with_replacement: bool, n: int, seed: int | None = None):
+    rng = RandomState(seed)
+    sample = rdd.zipWithUniqueId().takeSample(with_replacement, n, rng.randint(0, maxrandint))
+    sidx = np.argsort([u[1] for u in sample])
+    sample = [sample[i][0] for i in sidx]
+    sidx = np.argsort(rng.uniform(size=n))
+    return [sample[i] for i in sidx]
+
+
+def sample_seq_as_rdd(sc, dist, seq_len, count_per_split, num_splits, seed=None):
+    distB = sc.broadcast(dist)
+    seeds = RandomState(seed).randint(0, maxrandint, size=num_splits)
+
+    def fmap(u):
+        ddist = distB.value
+        sampler = [ddist.sampler(seed=h) for h in u]
+        return iter([v for h in sampler for v in h.sample_seq(seq_len, size=count_per_split)])
+
+    return sc.parallelize(seeds, num_splits).mapPartitions(fmap, True)
+
+
+def sample_rdd(sc, dist, count_per_split, num_splits, seed=None):
+    dd = pickle.dumps(dist, protocol=0)
+    distB = sc.broadcast(dd)
+    seeds = RandomState(seed).randint(0, maxrandint, size=num_splits)
+
+    def fmap(u):
+        ddist = pickle.loads(distB.value)
+        sampler = [ddist.sampler(seed=h) for h in u]
+        return iter([v for h in sampler for v in h.sample(size=count_per_split)])
+
+    return sc.parallelize(seeds, num_splits).mapPartitions(fmap, True)

pysp/doe/__init__.py

@@ -0,0 +1,91 @@
+"""Design of experiments (DoE) for pysparkplug.
+
+This package builds experiment designs over a bounded input space and (in later additions)
+sequential / Bayesian-optimization loops on top of the existing GP and regression machinery.
+
+The first surface is space-filling and classical design generators, all returning a plain
+``(n, d)`` numpy matrix of input points scaled into the supplied per-dimension bounds:
+
+    >>> from pysp.doe import latin_hypercube
+    >>> x = latin_hypercube([(0.0, 1.0), (-2.0, 2.0)], n=8, seed=0)
+    >>> x.shape
+    (8, 2)
+"""
+
+from __future__ import annotations
+
+from pysp.doe.bayesopt import (
+    BayesOptResult,
+    available_acquisitions,
+    expected_improvement,
+    minimize,
+    probability_of_improvement,
+    propose_batch,
+    propose_next,
+    register_acquisition,
+    upper_confidence_bound,
+)
+from pysp.doe.constrained import (
+    ConstrainedBayesOptResult,
+    constrained_minimize,
+    probability_of_feasibility,
+    propose_next_constrained,
+)
+from pysp.doe.designs import (
+    Bounds,
+    full_factorial,
+    halton_design,
+    latin_hypercube,
+    maximin_latin_hypercube,
+    random_design,
+    sobol_design,
+)
+from pysp.doe.multiobjective import (
+    MultiObjectiveResult,
+    multi_minimize,
+    pareto_mask,
+)
+from pysp.doe.optimal import (
+    a_criterion,
+    available_criteria,
+    d_criterion,
+    i_criterion,
+    optimal_design,
+    polynomial_features,
+    register_criterion,
+)
+from pysp.doe.optimizer import BayesianOptimizer
+
+__all__ = [
+    "Bounds",
+    "full_factorial",
+    "halton_design",
+    "latin_hypercube",
+    "maximin_latin_hypercube",
+    "random_design",
+    "sobol_design",
+    "BayesOptResult",
+    "expected_improvement",
+    "probability_of_improvement",
+    "upper_confidence_bound",
+    "register_acquisition",
+    "available_acquisitions",
+    "minimize",
+    "propose_next",
+    "propose_batch",
+    "optimal_design",
+    "polynomial_features",
+    "d_criterion",
+    "a_criterion",
+    "i_criterion",
+    "register_criterion",
+    "available_criteria",
+    "ConstrainedBayesOptResult",
+    "probability_of_feasibility",
+    "propose_next_constrained",
+    "constrained_minimize",
+    "MultiObjectiveResult",
+    "pareto_mask",
+    "multi_minimize",
+    "BayesianOptimizer",
+]