westpa 2022.13__cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl

westpa/fasthist/_fasthist.pyx

@@ -0,0 +1,222 @@
+
+from __future__ import division
+import numpy, sys
+cimport numpy, cython
+from cpython.buffer cimport *
+from cpython.mem cimport *
+from numpy cimport * #PyArray_DATA, PyArray_TYPE
+
+ctypedef fused real_numeric:
+    numpy.int8_t
+    numpy.int16_t
+    numpy.int32_t
+    numpy.int64_t
+    numpy.uint8_t
+    numpy.uint16_t
+    numpy.uint32_t
+    numpy.uint64_t    
+    numpy.float32_t
+    numpy.float64_t
+
+@cython.boundscheck(False)
+@cython.wraparound(False)
+cpdef histnd(values, binbounds, weights=1.0, out=None, binbound_check = True, ignore_out_of_range=False):
+    '''Generate an N-dimensional PDF (or contribution to a PDF) from the given values.
+    ``binbounds`` is a list of arrays of boundary values, with one entry for each
+    dimension (``values`` must have as many columns as there are entries in ``binbounds``)
+    ``weight``, if provided, specifies the weight each value contributes to the
+    histogram; this may be a scalar (for equal weights for all values) or a vector of
+    the same length as ``values`` (for unequal weights). If ``binbound_check`` is True, then
+    the boundaries are checked for strict positive monotonicity; set to False to shave a few
+    microseconds if you know your bin boundaries to be monotonically increasing. 
+    '''
+    
+    if values.ndim != 2:
+        values = numpy.atleast_2d(values)
+        if values.ndim > 2:
+            raise TypeError('values must be 2-D')
+    
+    cdef:
+        Py_ssize_t npts = values.shape[0]
+        Py_ssize_t ndim = values.shape[1] 
+        int typecode = PyArray_TYPE(values) 
+        
+    if len(binbounds) != ndim:
+        raise ValueError('number of sets of bin boundaries ({}) does not match dimensionality of data ({})'
+                         .format(len(binbounds), values.shape[1]))
+
+    if binbound_check:
+        for idim in xrange(ndim):
+            dq = numpy.diff(binbounds[idim])
+            if (dq <= 0).any():
+                raise ValueError('binbounds in dimension {} are not strictly monotonically increasing'.format(idim))
+
+    # Prepare bin boundaries arrays
+    _binbounds_vectors = numpy.empty((ndim,), numpy.object_)
+    _nbounds   = numpy.empty((ndim,), numpy.uint32)
+    for idim in range(ndim):
+        _binbounds = numpy.require(binbounds[idim], values.dtype, 'C')
+        _binbounds_vectors[idim] = _binbounds
+        _nbounds[idim] = _binbounds.shape[0]
+
+    # Prepare output array, if necessary        
+    if out is None:
+        _out = numpy.zeros([len(boundset)-1 for boundset in binbounds], numpy.float64)
+    else:
+        _out = out
+        if _out.dtype != numpy.float64:
+            raise TypeError('type of output array must be float64')
+        if not _out.flags.writeable:
+            raise TypeError('output is not writeable') 
+        
+    # Prepare weight array
+    _weights = numpy.require(weights, numpy.float64, 'C')
+    if _weights.shape == ():
+        # scalar
+        _weights = numpy.empty((len(values),), numpy.float64)
+        _weights[:] = weights
+    elif _weights.ndim > 1:
+        raise TypeError('weight must be scalar or one dimensional')
+    elif  _weights.shape[0] != values.shape[0]:
+        raise TypeError('weights and values must be equal in length')
+    
+    # ugh
+    if typecode == NPY_FLOAT32:      
+        return _histnd[numpy.float32_t](values,
+                                        _binbounds_vectors,
+                                        <numpy.uint32_t*> PyArray_DATA(_nbounds),
+                                        <numpy.float64_t*> PyArray_DATA(_weights),
+                                        ignore_out_of_range,
+                                        _out)
+    elif typecode == NPY_FLOAT64:
+        return _histnd[numpy.float64_t](values,
+                                        _binbounds_vectors,
+                                        <numpy.uint32_t*> PyArray_DATA(_nbounds),
+                                        <numpy.float64_t*> PyArray_DATA(_weights),
+                                        ignore_out_of_range,
+                                        _out)
+    elif typecode == NPY_INT8:
+        return _histnd[numpy.int8_t](values,
+                                        _binbounds_vectors,
+                                        <numpy.uint32_t*> PyArray_DATA(_nbounds),
+                                        <numpy.float64_t*> PyArray_DATA(_weights),
+                                        ignore_out_of_range,
+                                        _out)
+    elif typecode == NPY_INT16:
+        return _histnd[numpy.int16_t](values,
+                                        _binbounds_vectors,
+                                        <numpy.uint32_t*> PyArray_DATA(_nbounds),
+                                        <numpy.float64_t*> PyArray_DATA(_weights),
+                                        ignore_out_of_range,
+                                        _out)
+    elif typecode == NPY_INT32:
+        return _histnd[numpy.int32_t](values,
+                                        _binbounds_vectors,
+                                        <numpy.uint32_t*> PyArray_DATA(_nbounds),
+                                        <numpy.float64_t*> PyArray_DATA(_weights),
+                                        ignore_out_of_range,
+                                        _out)
+    elif typecode == NPY_INT64:
+        return _histnd[numpy.int64_t](values,
+                                        _binbounds_vectors,
+                                        <numpy.uint32_t*> PyArray_DATA(_nbounds),
+                                        <numpy.float64_t*> PyArray_DATA(_weights),
+                                        ignore_out_of_range,
+                                        _out)
+    elif typecode == NPY_UINT8:
+        return _histnd[numpy.uint8_t](values,
+                                        _binbounds_vectors,
+                                        <numpy.uint32_t*> PyArray_DATA(_nbounds),
+                                        <numpy.float64_t*> PyArray_DATA(_weights),
+                                        ignore_out_of_range,
+                                        _out)
+    elif typecode == NPY_UINT16:
+        return _histnd[numpy.uint16_t](values,
+                                        _binbounds_vectors,
+                                        <numpy.uint32_t*> PyArray_DATA(_nbounds),
+                                        <numpy.float64_t*> PyArray_DATA(_weights),
+                                        ignore_out_of_range,
+                                        _out)
+    elif typecode == NPY_UINT32:
+        return _histnd[numpy.uint32_t](values,
+                                        _binbounds_vectors,
+                                        <numpy.uint32_t*> PyArray_DATA(_nbounds),
+                                        <numpy.float64_t*> PyArray_DATA(_weights),
+                                        ignore_out_of_range,
+                                        _out)
+    elif typecode == NPY_UINT64:
+        return _histnd[numpy.uint64_t](values,
+                                        _binbounds_vectors,
+                                        <numpy.uint32_t*> PyArray_DATA(_nbounds),
+                                        <numpy.float64_t*> PyArray_DATA(_weights),
+                                        ignore_out_of_range,
+                                        _out)
+    else:
+        raise TypeError('real floating-point or integer input required')
+    
+@cython.boundscheck(False)
+@cython.wraparound(False)
+cdef _histnd(real_numeric[:,:] values, object[:] binbounds, numpy.uint32_t* nbounds, double* weights,
+             bint ignore_out_of_range, object output):
+    '''Bin the values stored in the 2-D array ``values`` with corresponding weights ``weights``
+    into the array of bins ``output``. The bin boundaries are specified in ``binbounds``, and the
+    length of each set of boundaries is stored in ``nbounds``.
+    
+    Pre-conditions:
+     * output is backed by 64-bit floating point storage
+     * len(binbounds) == len(nbounds) == values.ndim
+     '''
+     
+    cdef:
+        Py_buffer outputview, bbview
+        Py_ssize_t ndim = values.shape[1], npts = values.shape[0]
+        Py_ssize_t idim, ipt, ibound
+        char* outptr_bytes
+        double* outptr
+        real_numeric val, lb, ub
+        real_numeric* boundbuf
+        real_numeric** _binbounds
+        bint store_value
+
+    # Get pointers to the (contiguous) lists of bin boundaries in each dimension
+    _binbounds = <real_numeric**> PyMem_Malloc(ndim*sizeof(real_numeric*))
+    if not _binbounds:
+        raise MemoryError()
+    for idim in range(ndim):
+        PyObject_GetBuffer(binbounds[idim], &bbview, PyBUF_SIMPLE)
+        _binbounds[idim] = <real_numeric*> bbview.buf
+        PyBuffer_Release(&bbview)
+
+    # Get a view of our output array, so we can write directly into it 
+    PyObject_GetBuffer(output, &outputview, PyBUF_STRIDED)
+
+    try:
+        with nogil:
+            # loop over points
+            for ipt in range(npts):
+                outptr_bytes = <char*> outputview.buf
+                store_value = True
+                for idim in range(ndim):
+                    val = values[ipt,idim]
+                    for ibound in range(nbounds[idim]-1):
+                        lb = _binbounds[idim][ibound]
+                        ub = _binbounds[idim][ibound+1]
+                        if val >= lb and val < ub:
+                            outptr_bytes += outputview.strides[idim] * ibound
+                            break
+                    else:
+                        if not ignore_out_of_range:
+                            with gil:
+                                raise ValueError('value {} at index {} out of bin boundaries in dimension {}'
+                                                 .format(val,ipt,idim))
+                        else:
+                            store_value = False
+                if store_value:
+                    outptr = <double*> outptr_bytes
+                    outptr[0] += weights[ipt]
+        return output
+    finally:
+        PyMem_Free(_binbounds)
+        PyBuffer_Release(&outputview)
+
+

westpa/mclib/__init__.py

@@ -0,0 +1,271 @@
+'''A package for performing Monte Carlo bootstrap estimates of
+statistics.'''
+
+import numpy as np
+from numpy.random import Generator, MT19937
+
+from ._mclib import mcbs_correltime, get_bssize, mcbs_ci
+
+
+def msort(input_array):
+    return np.sort(input_array, axis=0)
+
+
+def mcbs_ci_correl(
+    estimator_datasets,
+    estimator,
+    alpha,
+    n_sets=None,
+    args=None,
+    autocorrel_alpha=None,
+    autocorrel_n_sets=None,
+    subsample=None,
+    do_correl=True,
+    mcbs_enable=None,
+    estimator_kwargs={},
+):
+    '''Perform a Monte Carlo bootstrap estimate for the (1-``alpha``) confidence interval
+    on the given ``dataset`` with the given ``estimator``.  This routine is appropriate
+    for time-correlated data, using the method described in Huber & Kim, "Weighted-ensemble
+    Brownian dynamics simulations for protein association reactions" (1996),
+    doi:10.1016/S0006-3495(96)79552-8 to determine a statistically-significant correlation time
+    and then reducing the dataset by a factor of that correlation time before running a "classic"
+    Monte Carlo bootstrap.
+
+    Returns ``(estimate, ci_lb, ci_ub, correl_time)`` where ``estimate`` is the application of the
+    given ``estimator`` to the input ``dataset``, ``ci_lb`` and ``ci_ub`` are the
+    lower and upper limits, respectively, of the (1-``alpha``) confidence interval on
+    ``estimate``, and ``correl_time`` is the correlation time of the dataset, significant to
+    (1-``autocorrel_alpha``).
+
+    ``estimator`` is called as ``estimator(dataset, *args, **kwargs)``. Common estimators include:
+      * np.mean -- calculate the confidence interval on the mean of ``dataset``
+      * np.median -- calculate a confidence interval on the median of ``dataset``
+      * np.std -- calculate a confidence interval on the standard deviation of ``datset``.
+
+    ``n_sets`` is the number of synthetic data sets to generate using the given ``estimator``,
+    which will be chosen using `get_bssize()`_ if ``n_sets`` is not given.
+
+    ``autocorrel_alpha`` (which defaults to ``alpha``) can be used to adjust the significance
+    level of the autocorrelation calculation. Note that too high a significance level (too low an
+    alpha) for evaluating the significance of autocorrelation values can result in a failure to
+    detect correlation if the autocorrelation function is noisy.
+
+    The given ``subsample`` function is used, if provided, to subsample the dataset prior to running
+    the full Monte Carlo bootstrap. If none is provided, then a random entry from each correlated
+    block is used as the value for that block.  Other reasonable choices include ``np.mean``,
+    ``np.median``, ``(lambda x: x[0])`` or ``(lambda x: x[-1])``.  In particular, using
+    ``subsample=np.mean`` will converge to the block averaged mean and standard error,
+    while accounting for any non-normality in the distribution of the mean.
+    '''
+
+    if alpha > 0.5:
+        raise ValueError('alpha ({}) > 0.5'.format(alpha))
+
+    autocorrel_alpha = alpha if not autocorrel_alpha else autocorrel_alpha
+
+    # We're now passing in dataset as a dict, so we need to enforce that for compatibility with older tools.
+    # This just takes our dataset and puts it into a dict, as it's likely that we're using
+    # mean or median as our estimators, which take "a" as argument input.
+    if not isinstance(estimator_datasets, dict):
+        # Enforcing the data structure.
+        pre_calculated = estimator_datasets
+        estimator_datasets = {'a': estimator_datasets}
+        # This also probably means our estimator isn't going to handle kwargs, so we'll watch out for that later in testing.
+        # We may have to replace the 'simple' estimator with a slightly more complex lambda function which simply ditches extra arguments.
+    for key, dset in estimator_datasets.items():
+        estimator_datasets[key] = np.asanyarray(dset)
+        dlen = dset.shape[0]
+
+    # Why do we have 'estimator_datasets'?
+    # Estimators may require many different sets of data to properly function; while we can send this in via the kwargs,
+    # we may wish to decimate only a certain subset (due to the block bootstrapping) of the input parameters.
+    # Therefore, 'estimator_datasets' should consist of datasets that must be sliced/decimated with the subsampling function.
+    # Some estimators (such as the reweighting) may not be able to be decimated in a straightforward manner with a subsample function,
+    # as we cannot pre-estimate the quantity without introducing error or bias.  In those cases, we may wish to pass on all the data,
+    # but ensure that our estimator only includes certain iterations (and only in a certain way).
+
+    n_sets = n_sets or get_bssize(alpha)
+    autocorrel_n_sets = autocorrel_n_sets or get_bssize(autocorrel_alpha)
+
+    if mcbs_enable is False:
+        # While it's odd to support NOT doing the bootstrap in a library specifically designed for bootstrapping,
+        # supporting this functionality here makes writing the code a lot easier, as we can just pass in a flag.
+        # Specifically, this is for situations in which error is not desired (that is, only a reasonable mean is desired).
+        # It's often useful when doing a quick analysis.
+        estimator_datasets.update(estimator_kwargs)
+        try:
+            estimator_datasets.update({'stride': 1})
+        except Exception:
+            pass
+
+        return_set = estimator(**estimator_datasets)
+        # We don't try and pretend we're doing any error analysis.
+        return return_set, return_set, return_set, 0, 1
+
+    rng = Generator(MT19937())  # RNG
+
+    # We need to pre-generate the data; why not do it here?  We're already set up for it...
+    precalc_kwargs = estimator_kwargs.copy()
+    precalc_kwargs['stride'] = 1
+    pre_calculated = []
+    for block in range(1, dlen + 1):
+        for key, dset in estimator_datasets.items():
+            precalc_kwargs[key] = dset[0:block]
+        pre_calculated.append(estimator(**precalc_kwargs))
+    # We need to get rid of any NaNs.
+    pre_calculated = np.asanyarray(pre_calculated)
+    pre_calculated = pre_calculated[np.isfinite(pre_calculated)]
+    # If this happens, we have a huge NaN problem.  That is, our estimator is failing to return meaningful
+    # numbers.  We should catch this when it happens, and so raise an exception, here.
+    # This is almost certainly due to estimator failure.  Double check that calculation.
+    if pre_calculated.shape == (0,):
+        raise NameError("Looks like the estimator failed.  This is likely a programming issue, and should be reported.")
+    # If pre-calculated is not None, we'll use that instead of dataset.
+    # We can also assume that it's a 1 dimensional set with nothing needed, so 'key' should work.
+    if do_correl is True:
+        correl_len = mcbs_correltime(pre_calculated, autocorrel_alpha, autocorrel_n_sets)
+    else:
+        correl_len = 0
+    if correl_len == len(pre_calculated):
+        # too correlated for meaningful calculations
+        estimator_datasets.update(estimator_kwargs)
+        try:
+            estimator_datasets.update({'stride': 1})
+        except Exception:
+            pass
+
+        return estimator(**estimator_datasets), pre_calculated.min(), pre_calculated.max(), (np.std(pre_calculated)), correl_len
+
+    # else, do a blocked bootstrap
+    stride = correl_len + 1
+
+    if stride == 1:
+        # Some estimators may require the stride, so we pass it in.
+        estimator_kwargs['stride'] = stride
+        return mcbs_ci(
+            dataset=estimator_datasets,
+            estimator=estimator,
+            alpha=alpha,
+            dlen=dlen,
+            n_sets=n_sets,
+            args=args,
+            kwargs=estimator_kwargs,
+            sort=msort,
+        ) + (correl_len,)
+    else:
+        subsample = subsample or (lambda x: x[rng.integers(len(x))])
+        # Let's make sure we decimate every array properly...
+        decim_list = {}
+        for key, dset in estimator_datasets.items():
+            dset_shape = list(dset.shape)
+            n_slices = dset_shape[0] // stride
+            dset_shape[0] = n_slices
+            decim_set = np.empty((dset_shape), dtype=dset.dtype)
+            for iout, istart in enumerate(range(0, dset.shape[0] - stride + 1, stride)):
+                sl = dset[istart : istart + stride]
+                # We assume time is the 0th axis.
+                # Okay, so non-optimal.  Population requires the axis subsampling to be done just so...
+                try:
+                    decim_set[iout] = subsample(sl, axis=0)
+                except Exception:
+                    decim_set[iout] = subsample(sl)
+            decim_list[key] = decim_set
+            dlen = dset_shape[0]
+            estimator_kwargs['stride'] = stride
+
+        return mcbs_ci(
+            dataset=decim_list,
+            estimator=estimator,
+            alpha=alpha,
+            dlen=dlen,
+            n_sets=n_sets,
+            args=args,
+            kwargs=estimator_kwargs,
+            sort=msort,
+        ) + (correl_len,)
+
+
+# These are blocks designed to evaluate simple information sets.
+# Whether they should go here or in westtoools is somewhat up for debate.
+# Currently, nothing actually uses them, so there's that.
+
+
+def _1D_simple_eval_block(
+    iblock,
+    start,
+    stop,
+    nstates,
+    data_input,
+    name,
+    mcbs_alpha,
+    mcbs_nsets,
+    mcbs_acalpha,
+    do_correl,
+    mcbs_enable,
+    subsample=np.mean,
+    **extra,
+):
+    # This is actually appropriate for anything with a directly measured, 1D dataset, i.e.,
+    # Fluxes, color populations, and state populations.
+    results = []
+    for istate in range(nstates):
+        # Not sure if we need a jstate for these estimators, but we'll see.
+        # kwargs = {'istate': istate, 'jstate': 'B'}
+        estimator_datasets = {'dataset': data_input['dataset'][:, istate]}
+        ci_res = mcbs_ci_correl(
+            estimator_datasets,
+            estimator=(lambda stride, dataset: np.mean(dataset)),
+            alpha=mcbs_alpha,
+            n_sets=mcbs_nsets,
+            autocorrel_alpha=mcbs_acalpha,
+            subsample=subsample,
+            do_correl=do_correl,
+            mcbs_enable=mcbs_enable,
+        )
+
+        results.append((name, iblock, istate, (start, stop) + ci_res))
+
+    return results
+
+
+def _2D_simple_eval_block(
+    iblock,
+    start,
+    stop,
+    nstates,
+    data_input,
+    name,
+    mcbs_alpha,
+    mcbs_nsets,
+    mcbs_acalpha,
+    do_correl,
+    mcbs_enable,
+    subsample=np.mean,
+    **extra,
+):
+    # This is really just a simple 2D block for less complex datasets, but there it is.
+    # It's probably limited in this use case to conditional_fluxes, but anything that's an i to j process that is directly measured
+    # is suitable for use with this.
+    results = []
+    for istate in range(nstates):
+        for jstate in range(nstates):
+            if istate == jstate:
+                continue
+            # kwargs = {'istate': istate, 'jstate': jstate}
+            # dataset = {'dataset': cond_fluxes[:, istate, jstate]}
+            estimator_datasets = {'dataset': data_input['dataset'][:, istate, jstate]}
+            ci_res = mcbs_ci_correl(
+                estimator_datasets,
+                estimator=(lambda stride, dataset: np.mean(dataset)),
+                alpha=mcbs_alpha,
+                n_sets=mcbs_nsets,
+                autocorrel_alpha=mcbs_acalpha,
+                subsample=subsample,
+                do_correl=do_correl,
+                mcbs_enable=mcbs_enable,
+            )
+
+            results.append((name, iblock, istate, jstate, (start, stop) + ci_res))
+
+    return results

westpa/mclib/__main__.py

@@ -0,0 +1,28 @@
+if __name__ == '__main__':
+    from . import autocorrel_elem
+    import numpy
+    from scipy.signal import correlate
+
+    n = 16
+    x = numpy.linspace(0, n * numpy.pi, 16 * n + 1)
+    a = numpy.cos(x) + numpy.exp(-((x / 2.0) ** 2)) + numpy.exp(-(x / 4.0))
+    pa = numpy.zeros((10000 * len(a),), numpy.float64)
+    pa[: len(a)] = a
+
+    print('<a> =', a.mean())
+    print('<a^2> =', ((a - a.mean()) ** 2).sum())
+    print('scipy.signal.correlate:')
+    acf0 = correlate(a, a)
+    acf0 = acf0[-len(a) :]
+    acf0 /= acf0.max()
+    print(acf0[: len(acf0) / 4])
+
+    #    print 'scipy.signal.correlate (-mean):'
+    #    acf0 = correlate(a-a.mean(),a-a.mean())
+    #    acf0 = acf0[-len(a):]
+    #    acf0 /= acf0.max()
+    #    print acf0[:len(acf0)/4]
+
+    print('this module:')
+    acf = numpy.array([autocorrel_elem(pa, k) for k in range(len(a))])
+    print(acf[: len(acf) / 4])