dclab 0.67.0__cp314-cp314-macosx_10_13_x86_64.whl

dclab/external/statsmodels/nonparametric/_kernel_base.py

@@ -0,0 +1,203 @@
+"""
+Module containing the base object for multivariate kernel density and
+regression, plus some utilities.
+"""
+import numpy as np
+
+from . import kernels
+
+
+kernel_func = dict(gaussian=kernels.gaussian)
+has_joblib = False
+
+
+class GenericKDE(object):
+    """
+    Base class for density estimation and regression KDE classes.
+    """
+
+    def _compute_bw(self, bw):
+        """
+        Computes the bandwidth of the data.
+
+        Parameters
+        ----------
+        bw: array_like or str
+            If array_like: user-specified bandwidth.
+            If a string, should be one of:
+
+                - cv_ml: cross validation maximum likelihood
+                - normal_reference: normal reference rule of thumb
+                - cv_ls: cross validation least squares
+
+        Notes
+        -----
+        The default values for bw is 'normal_reference'.
+        """
+        if bw is None:
+            bw = 'normal_reference'
+
+        if not isinstance(bw, str):
+            self._bw_method = "user-specified"
+            res = np.asarray(bw)
+        else:
+            # The user specified a bandwidth selection method
+            self._bw_method = bw
+            # Workaround to avoid instance methods in __dict__
+            if bw == 'normal_reference':
+                bwfunc = self._normal_reference
+            elif bw == 'cv_ml':
+                bwfunc = self._cv_ml
+            else:  # bw == 'cv_ls'
+                bwfunc = self._cv_ls
+            res = bwfunc()
+
+        return res
+
+    def _set_defaults(self, defaults):
+        """Sets the default values for the efficient estimation"""
+        self.n_res = defaults.n_res
+        self.n_sub = defaults.n_sub
+        self.randomize = defaults.randomize
+        self.return_median = defaults.return_median
+        self.efficient = defaults.efficient
+        self.return_only_bw = defaults.return_only_bw
+        self.n_jobs = defaults.n_jobs
+
+
+class EstimatorSettings(object):
+    """
+    Object to specify settings for density estimation or regression.
+
+    `EstimatorSettings` has several proporties related to how bandwidth
+    estimation for the `KDEMultivariate`, `KDEMultivariateConditional`,
+    `KernelReg` and `CensoredKernelReg` classes behaves.
+
+    Parameters
+    ----------
+    efficient: bool, optional
+        If True, the bandwidth estimation is to be performed
+        efficiently -- by taking smaller sub-samples and estimating
+        the scaling factor of each subsample.  This is useful for large
+        samples (nobs >> 300) and/or multiple variables (k_vars > 3).
+        If False (default), all data is used at the same time.
+    randomize: bool, optional
+        If True, the bandwidth estimation is to be performed by
+        taking `n_res` random resamples (with replacement) of size `n_sub` from
+        the full sample.  If set to False (default), the estimation is
+        performed by slicing the full sample in sub-samples of size `n_sub` so
+        that all samples are used once.
+    n_sub: int, optional
+        Size of the sub-samples.  Default is 50.
+    n_res: int, optional
+        The number of random re-samples used to estimate the bandwidth.
+        Only has an effect if ``randomize == True``.  Default value is 25.
+    return_median: bool, optional
+        If True (default), the estimator uses the median of all scaling factors
+        for each sub-sample to estimate the bandwidth of the full sample.
+        If False, the estimator uses the mean.
+    return_only_bw: bool, optional
+        If True, the estimator is to use the bandwidth and not the
+        scaling factor.  This is *not* theoretically justified.
+        Should be used only for experimenting.
+    n_jobs : int, optional
+        The number of jobs to use for parallel estimation with
+        ``joblib.Parallel``.  Default is -1, meaning ``n_cores - 1``, with
+        ``n_cores`` the number of available CPU cores.
+        See the `joblib documentation
+        <https://pythonhosted.org/joblib/parallel.html>`_ for more details.
+
+    Examples
+    --------
+    >>> settings = EstimatorSettings(randomize=True, n_jobs=3)
+    >>> k_dens = KDEMultivariate(data, var_type, defaults=settings)
+
+    """
+
+    def __init__(self, efficient=False, randomize=False, n_res=25, n_sub=50,
+                 return_median=True, return_only_bw=False, n_jobs=-1):
+        self.efficient = efficient
+        self.randomize = randomize
+        self.n_res = n_res
+        self.n_sub = n_sub
+        self.return_median = return_median
+        self.return_only_bw = return_only_bw  # TODO: remove this?
+        self.n_jobs = n_jobs
+
+
+def _adjust_shape(dat, k_vars):
+    """ Returns an array of shape (nobs, k_vars) for use with `gpke`."""
+    dat = np.asarray(dat)
+    if dat.ndim > 2:
+        dat = np.squeeze(dat)
+    if dat.ndim == 1 and k_vars > 1:  # one obs many vars
+        nobs = 1
+    elif dat.ndim == 1 and k_vars == 1:  # one obs one var
+        nobs = len(dat)
+    else:
+        if np.shape(dat)[0] == k_vars and np.shape(dat)[1] != k_vars:
+            dat = dat.T
+
+        nobs = np.shape(dat)[0]  # ndim >1 so many obs many vars
+
+    dat = np.reshape(dat, (nobs, k_vars))
+    return dat
+
+
+def gpke(bw, data, data_predict, var_type, ckertype='gaussian',
+         okertype='wangryzin', ukertype='aitchisonaitken', tosum=True):
+    r"""
+    Returns the non-normalized Generalized Product Kernel Estimator
+
+    Parameters
+    ----------
+    bw: 1-D ndarray
+        The user-specified bandwidth parameters.
+    data: 1D or 2-D ndarray
+        The training data.
+    data_predict: 1-D ndarray
+        The evaluation points at which the kernel estimation is performed.
+    var_type: str, optional
+        The variable type (continuous, ordered, unordered).
+    ckertype: str, optional
+        The kernel used for the continuous variables.
+    okertype: str, optional
+        The kernel used for the ordered discrete variables.
+    ukertype: str, optional
+        The kernel used for the unordered discrete variables.
+    tosum : bool, optional
+        Whether or not to sum the calculated array of densities.  Default is
+        True.
+
+    Returns
+    -------
+    dens: array-like
+        The generalized product kernel density estimator.
+
+    Notes
+    -----
+    The formula for the multivariate kernel estimator for the pdf is:
+
+    .. math:: f(x)=\frac{1}{nh_{1}...h_{q}}\sum_{i=1}^
+                        {n}K\left(\frac{X_{i}-x}{h}\right)
+
+    where
+
+    .. math:: K\left(\frac{X_{i}-x}{h}\right) =
+                k\left( \frac{X_{i1}-x_{1}}{h_{1}}\right)\times
+                k\left( \frac{X_{i2}-x_{2}}{h_{2}}\right)\times...\times
+                k\left(\frac{X_{iq}-x_{q}}{h_{q}}\right)
+    """
+    kertypes = dict(c=ckertype, o=okertype, u=ukertype)
+
+    Kval = np.empty(data.shape)
+    for ii, vtype in enumerate(var_type):
+        func = kernel_func[kertypes[vtype]]
+        Kval[:, ii] = func(bw[ii], data[:, ii], data_predict[ii])
+
+    iscontinuous = np.array([c == 'c' for c in var_type])
+    dens = Kval.prod(axis=1) / np.prod(bw[iscontinuous])
+    if tosum:
+        return dens.sum(axis=0)
+    else:
+        return dens

dclab/external/statsmodels/nonparametric/kernel_density.py

@@ -0,0 +1,165 @@
+"""
+Multivariate Conditional and Unconditional Kernel Density Estimation
+with Mixed Data Types.
+
+References
+----------
+[1] Racine, J., Li, Q. Nonparametric econometrics: theory and practice.
+    Princeton University Press. (2007)
+[2] Racine, Jeff. "Nonparametric Econometrics: A Primer," Foundation
+    and Trends in Econometrics: Vol 3: No 1, pp1-88. (2008)
+    https://dx.doi.org/10.1561/0800000009
+[3] Racine, J., Li, Q. "Nonparametric Estimation of Distributions
+    with Categorical and Continuous Data." Working Paper. (2000)
+[4] Racine, J. Li, Q. "Kernel Estimation of Multivariate Conditional
+    Distributions Annals of Economics and Finance 5, 211-235 (2004)
+[5] Liu, R., Yang, L. "Kernel estimation of multivariate
+    cumulative distribution function."
+    Journal of Nonparametric Statistics (2008)
+[6] Li, R., Ju, G. "Nonparametric Estimation of Multivariate CDF
+    with Categorical and Continuous Data." Working Paper
+[7] Li, Q., Racine, J. "Cross-validated local linear nonparametric
+    regression" Statistica Sinica 14(2004), pp. 485-512
+[8] Racine, J.: "Consistent Significance Testing for Nonparametric
+        Regression" Journal of Business & Economics Statistics
+[9] Racine, J., Hart, J., Li, Q., "Testing the Significance of
+        Categorical Predictor Variables in Nonparametric Regression
+        Models", 2006, Econometric Reviews 25, 523-544
+
+"""
+
+import numpy as np
+
+from ._kernel_base import GenericKDE, EstimatorSettings, gpke, _adjust_shape
+
+
+__all__ = ['KDEMultivariate', 'EstimatorSettings']
+
+
+class KDEMultivariate(GenericKDE):
+    """
+    Multivariate kernel density estimator.
+
+    This density estimator can handle univariate as well as multivariate data,
+    including mixed continuous / ordered discrete / unordered discrete data.
+    It also provides cross-validated bandwidth selection methods (least
+    squares, maximum likelihood).
+
+    Parameters
+    ----------
+    data: list of ndarrays or 2-D ndarray
+        The training data for the Kernel Density Estimation, used to determine
+        the bandwidth(s).  If a 2-D array, should be of shape
+        (num_observations, num_variables).  If a list, each list element is a
+        separate observation.
+    var_type: str
+        The type of the variables:
+
+            - c : continuous
+            - u : unordered (discrete)
+            - o : ordered (discrete)
+
+        The string should contain a type specifier for each variable, so for
+        example ``var_type='ccuo'``.
+    bw: array_like or str, optional
+        If an array, it is a fixed user-specified bandwidth.  If a string,
+        should be one of:
+
+            - normal_reference: normal reference rule of thumb (default)
+            - cv_ml: cross validation maximum likelihood
+            - cv_ls: cross validation least squares
+
+    defaults: EstimatorSettings instance, optional
+        The default values for (efficient) bandwidth estimation.
+
+    Attributes
+    ----------
+    bw: array_like
+        The bandwidth parameters.
+
+    See Also
+    --------
+    KDEMultivariateConditional
+
+    Examples
+    --------
+    >>> import statsmodels.api as sm
+    >>> nobs = 300
+    >>> np.random.seed(1234)  # Seed random generator
+    >>> c1 = np.random.normal(size=(nobs,1))
+    >>> c2 = np.random.normal(2, 1, size=(nobs,1))
+
+    Estimate a bivariate distribution and display the bandwidth found:
+
+    >>> dens_u = sm.nonparametric.KDEMultivariate(data=[c1,c2],
+    ...     var_type='cc', bw='normal_reference')
+    >>> dens_u.bw
+    array([ 0.39967419,  0.38423292])
+    """
+
+    def __init__(self, data, var_type, bw=None, defaults=None):
+        self.var_type = var_type
+        self.k_vars = len(self.var_type)
+        self.data = _adjust_shape(data, self.k_vars)
+        self.data_type = var_type
+        self.nobs, self.k_vars = np.shape(self.data)
+        if self.nobs <= self.k_vars:
+            raise ValueError("The number of observations must be larger "
+                             "than the number of variables.")
+        defaults = EstimatorSettings() if defaults is None else defaults
+        self._set_defaults(defaults)
+        if not self.efficient:
+            self.bw = self._compute_bw(bw)
+        else:
+            self.bw = self._compute_efficient(bw)
+
+    def __repr__(self):
+        """Provide something sane to print."""
+        rpr = "KDE instance\n"
+        rpr += "Number of variables: k_vars = " + str(self.k_vars) + "\n"
+        rpr += "Number of samples:   nobs = " + str(self.nobs) + "\n"
+        rpr += "Variable types:      " + self.var_type + "\n"
+        rpr += "BW selection method: " + self._bw_method + "\n"
+        return rpr
+
+    def pdf(self, data_predict=None):
+        r"""
+        Evaluate the probability density function.
+
+        Parameters
+        ----------
+        data_predict: array_like, optional
+            Points to evaluate at.  If unspecified, the training data is used.
+
+        Returns
+        -------
+        pdf_est: array_like
+            Probability density function evaluated at `data_predict`.
+
+        Notes
+        -----
+        The probability density is given by the generalized product kernel
+        estimator:
+
+        .. math:: K_{h}(X_{i},X_{j}) =
+            \prod_{s=1}^{q}h_{s}^{-1}k\left(\frac{X_{is}-X_{js}}{h_{s}}\right)
+        """
+        if data_predict is None:
+            data_predict = self.data
+        else:
+            data_predict = _adjust_shape(data_predict, self.k_vars)
+
+        pdf_est = []
+        for i in range(np.shape(data_predict)[0]):
+            pdf_est.append(gpke(self.bw, data=self.data,
+                                data_predict=data_predict[i, :],
+                                var_type=self.var_type) / self.nobs)
+
+        pdf_est = np.squeeze(pdf_est)
+        return pdf_est
+
+    def _get_class_vars_type(self):
+        """Helper method to be able to pass needed vars to _compute_subset."""
+        class_type = 'KDEMultivariate'
+        class_vars = (self.var_type, )
+        return class_type, class_vars

dclab/external/statsmodels/nonparametric/kernels.py

@@ -0,0 +1,36 @@
+"""
+Module of kernels that are able to handle continuous as well as categorical
+variables (both ordered and unordered).
+
+This is a slight deviation from the current approach in
+statsmodels.nonparametric.kernels where each kernel is a class object.
+
+Having kernel functions rather than classes makes extension to a multivariate
+kernel density estimation much easier.
+
+NOTE: As it is, this module does not interact with the existing API
+"""
+
+
+import numpy as np
+
+
+def gaussian(h, Xi, x):
+    """
+    Gaussian Kernel for continuous variables
+    Parameters
+    ----------
+    h : 1-D ndarray, shape (K,)
+        The bandwidths used to estimate the value of the kernel function.
+    Xi : 1-D ndarray, shape (K,)
+        The value of the training set.
+    x : 1-D ndarray, shape (K,)
+        The value at which the kernel density is being estimated.
+
+    Returns
+    -------
+    kernel_value : ndarray, shape (nobs, K)
+        The value of the kernel function at each training point for each var.
+
+    """
+    return (1. / np.sqrt(2 * np.pi)) * np.exp(-(Xi - x)**2 / (h**2 * 2.))

dclab/features/__init__.py

@@ -0,0 +1,9 @@
+"""Basic methods for event feature computation"""
+from . import contour  # noqa: F401
+from . import bright  # noqa: F401
+from . import bright_bc  # noqa: F401
+from . import bright_perc  # noqa: F401
+from . import emodulus  # noqa: F401
+from . import fl_crosstalk  # noqa: F401
+from . import inert_ratio  # noqa: F401
+from . import volume  # noqa: F401

dclab/features/bright.py

@@ -0,0 +1,81 @@
+"""
+Computation of mean and standard deviation of grayscale values inside the
+RT-DC event image mask.
+"""
+import numpy as np
+
+
+def get_bright(mask, image, ret_data="avg,sd"):
+    """Compute avg and/or std of the event brightness
+
+    The event brightness is defined by the gray-scale values of the
+    image data within the event mask area.
+
+    Parameters
+    ----------
+    mask: ndarray or list of ndarrays of shape (M,N) and dtype bool
+        The mask values, True where the event is located in `image`.
+    image: ndarray or list of ndarrays of shape (M,N)
+        A 2D array that holds the image in form of grayscale values
+        of an event.
+    ret_data: str
+        A comma-separated list of metrices to compute
+        - "avg": compute the average
+        - "sd": compute the standard deviation
+        Selected metrics are returned in alphabetical order.
+
+    Returns
+    -------
+    bright_avg: float or ndarray of size N
+        Average image data within the contour
+    bright_std: float or ndarray of size N
+        Standard deviation of image data within the contour
+    """
+    # This method is based on a pull request by Maik Herbig.
+    ret_avg = "avg" in ret_data
+    ret_std = "sd" in ret_data
+
+    if ret_avg + ret_std == 0:
+        raise ValueError("No valid metrices selected!")
+
+    if isinstance(mask, np.ndarray) and len(mask.shape) == 2:
+        # We have a single image
+        image = [image]
+        mask = [mask]
+        ret_list = False
+    else:
+        ret_list = True
+
+    length = min(len(mask), len(image))
+
+    # Results are stored in a separate array initialized with nans
+    if ret_avg:
+        avg = np.zeros(length, dtype=np.float64) * np.nan
+    if ret_std:
+        std = np.zeros(length, dtype=np.float64) * np.nan
+
+    for ii in range(length):
+        imgi = image[ii]
+        mski = mask[ii]
+        # Assign results
+        if ret_avg:
+            avg[ii] = np.mean(imgi[mski])
+        if ret_std:
+            std[ii] = np.std(imgi[mski])
+
+    results = []
+    # Keep alphabetical order
+    if ret_avg:
+        results.append(avg)
+    if ret_std:
+        results.append(std)
+
+    if not ret_list:
+        # Only return scalars
+        results = [r[0] for r in results]
+
+    if ret_avg + ret_std == 1:
+        # Only return one column
+        return results[0]
+
+    return results

dclab/features/bright_bc.py

@@ -0,0 +1,93 @@
+"""
+Computation of mean and standard deviation of grayscale values inside the
+RT-DC event image mask with background-correction taken into account.
+"""
+import numpy as np
+
+
+def get_bright_bc(mask, image, image_bg, bg_off=None, ret_data="avg,sd"):
+    """Compute avg and/or std of the background-corrected event brightness
+
+    The background-corrected event brightness is defined by the
+    gray-scale values of the background-corrected image data
+    within the event mask area.
+
+    Parameters
+    ----------
+    mask: ndarray or list of ndarrays of shape (M,N) and dtype bool
+        The mask values, True where the event is located in `image`.
+    image: ndarray or list of ndarrays of shape (M,N)
+        A 2D array that holds the image in form of grayscale values
+        of an event.
+    image_bg: ndarray or list of ndarrays of shape (M,N)
+        A 2D array that holds the background image for the same event.
+    bg_off: float or 1D ndarray
+        Additional offset value that is added to `image_bg` before
+        background correction
+    ret_data: str
+        A comma-separated list of metrices to compute
+        - "avg": compute the average
+        - "sd": compute the standard deviation
+        Selected metrics are returned in alphabetical order.
+
+    Returns
+    -------
+    bright_avg: float or ndarray of size N
+        Average image data within the contour
+    bright_std: float or ndarray of size N
+        Standard deviation of image data within the contour
+    """
+    # This method is based on a pull request by Maik Herbig.
+    ret_avg = "avg" in ret_data
+    ret_std = "sd" in ret_data
+
+    if ret_avg + ret_std == 0:
+        raise ValueError("No valid metrices selected!")
+
+    if isinstance(mask, np.ndarray) and len(mask.shape) == 2:
+        # We have a single image
+        image_bg = [image_bg]
+        image = [image]
+        mask = [mask]
+        if bg_off is not None:
+            bg_off = np.atleast_1d(bg_off)
+        ret_list = False
+    else:
+        ret_list = True
+
+    length = min(len(mask), len(image), len(image_bg))
+
+    # Results are stored in a separate array initialized with nans
+    if ret_avg:
+        avg = np.zeros(length, dtype=np.float64) * np.nan
+    if ret_std:
+        std = np.zeros(length, dtype=np.float64) * np.nan
+
+    for ii in range(length):
+        # cast to integer before subtraction
+        imgi = np.array(image[ii], dtype=int) - image_bg[ii]
+        mski = mask[ii]
+        # Assign results
+        if ret_avg:
+            avg[ii] = np.mean(imgi[mski])
+        if ret_std:
+            std[ii] = np.std(imgi[mski])
+
+    results = []
+    # Keep alphabetical order
+    if ret_avg:
+        if bg_off is not None:
+            avg -= bg_off
+        results.append(avg)
+    if ret_std:
+        results.append(std)
+
+    if not ret_list:
+        # Only return scalars
+        results = [r[0] for r in results]
+
+    if ret_avg + ret_std == 1:
+        # Only return one column
+        return results[0]
+
+    return results

dclab/features/bright_perc.py

@@ -0,0 +1,63 @@
+"""
+Computation of the 10th and 90th percentile of grayscale values inside the
+RT-DC event image mask with background-correction taken into account.
+"""
+import numpy as np
+
+
+def get_bright_perc(mask, image, image_bg, bg_off=None):
+    """Compute 10th and 90th percentile of the bg-corrected event brightness
+
+    The background-corrected event brightness is defined by the
+    gray-scale values of the background-corrected image data
+    within the event mask area.
+
+    Parameters
+    ----------
+    mask: ndarray or list of ndarrays of shape (M,N) and dtype bool
+        The mask values, True where the event is located in `image`.
+    image: ndarray or list of ndarrays of shape (M,N)
+        A 2D array that holds the image in form of grayscale values
+        of an event.
+    image_bg: ndarray or list of ndarrays of shape (M,N)
+        A 2D array that holds the background image for the same event.
+    bg_off: float or 1D ndarray
+        Additional offset value that is added to `image_bg` before
+        background correction
+
+    Returns
+    -------
+    bright_perc_10: float or ndarray of size N
+        10th percentile of brightness
+    bright_perc_10: float or ndarray of size N
+        90th percentile of brightness
+    """
+    if isinstance(mask, np.ndarray) and len(mask.shape) == 2:
+        # We have a single image
+        image_bg = [image_bg]
+        image = [image]
+        mask = [mask]
+        ret_list = False
+    else:
+        ret_list = True
+
+    length = min(len(mask), len(image), len(image_bg))
+
+    p10 = np.zeros(length, dtype=np.float64) * np.nan
+    p90 = np.zeros(length, dtype=np.float64) * np.nan
+
+    for ii in range(length):
+        # cast to integer before subtraction
+        imgi = np.array(image[ii], dtype=int) - image_bg[ii]
+        mski = mask[ii]
+        # Assign results
+        p10[ii], p90[ii] = np.percentile(imgi[mski], q=[10, 90])
+
+    if bg_off:
+        p10 -= bg_off
+        p90 -= bg_off
+
+    if ret_list:
+        return p10, p90
+    else:
+        return p10[0], p90[0]