machinegnostics 0.0.1__py3-none-any.whl

machinegnostics/magcal/gdf/scedasticity.py

@@ -0,0 +1,197 @@
+'''
+Gnostic - Homoscedasticity and Heteroscedasticity
+
+This module to check for homoscedasticity and heteroscedasticity in data.
+
+Author: Nirmal Parmar
+Machine Gnostics
+'''
+import numpy as np
+import logging
+from machinegnostics.magcal.util.logging import get_logger
+
+class DataScedasticity:
+    """
+    Gnostic Scedasticity Test for Homoscedasticity and Heteroscedasticity
+
+    This class provides a method to check for homoscedasticity and heteroscedasticity in data,
+    inspired by fundamental principles rather than standard statistical tests. Unlike classical
+    approaches, this implementation uses gnostic variance and gnostic linear regression, which are
+    based on the Machine Gnostics framework.
+
+    Key Differences from Standard Methods:
+    - **Variance Calculation:** The variance used here is the gnostic variance, which may differ in
+      definition and properties from classical statistical variance. It is designed to capture
+      uncertainty and spread in a way that aligns with gnostic principles.
+    - **Regression Model:** The linear regression model employed is a gnostic linear regression,
+      not the standard least squares regression. This model is tailored to the gnostic approach and
+      may use different loss functions, optimization criteria, or regularization.
+    - **Test Philosophy:** This is not a formal statistical test (such as Breusch-Pagan or White's test),
+      but rather a diagnostic inspired by the fundamentals of the gnostic framework. The method splits
+      residuals based on the median of the independent variable and compares the gnostic variances of
+      the squared residuals in each half.
+
+    Usage:
+        1. Initialize the class with desired gnostic regression parameters.
+        2. Call `fit(x, y)` with your data.
+        3. Check the `is_homoscedastic` attribute or returned value to determine if the data is
+           homoscedastic (equal gnostic variance across splits) or heteroscedastic.
+
+    Attributes:
+        x (np.ndarray): Independent variable data.
+        y (np.ndarray): Dependent variable data.
+        model (LinearRegressor): Gnostic linear regression model.
+        residuals (np.ndarray): Residuals from the fitted model.
+        params (dict): Stores calculated variances and variance ratio.
+        variance_ratio (float): Ratio of gnostic variances between data splits.
+        is_homoscedastic (bool): True if data is homoscedastic under gnostic test, else False.
+
+    Example:
+        >>> import numpy as np
+        >>> from machinegnostics.magcal import DataScedasticity
+        >>> x = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
+        >>> y = np.array([2.1, 4.2, 6.1, 8.3, 10.2, 12.1, 14.2, 16.1, 18.2, 20.1])
+        >>> sced = DataScedasticity()
+        >>> is_homo = sced.fit(x, y)
+        >>> print(f"Is data homoscedastic? {is_homo}")
+        >>> print(f"Variance ratio: {sced.variance_ratio}")
+
+    Note:
+        This class is intended for users interested in gnostic data analysis. Results and interpretations
+        may not align with classical statistical methods. For more details on gnostic variance and regression,
+        refer to the Machine Gnostics documentation.
+    """
+
+    def __init__(self,
+                 scale: str | int | float = 'auto',
+                 max_iter: int = 100,
+                 tol: float = 0.001,
+                 mg_loss: str = 'hi',
+                 early_stopping: bool = True,
+                 verbose: bool = False,
+                 data_form: str = 'a',
+                 gnostic_characteristics: bool = True,
+                 history: bool = True):
+        
+        from machinegnostics.models.regression import LinearRegressor
+        self.x = None
+        self.y = None
+        self.model = LinearRegressor(scale=scale,
+                                     max_iter=max_iter,
+                                     tol=tol,
+                                     mg_loss=mg_loss,
+                                     early_stopping=early_stopping,
+                                     verbose=verbose,
+                                     data_form=data_form,
+                                     gnostic_characteristics=gnostic_characteristics,
+                                     history=history)
+        self.residuals = None
+        self.params = {}
+        self.logger = get_logger(self.__class__.__name__, logging.DEBUG if verbose else logging.WARNING)
+        self.logger.debug(f"{self.__class__.__name__} initialized:")
+
+
+    def _split_residuals(self):
+        """
+        Split residuals into two halves based on the median of x. zip x and residuals.
+        sorted(zip(x, residuals))
+        """
+        self.logger.info("Splitting residuals based on median of x.")
+        median_x = np.median(self.x)
+        left_half = [(xi, ri) for xi, ri in zip(self.x, self.residuals) if xi <= median_x]
+        right_half = [(xi, ri) for xi, ri in zip(self.x, self.residuals) if xi > median_x]
+        return left_half, right_half
+    
+    def _variance_ratio(self):
+        """
+        Calculate the variance ratio of the squared residuals in the two halves.
+
+        Returns:
+            float: Variance ratio of the squared residuals.
+        """
+        from machinegnostics import variance
+        self.logger.info("Calculating variance ratio.")
+        left_half, right_half = self._split_residuals()
+        left_residuals = np.array([ri for xi, ri in left_half])
+        right_residuals = np.array([ri for xi, ri in right_half])
+        var_left = variance(left_residuals ** 2)
+        var_right = variance(right_residuals ** 2)
+
+        self.logger.debug(f"Left variance: {var_left}, Right variance: {var_right}")
+        # cap values between [1, 1e-9]
+        var_left = float(var_left)
+        var_right = float(np.maximum(var_right, 1e-9)) # to avoid division by zero
+        if var_right == 0 and var_left == 0:
+            variance_ratio = 1.0
+        elif var_right == 0:
+            variance_ratio = np.inf
+        else:
+            variance_ratio = var_left / var_right
+
+        # params
+        self.logger.info(f"Variance ratio calculated: {variance_ratio}")
+        self.params['var_left'] = var_left
+        self.params['var_right'] = var_right
+        self.params['variance_ratio'] = variance_ratio
+        return variance_ratio
+        
+    
+    def _is_homoscedastic(self, threshold: float = 0.001):
+        """
+        Check if the data is homoscedastic based on the variance ratio.
+
+        Args:
+            threshold (float): Threshold to determine homoscedasticity.
+
+        Returns:
+            bool: True if homoscedastic, False if heteroscedastic.
+        """
+        if self.variance_ratio is None:
+            self.logger.error("Variance ratio not calculated. Please run fit() first.")
+            raise ValueError("Variance ratio not calculated. Please run fit() first.")
+        return abs(self.variance_ratio - 1) < threshold
+
+    def fit(self, x: np.ndarray, y: np.ndarray) -> bool:
+        """
+        Fit the gnostic linear regression model to the data and assess scedasticity.
+
+        This method fits the gnostic linear regression model to the provided data, computes the residuals,
+        and evaluates homoscedasticity or heteroscedasticity using the gnostic variance approach. Unlike
+        standard statistical tests, this method uses gnostic variance and gnostic regression, which are
+        based on the Machine Gnostics framework and may yield different results from classical methods.
+
+        The method splits the data based on the median of the independent variable, calculates the gnostic
+        variance of squared residuals in each half, and determines if the data is homoscedastic (equal
+        gnostic variance) or heteroscedastic.
+
+        Args:
+            x (np.ndarray): Independent variable data.
+            y (np.ndarray): Dependent variable data.
+
+        Returns:
+            bool: True if data is homoscedastic under the gnostic test, False if heteroscedastic.
+
+        Note:
+            This is not a standard statistical test. For details on the gnostic approach, see the
+            Machine Gnostics documentation.
+        """
+        self.logger.info("Fitting DataScedasticity model...")
+        self.x = x
+        self.y = y
+
+        self.logger.info("Fitting gnostic regression model.")
+        self.model.fit(x, y)
+        self.logger.debug(f"Model calculations complete.")
+
+        self.logger.info("Calculating residuals.")  
+        self.residuals = y - self.model.predict(x)
+
+        # calculate variance ratio
+        self.logger.info("Calculating variance ratio.")
+        self.variance_ratio = self._variance_ratio()
+
+        # check
+        self.logger.info("Checking homoscedasticity.")
+        self.is_homoscedastic = self._is_homoscedastic()
+        self.logger.info(f"Homoscedasticity check result - is_homoscedastic: {self.is_homoscedastic}")
+        return self.is_homoscedastic

machinegnostics/magcal/gdf/wedf.py

@@ -0,0 +1,181 @@
+from machinegnostics.magcal.util.logging import get_logger
+import numpy as np
+import logging
+
+class WEDF:
+    """
+    Weighted Empirical Distribution Function (WEDF)
+    
+    This class implements the WEDF that accounts for data weights, which is useful
+    when dealing with repeated values or data points of varying importance.
+    """
+    
+    def __init__(self, data, weights=None, data_lb=None, data_ub=None, verbose=False):
+        """
+        Initialize the WEDF with data points and optional weights.
+        
+        Parameters
+        ----------
+        data : array-like
+            Input data values
+        weights : array-like, optional
+            A priori weights for each data point. If None, equal weights are assigned.
+        data_lb : float, optional
+            Lower bound for the data range
+        data_ub : float, optional
+            Upper bound for the data range
+        verbose : bool, optional
+            If True, set logging level to DEBUG. Default is False.
+        """
+        self.logger = get_logger(self.__class__.__name__, logging.DEBUG if verbose else logging.WARNING)
+        self.logger.debug(f"{self.__class__.__name__} initialized with parameters: %s", self.__dict__)
+
+        # Convert inputs to numpy arrays and sort data
+        self.data = np.asarray(data)
+        if data_lb is None:
+            self.data_lb = np.min(self.data)
+        else:
+            self.data_lb = data_lb
+        if data_ub is None:
+            self.data_ub = np.max(self.data)
+        else:
+            self.data_ub = data_ub
+        if self.data_lb >= self.data_ub:
+            self.logger.info("data_lb must be less than data_ub")
+        if self.data.size == 0:
+            self.logger.error("data must contain at least one element")
+            raise ValueError("data must contain at least one element")
+        if not np.issubdtype(self.data.dtype, np.number):
+            self.logger.error("data must be numeric")
+            raise ValueError("data must be numeric")
+        
+        # Sort data and corresponding weights
+        sort_idx = np.argsort(self.data)
+        self.data = self.data[sort_idx]
+        
+        if weights is None:
+            # Equal weights if none provided
+            self.weights = np.ones_like(self.data)
+        else:
+            weights = np.asarray(weights)
+            self.weights = weights[sort_idx]
+            
+        # Normalize weights
+        self.normalized_weights = self.weights / np.sum(self.weights)
+        
+        # Calculate WEDF values
+        self._calculate_wedf()
+    
+    def _calculate_wedf(self):
+        """Calculate the WEDF values at each data point."""
+        n = len(self.data)
+        self.wedf_values = np.zeros(n)
+        
+        # First value
+        self.wedf_values[0] = self.normalized_weights[0] / 2
+        
+        # Remaining values using recursive relation
+        for k in range(1, n):
+            self.wedf_values[k] = (self.wedf_values[k-1] + 
+                                  (self.normalized_weights[k-1] + self.normalized_weights[k]) / 2)
+            
+    def fit(self, z):
+        """
+        Fit the WEDF at given points.
+        
+        Parameters
+        ----------
+        z : float or array-like
+            Points at which to fit the WEDF
+        
+        Returns
+        -------
+        float or ndarray
+            WEDF values at the given points
+        """
+        self.logger.info("Fitting WEDF at given points.")
+        z = np.asarray(z)
+        single_value = z.ndim == 0
+        
+        if single_value:
+            z = np.array([z])
+            
+        result = np.zeros_like(z, dtype=float)
+        
+        for i, point in enumerate(z):
+            if point <= self.data[0]:
+                result[i] = 0.0
+            elif point >= self.data[-1]:
+                result[i] = 1.0
+            else:
+                # Find the index of the largest data point less than z
+                idx = np.searchsorted(self.data, point) - 1
+                result[i] = self.wedf_values[idx]
+        
+        self.logger.info("WEDF fitting completed.")
+        return result[0] if single_value else result
+    
+    def plot(self, ax=None):
+        """
+        Plot the WEDF.
+        
+        Parameters
+        ----------
+        ax : matplotlib.axes.Axes, optional
+            Axes to plot on. If None, a new figure and axes are created.
+            
+        Returns
+        -------
+        matplotlib.axes.Axes
+            The axes containing the plot
+        """
+        try:
+            import matplotlib.pyplot as plt
+            if ax is None:
+                fig, ax = plt.subplots()
+                
+            # Create a step function representation
+            x = np.repeat(self.data, 2)[1:]
+            y = np.repeat(self.wedf_values, 2)[:-1]
+            
+            # Add endpoints for proper step function
+            x = np.concatenate([[self.data[0]], x, [self.data[-1]]])
+            y = np.concatenate([[0], y, [1]])
+            
+            ax.plot(x, y, 'b-', label='WEDF')
+            ax.set_xlabel('Data Value')
+            ax.set_ylabel('Cumulative Probability')
+            ax.set_title('Weighted Empirical Distribution Function')
+            ax.grid(True)
+            return ax
+            
+        except ImportError:
+            self.logger.warning("Matplotlib is required for plotting.")
+            return None
+        
+    def generate_ks_points(self, num_points=None):
+        """
+        Generate Kolmogorov-Smirnov points for distribution fitting.
+        
+        Parameters
+        ----------
+        num_points : int, optional
+            Number of K-S points to generate. If None, uses the length of the data.
+        Returns
+        -------
+        Z0 : ndarray
+            Generated K-S points
+        ks_probs : ndarray
+            Corresponding probabilities for the K-S points
+        """
+        # Use data length if not specified
+        L = num_points if num_points is not None else len(self.data)
+
+        # Generate K-S probabilities
+        ks_probs = np.arange(1, 2*L, 2) / (2*L)
+
+        # Generate corresponding points
+        data_range = self.data_ub - self.data_lb
+        Z0 = self.data_lb + data_range * ks_probs
+
+        return Z0, ks_probs