QuantileFlow 0.0.1__py3-none-any.whl

QuantileFlow/__init__.py

@@ -0,0 +1,31 @@
+"""
+QuantileFlow: Efficient Quantile Computation for Anomaly Detection
+
+This package provides APIs and algorithms to efficiently compute quantiles for anomaly detection
+in service and system logs. It implements multiple sketching algorithms optimized for:
+
+- Low memory footprint
+- Fast updates and queries
+- Distributed computation support through mergeable sketches
+- Accuracy guarantees for quantile approximation
+
+The package includes three main implementations:
+
+1. DDSketch: A deterministic algorithm with relative error guarantees
+2. MomentSketch: A moment-based algorithm using maximum entropy optimization
+3. HDRHistogram: A high dynamic range histogram for tracking values across wide ranges
+
+All implementations are designed to handle high-throughput data streams and provide
+accurate quantile estimates with minimal memory overhead.
+"""
+from QuantileFlow.momentsketch.core import MomentSketch
+from QuantileFlow.hdrhistogram.core import HDRHistogram
+
+__version__ = "0.0.1"
+__all__ = [
+    "MomentSketch",
+    "HDRHistogram",
+]
+
+if __name__ == "__main__":
+    print("This is root of QuantileFlow module. API not to be exposed as a script!")

QuantileFlow/momentsketch/__init__.py

@@ -0,0 +1,22 @@
+"""
+MomentSketch: Quantile Estimation Using Moment-Based Sketching
+
+This module provides an efficient implementation of the moment-based sketching algorithm
+for quantile estimation and summary statistics. Key features include:
+
+- Memory efficiency: Uses a fixed number of moments regardless of data size
+- Mergeable: Supports distributed computation through sketch merging
+- Accurate: Employs maximum entropy optimization for accurate distribution estimation
+- Flexible: Supports optional data compression for handling widely distributed values
+- Comprehensive: Provides various summary statistics beyond just quantiles
+
+The implementation is based on power sums and maximum entropy optimization,
+making it suitable for streaming data applications where memory efficiency
+and accuracy are important.
+"""
+
+from .core import MomentSketch
+
+__all__ = [
+    "MomentSketch"
+] 

QuantileFlow/momentsketch/core.py

@@ -0,0 +1,200 @@
+"""
+Core MomentSketch implementation.
+
+This module provides the main MomentSketch class which serves as the public API
+for the moment-based quantile estimation algorithm. The MomentSketch implementation:
+
+- Maintains a compact representation of a dataset using power sums
+- Provides accurate quantile estimates through maximum entropy optimization
+- Supports merging sketches for distributed computation
+- Offers comprehensive summary statistics and visualization capabilities
+- Handles data compression for widely distributed values
+
+The implementation is designed to be memory-efficient and accurate, making it suitable
+for streaming data applications and monitoring systems where traditional approaches
+would require excessive memory.
+"""
+
+from typing import List, Union, Dict
+import numpy as np
+
+from .simple_moment_sketch import SimpleMS
+
+class MomentSketch:
+    """
+    MomentSketch implementation for quantile approximation using the moment-based approach.
+    
+    This implementation uses power sums, Chebyshev moment conversion, and maximum entropy
+    optimization to estimate the probability distribution of data and compute quantiles.
+    It supports merging sketches from distributed sources and provides accurate quantile
+    estimates with a compact representation.
+    
+    Reference:
+        "Space- and Computationally-Efficient Set Similarity via Locality Sensitive Sketching"
+        by Anshumali Shrivastava
+    """
+    
+    def __init__(
+        self,
+        num_moments: int = 20,
+        compress_values: bool = False
+    ):
+        """
+        Initialize MomentSketch.
+        
+        Args:
+            num_moments: Number of moments to track (default 20).
+                         Higher values increase accuracy at the cost of computation.
+            compress_values: Whether to compress values using arcsinh transformation (default False).
+                            Useful for handling widely distributed data with extreme values.
+        """
+        self.sketch = SimpleMS(num_moments)
+        self.sketch.set_compressed(compress_values)
+    
+    def insert(self, value: Union[int, float]) -> None:
+        """
+        Insert a single value into the sketch.
+        
+        Args:
+            value: The value to insert.
+        """
+        self.sketch.add(value)
+    
+    def insert_batch(self, values: Union[List[float], np.ndarray]) -> None:
+        """
+        Insert multiple values into the sketch.
+        
+        Args:
+            values: Array or list of values to insert.
+        """
+        self.sketch.add_many(values)
+    
+    def merge(self, other: 'MomentSketch') -> None:
+        """
+        Merge another MomentSketch into this one.
+        
+        Args:
+            other: Another MomentSketch instance to merge.
+        
+        Raises:
+            ValueError: If the sketches are incompatible (different compression settings).
+        """
+        self.sketch.merge(other.sketch)
+    
+    def quantile(self, fraction: float) -> float:
+        """
+        Get the value at a given quantile.
+        
+        Args:
+            fraction: Quantile fraction between 0 and 1 (e.g., 0.5 for median).
+        
+        Returns:
+            Estimated value at the requested quantile.
+            
+        Raises:
+            ValueError: If fraction is not between 0 and 1.
+        """
+        if not 0 <= fraction <= 1:
+            raise ValueError("Quantile must be between 0 and 1")
+            
+        return self.sketch.get_quantile(fraction)
+    
+    def quantiles(self, fractions: List[float]) -> List[float]:
+        """
+        Get values at multiple quantiles.
+        
+        Args:
+            fractions: List of quantile fractions between 0 and 1.
+        
+        Returns:
+            List of estimated values at the requested quantiles.
+            
+        Raises:
+            ValueError: If any fraction is not between 0 and 1.
+        """
+        for q in fractions:
+            if not 0 <= q <= 1:
+                raise ValueError("All quantiles must be between 0 and 1")
+                
+        return self.sketch.get_quantiles(fractions)
+    
+    def median(self) -> float:
+        """
+        Get the median value (50th percentile).
+        
+        Returns:
+            Estimated median value.
+        """
+        return self.sketch.get_median()
+    
+    def percentile(self, p: float) -> float:
+        """
+        Get the p-th percentile value.
+        
+        Args:
+            p: Percentile between 0 and 100 (e.g., 75 for 75th percentile).
+        
+        Returns:
+            Estimated value at the requested percentile.
+            
+        Raises:
+            ValueError: If p is not between 0 and 100.
+        """
+        if not 0 <= p <= 100:
+            raise ValueError("Percentile must be between 0 and 100")
+            
+        return self.sketch.get_percentile(p)
+    
+    def interquartile_range(self) -> float:
+        """
+        Get the interquartile range (IQR).
+        
+        Returns:
+            Estimated IQR (difference between 75th and 25th percentiles).
+        """
+        return self.sketch.get_iqr()
+    
+    def summary_statistics(self) -> Dict[str, float]:
+        """
+        Get summary statistics.
+        
+        Returns:
+            Dictionary containing min, q1, median, q3, max, count, and mean.
+        """
+        return self.sketch.get_stats()
+    
+    def plot_distribution(self, figsize=(10, 6)):
+        """
+        Plot the estimated probability distribution.
+        
+        Args:
+            figsize: Figure size (width, height) in inches.
+        
+        Returns:
+            Matplotlib figure object.
+        """
+        return self.sketch.plot_dist(figsize=figsize)
+    
+    def to_dict(self) -> Dict:
+        """
+        Convert sketch to a dictionary for serialization.
+        
+        Returns:
+            Dictionary representation of the sketch.
+        """
+        return self.sketch.to_dict()
+    
+    @classmethod
+    def from_dict(cls, data: Dict) -> 'MomentSketch':
+        """
+        Create a sketch from a dictionary.
+        
+        Args:
+            data: Dictionary representation of a sketch.
+        
+        Returns:
+            New MomentSketch instance.
+        """
+        sketch = cls()
+        sketch.sketch = SimpleMS.from_dict(data)
+        return sketch 

QuantileFlow/momentsketch/example.py

@@ -0,0 +1,160 @@
+"""
+Example script demonstrating the usage of MomentSketch for quantile estimation.
+"""
+
+import numpy as np
+import matplotlib.pyplot as plt
+import time
+from QuantileFlow import MomentSketch
+
+def basic_usage_demo():
+    """Demonstrate basic usage of the MomentSketch"""
+    print("\nBasic MomentSketch Usage Example")
+    print("-" * 50)
+
+    # Generate sample data - skewed distribution
+    np.random.seed(8990)
+    data = np.concatenate([
+        np.random.normal(10, 2, 10000),
+        np.random.exponential(5, 5000)
+    ])
+
+    # Create a MomentSketch with 20 moments
+    sketch = MomentSketch(num_moments=20)
+
+    # Add data to the sketch
+    start_time = time.time()
+    sketch.insert_batch(data)
+    sketch_time = time.time() - start_time
+
+    # Get quantiles
+    start_time = time.time()
+    percentiles = [0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99]
+    quantiles = sketch.quantiles(percentiles)
+    query_time = time.time() - start_time
+
+    # Calculate true quantiles for comparison
+    true_quantiles = np.quantile(data, percentiles)
+
+    # Print results
+    print(f"Data size: {len(data):,d} points")
+    print(f"Time to build sketch: {sketch_time * 1000:.2f} ms")
+    print(f"Time to query quantiles: {query_time * 1000:.2f} ms")
+    print("\nQuantile Comparison:")
+    print(f"{'Percentile':>10} | {'True':>10} | {'Estimated':>10} | {'Error %':>10}")
+    print("-" * 50)
+
+    for p, tq, eq in zip(percentiles, true_quantiles, quantiles):
+        error_pct = 100 * abs(tq - eq) / abs(tq) if tq != 0 else 0
+        print(f"{p * 100:10.1f}% | {tq:10.4f} | {eq:10.4f} | {error_pct:10.2f}%")
+
+    # Get summary statistics
+    stats = sketch.summary_statistics()
+    print("\nSummary Statistics:")
+    for key, value in stats.items():
+        print(f"{key}: {value}")
+
+    return sketch, data
+
+
+def distribution_demo(sketch, data):
+    """Demonstrate plotting the estimated distribution"""
+    print("\nDistribution Visualization")
+    print("-" * 50)
+    
+    # Plot the distribution
+    fig = sketch.plot_distribution(figsize=(10, 6))
+    
+    # Add a histogram of the original data for comparison
+    ax = fig.axes[0]
+    ax.hist(data, bins=50, density=True, alpha=0.5, color='blue', label='Original Data')
+    ax.legend()
+    
+    return fig
+
+
+def merge_demo():
+    """Demonstrate merging multiple sketches"""
+    print("\nMerging Sketches Example")
+    print("-" * 50)
+
+    # Create two different data streams
+    np.random.seed(8990)
+    data1 = np.random.normal(1, 2, 10000)
+    data2 = np.random.normal(2, 3, 15000)
+
+    # Create sketches for each stream
+    sketch1 = MomentSketch(num_moments=20)
+    sketch2 = MomentSketch(num_moments=20)
+
+    # Add data to sketches
+    sketch1.insert_batch(data1)
+    sketch2.insert_batch(data2)
+
+    # Create a combined sketch by merging
+    combined_sketch = MomentSketch(num_moments=20)
+    combined_sketch.merge(sketch1)
+    combined_sketch.merge(sketch2)
+
+    # Create a reference by combining the data
+    combined_data = np.concatenate([data1, data2])
+
+    # Compare median estimates
+    median1 = sketch1.median()
+    median2 = sketch2.median()
+    combined_median = combined_sketch.median()
+    true_combined_median = np.median(combined_data)
+
+    print(f"Data 1 size: {len(data1):,d}, Median: {median1:.4f}")
+    print(f"Data 2 size: {len(data2):,d}, Median: {median2:.4f}")
+    print(f"Combined data size: {len(combined_data):,d}")
+    print(f"True combined median: {true_combined_median:.4f}")
+    print(f"Estimated combined median: {combined_median:.4f}")
+    print(f"Error: {100 * abs(combined_median - true_combined_median) / true_combined_median:.2f}%")
+
+    return combined_sketch, combined_data
+
+
+def serialization_demo():
+    """Demonstrate sketch serialization"""
+    print("\nSerialization Example")
+    print("-" * 50)
+
+    # Create and populate a sketch
+    np.random.seed(8990)
+    data = np.random.lognormal(0, 1, 20000)
+
+    sketch = MomentSketch(num_moments=20)
+    sketch.insert_batch(data)
+
+    # Serialize to dictionary
+    sketch_dict = sketch.to_dict()
+    print(f"Serialized sketch keys: {list(sketch_dict.keys())}")
+
+    # Deserialize
+    restored_sketch = MomentSketch.from_dict(sketch_dict)
+
+    # Compare results
+    original_quantiles = sketch.quantiles([0.25, 0.5, 0.75])
+    restored_quantiles = restored_sketch.quantiles([0.25, 0.5, 0.75])
+
+    print("\nQuantile comparison:")
+    print(f"{'Percentile':>10} | {'Original':>10} | {'Restored':>10}")
+    print("-" * 50)
+
+    for p, oq, rq in zip([0.25, 0.5, 0.75], original_quantiles, restored_quantiles):
+        print(f"{p * 100:10.1f}% | {oq:10.4f} | {rq:10.4f}")
+
+    return sketch_dict
+
+
+if __name__ == "__main__":
+    # Run all demonstrations
+    sketch, data = basic_usage_demo()
+    fig = distribution_demo(sketch, data)
+    
+    combined_sketch, combined_data = merge_demo()
+    serialized_sketch = serialization_demo()
+    
+    plt.tight_layout()
+    plt.show() 

QuantileFlow/momentsketch/optimizer.py

@@ -0,0 +1,205 @@
+"""
+Optimization algorithms for the MomentSketch implementation.
+
+This module provides optimization algorithms used to solve the maximum entropy problem
+in the MomentSketch implementation. It includes:
+
+- BaseOptimizer: An abstract base class defining the common interface for optimizers
+- NewtonOptimizer: An implementation of damped Newton's method for convex optimization
+
+The optimizers handle numerical stability issues that can arise in the maximum entropy
+optimization problem, including matrix conditioning problems and numerical precision issues.
+"""
+
+import numpy as np
+from .utils import Util
+from scipy.linalg import svd, solve
+
+
+class BaseOptimizer:
+    """
+    Base class for optimization algorithms
+    """
+
+    def set_verbose(self, flag):
+        """Set verbose output flag"""
+        pass
+
+    def set_max_iterations(self, max_iterations):
+        """Set maximum iterations"""
+        pass
+
+    def is_converged(self):
+        """Check if optimization has converged"""
+        pass
+
+    def get_iteration_count(self):
+        """Get the number of steps taken"""
+        pass
+
+    def get_function(self):
+        """Get the function being optimized"""
+        pass
+
+    def solve(self, initial_point, gradient_tolerance):
+        """Solve the optimization problem"""
+        pass
+
+
+class NewtonOptimizer(BaseOptimizer):
+    """
+    Minimizes a convex function using damped Newton's
+    """
+
+    def __init__(self, objective_function):
+        """
+        Initialize the optimizer
+
+        Args:
+            objective_function: FunctionWithHessian to optimize
+        """
+        self.objective_function = objective_function
+        self.max_iterations = 200
+        self.iteration_count = 0
+        self.steps = 0
+        self.converged = False
+
+        self.alpha = 0.3
+        self.backtracking_rate = 0.25
+        self.verbose = False
+
+    def set_verbose(self, flag):
+        """Set verbose output flag"""
+        self.verbose = flag
+
+    def set_max_iterations(self, max_iterations):
+        """Set maximum iterations"""
+        self.max_iterations = max_iterations
+
+    def get_iteration_count(self):
+        """Get the number of steps taken"""
+        return self.iteration_count
+
+    def is_converged(self):
+        """Check if optimization has converged"""
+        return self.converged
+
+    def get_backtracking_count(self):
+        """Get the number of damped steps"""
+        return self.steps
+
+    def get_function(self):
+        """Get the function being optimized"""
+        return self.objective_function
+
+    def solve(self, initial_point, grad_tolerance):
+        """
+        Solve the optimization problem
+
+        Args:
+            initial_point: Initial point
+            grad_tolerance: Grad tolerance for convergence
+
+        Returns:
+            Optimal point
+        """
+        dimension = self.objective_function.dim
+
+        current_point = np.array(initial_point).copy()
+
+        required_precision = grad_tolerance / 10
+        self.objective_function.compute_all(current_point, required_precision)
+
+        squared_tolerance = grad_tolerance * grad_tolerance
+        self.converged = False
+
+        for iteration in range(self.max_iterations):
+            function_value = self.objective_function.get_value()
+            gradient = self.objective_function.get_gradient()
+            hessian = self.objective_function.get_hessian()
+            
+            # Check for NaN or Inf in gradient or hessian
+            if not np.all(np.isfinite(gradient)) or not np.all(np.isfinite(hessian)):
+                if self.verbose:
+                    print("Warning: NaN or Inf detected in gradient or Hessian. Using fallback approach.")
+                # Fallback to diagonal Hessian
+                hessian = np.eye(dimension)
+                # Clean gradient if needed
+                gradient = np.nan_to_num(gradient, nan=0.0, posinf=1.0, neginf=-1.0)
+            
+            mean_squared_error = Util.get_mse(gradient)
+
+            if self.verbose:
+                print(f"Iteration: {iteration:3d} GradRMSE: {np.sqrt(mean_squared_error):10.5g} Value: {function_value:10.5g}")
+
+            if mean_squared_error < squared_tolerance:
+                self.converged = True
+                break
+
+            # Try to solve using Cholesky decomposition
+            try:
+                # Add small regularization to ensure positive definiteness if needed
+                if np.any(np.diag(hessian) <= 0):
+                    if self.verbose:
+                        print("Adding regularization to Hessian")
+                    hessian = hessian + 1e-8 * np.eye(dimension)
+                    
+                newton_direction = solve(hessian, gradient, assume_a='pos')
+            except np.linalg.LinAlgError:
+                # Fall back to SVD if Cholesky fails
+                if self.verbose:
+                    print("Cholesky decomposition failed, falling back to SVD")
+                u, singular_values, vh = svd(hessian)
+                pseudoinverse_values = np.array([1 / x if x > 1e-10 else 0 for x in singular_values])
+                newton_direction = (vh.T * pseudoinverse_values) @ (u.T @ gradient)
+
+            newton_direction = -newton_direction
+
+            # Directional derivative
+            directional_derivative = np.sum(newton_direction * gradient)
+
+            step_size = 1.0
+            candidate_point = current_point + step_size * newton_direction
+
+            # Warning: this overwrites gradient and hessian
+            try:
+                self.objective_function.compute_all(candidate_point, required_precision)
+            except Exception as e:
+                if self.verbose:
+                    print(f"Error computing objective at candidate point: {e}")
+                step_size = 0.1
+                candidate_point = current_point + step_size * newton_direction
+                self.objective_function.compute_all(candidate_point, required_precision)
+
+            # Do not look for damped steps if we are near stationary point
+            if directional_derivative * directional_derivative > squared_tolerance:
+                max_backtracking_steps = 10
+                backtracking_steps = 0
+                
+                while True:
+                    new_function_value = self.objective_function.get_value()
+                    improvement = function_value + self.alpha * step_size * directional_derivative - new_function_value
+
+                    if improvement >= -grad_tolerance or step_size < 1e-3 or backtracking_steps >= max_backtracking_steps:
+                        break
+                    else:
+                        step_size *= self.backtracking_rate
+                        backtracking_steps += 1
+
+                    candidate_point = current_point + step_size * newton_direction
+                    try:
+                        self.objective_function.compute_all(candidate_point, required_precision)
+                    except Exception:
+                        # If computation fails, just use the current step size and break
+                        break
+
+            if step_size < 1.0:
+                self.steps += 1
+
+            if self.verbose and step_size < 1.0:
+                print(f"Step Size: {step_size}")
+
+            current_point = candidate_point
+
+        self.iteration_count = iteration + 1
+        return current_point