pivtools 0.1.3__cp311-cp311-win_amd64.whl

pivtools_cli/piv/piv_backend/outlier_detection.py

@@ -0,0 +1,306 @@
+"""
+Outlier detection methods for PIV velocity fields.
+
+This module provides various methods for detecting outliers in PIV data:
+- Peak magnitude thresholding
+- Median-based 2D outlier detection
+- Sigma-based outlier detection
+- Divergence/vorticity-based outlier detection
+"""
+
+import numpy as np
+import bottleneck as bn
+from scipy.signal import convolve2d
+from scipy import ndimage as ndi
+
+
+def peak_magnitude_detection(
+    peak_mag: np.ndarray,
+    threshold: float = 0.5,
+) -> np.ndarray:
+    """
+    Detect outliers based on peak magnitude threshold.
+    
+    Vectors with peak magnitudes below the threshold are marked as outliers.
+
+    Parameters
+    ----------
+    peak_mag : np.ndarray
+        Peak magnitude array from PIV correlation.
+    threshold : float, optional
+        Minimum acceptable peak magnitude, defaults to 0.5.
+
+    Returns
+    -------
+    np.ndarray
+        Boolean array indicating outliers (True = outlier).
+    """
+    b_filter = (peak_mag < threshold) | np.isnan(peak_mag)
+    return b_filter
+
+
+def median_outlier_detection(
+    ux: np.ndarray,
+    uy: np.ndarray,
+    epsilon: float = 0.2,
+    threshold: float = 2.0,
+) -> np.ndarray:
+    """
+    Fast median-based outlier detection for 2D PIV velocity fields,
+    using bottleneck for nan-aware reductions.
+    
+    This method compares each vector to the median of its 8 neighbors.
+    Outliers are identified based on the normalized residual exceeding a threshold.
+
+    Parameters
+    ----------
+    ux : np.ndarray
+        Horizontal velocity component (2D).
+    uy : np.ndarray
+        Vertical velocity component (2D).
+    epsilon : float, optional
+        Regularization term for division stability, defaults to 0.2.
+    threshold : float, optional
+        Threshold for outlier detection, defaults to 2.0.
+
+    Returns
+    -------
+    np.ndarray
+        Boolean mask of outliers (True = outlier).
+    """
+    if ux.shape != uy.shape:
+        raise ValueError("ux and uy must have identical shapes")
+
+    n_wx, n_wy = ux.shape
+    ui = np.stack((ux, uy), axis=-1).astype(np.float32, copy=False)
+
+    r_0p = np.zeros((n_wx, n_wy, 2), dtype=np.float32)
+    n_neighbours = np.zeros((n_wx, n_wy, 2), dtype=np.float32)
+
+    ones3 = np.ones((3, 3), dtype=np.float32)
+
+    for c in range(2):
+        U = ui[..., c]
+        U_pad = np.pad(U, 1, mode="constant", constant_values=np.nan)
+
+        # Collect 8-neighbor pixels
+        U_nn = np.stack([
+            U_pad[:-2, :-2],  # top-left
+            U_pad[:-2, 1:-1],  # top
+            U_pad[:-2, 2:],    # top-right
+            U_pad[1:-1, :-2],  # left
+            U_pad[1:-1, 2:],   # right
+            U_pad[2:, 2:],     # bottom-right
+            U_pad[2:, 1:-1],   # bottom
+            U_pad[2:, :-2],    # bottom-left
+        ], axis=-1)  # (H, W, 8)
+
+        # --- bottleneck median operations (C-accelerated) ---
+        U_med = bn.nanmedian(U_nn, axis=-1)  # neighbor median
+        r_0 = np.abs(U_med - U)
+        r_i = np.abs(U_nn - U_med[..., None])
+        r_m = bn.nanmedian(r_i, axis=-1)     # median absolute deviation
+
+        r_0p[..., c] = r_0 / (r_m + epsilon)
+
+        # Valid neighbor count via convolution (3×3 kernel)
+        valid = (~np.isnan(U)).astype(np.float32)
+        n_neigh = convolve2d(valid, ones3, mode="same", boundary="fill", fillvalue=0.0)
+        n_neighbours[..., c] = n_neigh
+
+    r_0_combined = bn.nanmax(r_0p, axis=2)
+
+    # Boolean mask: true = outlier
+    b_filter = (
+        (r_0_combined > threshold)
+        | np.isnan(r_0p).any(axis=2)
+        | (n_neighbours < 6).any(axis=2)
+    )
+    return b_filter
+
+
+def sigma_outlier_detection(
+    ux: np.ndarray,
+    uy: np.ndarray,
+    sigma_threshold: float = 2.0,
+) -> np.ndarray:
+    """
+    Detect outliers based on local standard deviation (sigma-based).
+    
+    Vectors that deviate from the 8-neighbor mean by more than sigma_threshold
+    times the local standard deviation are marked as outliers.
+
+    Parameters
+    ----------
+    ux : np.ndarray
+        Horizontal velocity component (2D).
+    uy : np.ndarray
+        Vertical velocity component (2D).
+    sigma_threshold : float, optional
+        Number of standard deviations for outlier threshold, defaults to 2.0.
+
+    Returns
+    -------
+    np.ndarray
+        Boolean mask of outliers (True = outlier).
+    """
+    v = np.sqrt(ux**2 + uy**2).astype(np.float32, copy=False)
+    finite = np.isfinite(v).astype(np.float32)
+    v0 = np.where(np.isfinite(v), v, 0.0)
+
+    # 3x3 window means for value, value^2, and for the finite-mask
+    m9_v = ndi.uniform_filter(v0, size=3, mode='constant', cval=0.0)
+    m9_v2 = ndi.uniform_filter(v0 * v0, size=3, mode='constant', cval=0.0)
+    m9_cnt = ndi.uniform_filter(finite, size=3, mode='constant', cval=0.0)
+
+    # uniform_filter returns MEAN; convert to SUM by * 9
+    sum9 = m9_v * 9.0
+    sumsq9 = m9_v2 * 9.0
+    cnt9 = m9_cnt * 9.0
+
+    # Exclude the center pixel to get 8-neighbour stats
+    center_val = np.where(np.isfinite(v), v, 0.0)
+    center_cnt = finite
+    sum8 = sum9 - center_val
+    sumsq8 = sumsq9 - center_val * center_val
+    cnt8 = cnt9 - center_cnt
+
+    # Mean and std over the 8 neighbours (ignore divisions by <=0 count)
+    with np.errstate(invalid='ignore', divide='ignore'):
+        mean8 = sum8 / cnt8
+        var8 = (sumsq8 / cnt8) - mean8 * mean8
+    var8 = np.maximum(var8, 0.0)
+    std8 = np.sqrt(var8)
+
+    # Outlier mask
+    b_filter = (
+        (np.abs(v - mean8) > sigma_threshold * std8)
+        | ~np.isfinite(v)
+        | ~np.isfinite(mean8)
+        | (std8 == 0)
+        | (cnt8 < 1)
+    )
+    return b_filter
+
+
+def div_vort_outliers(
+    ux: np.ndarray,
+    uy: np.ndarray,
+    div_thresh: float = None,
+    vort_thresh: float = None,
+) -> np.ndarray:
+    """
+    Detect outliers based on divergence and vorticity thresholds.
+    
+    Computes divergence and vorticity using central differences, then
+    identifies outliers as vectors with extreme values. Thresholds are
+    automatically computed from the field statistics if not provided.
+
+    Parameters
+    ----------
+    ux : np.ndarray
+        Horizontal velocity component (2D).
+    uy : np.ndarray
+        Vertical velocity component (2D).
+    div_thresh : float, optional
+        Divergence threshold. If None, computed as 6*MAD.
+    vort_thresh : float, optional
+        Vorticity threshold. If None, computed as 6*MAD.
+
+    Returns
+    -------
+    np.ndarray
+        Boolean mask of outliers (True = outlier).
+    """
+    # Central differences
+    dudx = 0.5 * (np.roll(ux, -1, 1) - np.roll(ux, 1, 1))
+    dudy = 0.5 * (np.roll(ux, -1, 0) - np.roll(ux, 1, 0))
+    dvdx = 0.5 * (np.roll(uy, -1, 1) - np.roll(uy, 1, 1))
+    dvdy = 0.5 * (np.roll(uy, -1, 0) - np.roll(uy, 1, 0))
+    
+    div = dudx + dvdy
+    vort = dvdx - dudy
+    
+    # Robust thresholds from field using MAD (Median Absolute Deviation)
+    if div_thresh is None:
+        div_finite = div[np.isfinite(div)]
+        if div_finite.size > 0:
+            div_median = np.median(div_finite)
+            mad = 1.4826 * np.median(np.abs(div_finite - div_median))
+            div_thresh = 6 * mad
+        else:
+            div_thresh = np.inf
+    
+    if vort_thresh is None:
+        vort_finite = vort[np.isfinite(vort)]
+        if vort_finite.size > 0:
+            vort_median = np.median(vort_finite)
+            mad = 1.4826 * np.median(np.abs(vort_finite - vort_median))
+            vort_thresh = 6 * mad
+        else:
+            vort_thresh = np.inf
+    
+    return (np.abs(div) > div_thresh) | (np.abs(vort) > vort_thresh)
+
+
+def apply_outlier_detection(
+    ux: np.ndarray,
+    uy: np.ndarray,
+    methods: list,
+    peak_mag: np.ndarray = None,
+) -> np.ndarray:
+    """
+    Apply multiple outlier detection methods and combine results.
+    
+    This function applies a stack of outlier detection methods configured
+    in the YAML file and combines their results with logical OR.
+
+    Parameters
+    ----------
+    ux : np.ndarray
+        Horizontal velocity component (2D).
+    uy : np.ndarray
+        Vertical velocity component (2D).
+    methods : list
+        List of method dictionaries from config, each with 'type' and parameters.
+    peak_mag : np.ndarray, optional
+        Peak magnitude array (required for 'peak_mag' method).
+
+    Returns
+    -------
+    np.ndarray
+        Combined boolean mask of outliers (True = outlier).
+    """
+    combined_mask = np.zeros(ux.shape, dtype=bool)
+    
+    for method_cfg in methods:
+        method_type = method_cfg.get('type', '').lower()
+        
+        if method_type == 'peak_mag':
+            if peak_mag is None:
+                raise ValueError("peak_mag array required for 'peak_mag' outlier detection")
+            threshold = method_cfg.get('threshold', 0.5)
+            mask = peak_magnitude_detection(peak_mag, threshold=threshold)
+            combined_mask |= mask
+            
+        elif method_type == 'median_2d':
+            epsilon = method_cfg.get('epsilon', 0.2)
+            threshold = method_cfg.get('threshold', 2.0)
+            mask = median_outlier_detection(ux, uy, epsilon=epsilon, threshold=threshold)
+            combined_mask |= mask
+            
+        elif method_type == 'sigma':
+            sigma_threshold = method_cfg.get('sigma_threshold', 2.0)
+            mask = sigma_outlier_detection(ux, uy, sigma_threshold=sigma_threshold)
+            combined_mask |= mask
+            
+        elif method_type == 'div_vort':
+            div_thresh = method_cfg.get('div_thresh', None)
+            vort_thresh = method_cfg.get('vort_thresh', None)
+            mask = div_vort_outliers(ux, uy, div_thresh=div_thresh, vort_thresh=vort_thresh)
+            combined_mask |= mask
+            
+        else:
+            raise ValueError(f"Unknown outlier detection method: {method_type}")
+    
+    return combined_mask

pivtools_cli/piv/piv_backend/profile_cpu_instantaneous.py

@@ -0,0 +1,230 @@
+import cProfile
+import io
+import pstats
+import time
+from pathlib import Path
+import sys
+import numpy as np
+
+# Add src to path for unified imports
+sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent / "src"))
+
+from config import Config
+from image_handling.load_images import load_images, load_mask_for_camera
+from pivtools_cli.piv.piv_backend.cpu_instantaneous import InstantaneousCorrelatorCPU
+
+
+def profile_cpu_instantaneous():
+    """
+    Profile the CPU instantaneous correlator with comprehensive timing breakdown.
+    Runs 10 images: first one warms up caches, next 9 are profiled.
+    Provides timing info per pass, memory usage, and detailed profiling.
+    """
+    overall_start = time.perf_counter()
+
+    config = Config()
+
+    # Use first camera and source path
+    camera_num = config.camera_numbers[0]
+    source_path = config.source_paths[0]
+
+    print("=" * 60)
+    print("COMPREHENSIVE CPU INSTANTANEOUS CORRELATOR PROFILING")
+    print("=" * 60)
+
+    # PHASE 1: Data Loading
+    print("\n[PHASE 1] Loading images...")
+    phase_start = time.perf_counter()
+    all_images = load_images(camera_num, config, source=source_path)
+
+    # Handle color images: convert to grayscale if needed
+    if len(all_images.shape) == 5:
+        print("Detected color images, converting to grayscale...")
+        import dask.array as da
+        all_images = da.mean(all_images, axis=-1, dtype=np.float32)
+
+    # all_images is now (num_images, 2, H, W)
+    num_images = min(10, all_images.shape[0])
+    images = all_images[:num_images]
+
+    print(f"Slicing {num_images} image pairs from dask array")
+
+    # CRITICAL FIX: Compute dask array to numpy array BEFORE profiling
+    # This matches the real workflow where image_block.compute() is called in _piv_single_pass
+    print("Computing dask array to numpy (this may take a moment)...")
+    start_compute = time.perf_counter()
+    images = images.compute()  # Convert from dask to numpy array
+    compute_time = time.perf_counter() - start_compute
+    print(".2f")
+    print(f"Images shape: {images.shape}, dtype: {images.dtype}")
+
+    data_loading_time = time.perf_counter() - phase_start
+    print(".2f")
+
+    # PHASE 2: Mask Loading
+    print("\n[PHASE 2] Loading masks...")
+    phase_start = time.perf_counter()
+    vector_masks = None
+    if config.masking_enabled:
+        mask = load_mask_for_camera(camera_num, config, source_path_idx=0)
+        if mask is not None:
+            from image_handling.load_images import compute_vector_mask
+            vector_masks = compute_vector_mask(mask, config)
+            print("Loaded mask and computed vector masks")
+        else:
+            print("Masking enabled but no mask found")
+    else:
+        print("Masking disabled")
+
+    mask_loading_time = time.perf_counter() - phase_start
+    print(".2f")
+
+    # PHASE 3: Correlator Initialization
+    print("\n[PHASE 3] Initializing correlator...")
+    phase_start = time.perf_counter()
+    correlator = InstantaneousCorrelatorCPU(config)
+    correlator_init_time = time.perf_counter() - phase_start
+    print(".2f")
+
+    # PHASE 4: Warmup
+    print("\n[PHASE 4] Warming up caches with first image...")
+    phase_start = time.perf_counter()
+    warmup_result = correlator.correlate_batch(images[:1], config, vector_masks)
+    warmup_time = time.perf_counter() - phase_start
+    print(".2f")
+
+    # PHASE 5: Profiling
+    print(f"\n[PHASE 5] Profiling correlation on {num_images-1} images...")
+    images_to_profile = images[1:num_images]  # Take remaining images
+    num_to_profile = len(images_to_profile)
+
+    if num_to_profile == 0:
+        print("Not enough images to profile (need at least 2 total)")
+        return
+
+    print(f"Profiling {num_to_profile} images...")
+
+    # Use cProfile for detailed line-by-line profiling
+    pr = cProfile.Profile()
+
+    def run_correlation_batch():
+        results = []
+        all_pass_times = []
+        global_img_idx = 1  # Start from 1 since 0 is warmup
+        for i, img_pair in enumerate(images_to_profile):
+            # Removed print statement for cleaner output
+            result = correlator.correlate_batch(img_pair[np.newaxis], config, vector_masks)
+            # Extend with global image index
+            for pass_data in correlator.pass_times:
+                all_pass_times.append((global_img_idx, pass_data[1], pass_data[2]))
+            global_img_idx += 1
+            results.append(result)
+        return results, all_pass_times
+
+    # Run with profiling
+    pr.enable()
+    start_profile = time.perf_counter()
+    results, all_pass_times = run_correlation_batch()
+    end_profile = time.perf_counter()
+    pr.disable()
+
+    total_profile_time = end_profile - start_profile
+    avg_time_per_image = total_profile_time / num_to_profile
+
+    print(".2f")
+    print(".4f")
+
+    # PHASE 6: Analysis and Output
+    print("\n[PHASE 6] Analysis and Output...")
+
+    # Comprehensive timing summary
+    print("\n" + "=" * 60)
+    print("COMPREHENSIVE TIMING SUMMARY")
+    print("=" * 60)
+    print(f"{'Phase':<25} {'Time (s)':<10} {'% of Total':<10}")
+    print("-" * 60)
+
+    total_time = data_loading_time + mask_loading_time + correlator_init_time + warmup_time + total_profile_time
+    phases = [
+        ("Data Loading", data_loading_time),
+        ("Mask Loading", mask_loading_time),
+        ("Correlator Init", correlator_init_time),
+        ("Warmup", warmup_time),
+        ("Profiling", total_profile_time),
+    ]
+
+    for phase_name, phase_time in phases:
+        pct = (phase_time / total_time * 100) if total_time > 0 else 0
+        print(f"{phase_name:<25} {phase_time:<10.2f} {pct:<10.1f}")
+
+    print("-" * 60)
+    print(f"{'TOTAL TIME':<25} {total_time:<10.2f} {'100.0':<10}")
+    print(f"{'Avg time per image':<25} {avg_time_per_image:<10.4f}")
+
+    # Write line profiling results to file
+    print("\nWriting detailed profiling results...")
+    profile_txt_path = Path(__file__).parent / "profile_results.txt"
+    with open(profile_txt_path, 'w') as f:
+        f.write("Detailed profiling statistics (top 20 functions by cumulative time):\n")
+        ps = pstats.Stats(pr, stream=f).sort_stats('cumulative')
+        ps.print_stats(20)
+        f.write("\nTop 20 functions by total time:\n")
+        ps2 = pstats.Stats(pr, stream=f).sort_stats('time')
+        ps2.print_stats(20)
+
+    print(f"Line profiling results saved to: {profile_txt_path}")
+    
+    # Collect and analyze per-pass times (excluding warmup)
+    pass_times_data = all_pass_times
+    print(f"Collected {len(pass_times_data)} pass timing records")
+    if pass_times_data:
+        print(f"Sample record: {pass_times_data[0] if pass_times_data else 'None'}")
+        # Filter out warmup (image 0) - but since we start from 1, all are valid
+        profiled_times = [(pass_idx, time_val) for img_idx, pass_idx, time_val in pass_times_data]
+        print(f"After filtering warmup: {len(profiled_times)} records")
+        if profiled_times:
+            print(f"Sample filtered: {profiled_times[0]}")
+        # Filter out warmup (image 0)
+        profiled_times = [(pass_idx, time_val) for img_idx, pass_idx, time_val in pass_times_data if img_idx > 0]
+        
+        # Group by pass and compute averages
+        from collections import defaultdict
+        pass_stats = defaultdict(list)
+        for pass_idx, time_val in profiled_times:
+            pass_stats[pass_idx].append(time_val)
+        
+        # Compute averages and sort by time descending
+        avg_times = []
+        for pass_idx, times in pass_stats.items():
+            avg_time = sum(times) / len(times)
+            avg_times.append((pass_idx, avg_time, len(times)))
+        
+        avg_times.sort(key=lambda x: x[1], reverse=True)  # Sort by average time descending
+        
+        # Write per-pass averages to file
+        pass_avg_txt_path = Path(__file__).parent / "pass_averages.txt"
+        with open(pass_avg_txt_path, 'w') as f:
+            f.write("Per-Pass Average Times (excluding warmup, ranked by longest time)\n")
+            f.write("=" * 60 + "\n")
+            f.write(f"{'Pass':<5} {'Avg Time (s)':<12} {'Samples':<8} {'% of Total':<10}\n")
+            f.write("-" * 60 + "\n")
+            
+            total_avg_time = sum(avg for _, avg, _ in avg_times)
+            for pass_idx, avg_time, count in avg_times:
+                pct = (avg_time / total_avg_time * 100) if total_avg_time > 0 else 0
+                f.write(f"{pass_idx:<5} {avg_time:<12.4f} {count:<8} {pct:<10.1f}%\n")
+        
+        print(f"Per-pass averages saved to: {pass_avg_txt_path}")
+    
+    try:
+        import line_profiler
+        print("\nFor detailed line-by-line profiling, run:")
+        print("kernprof -l -v profile_cpu_instantaneous.py")
+    except ImportError:
+        print("\nFor detailed line-by-line profiling, install line_profiler:")
+        print("pip install line_profiler")
+        print("Then run: kernprof -l -v profile_cpu_instantaneous.py")
+
+
+if __name__ == "__main__":
+    profile_cpu_instantaneous()

pivtools_cli/piv/piv_result.py

@@ -0,0 +1,40 @@
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+import numpy as np
+
+
+@dataclass
+class PIVPassResult:
+    n_windows: Optional[np.ndarray] = None
+    ux_mat: Optional[np.ndarray] = None
+    uy_mat: Optional[np.ndarray] = None
+    nan_mask: Optional[np.ndarray] = None
+    peak_mag: Optional[np.ndarray] = None
+    peak_choice: Optional[np.ndarray] = None
+    predictor_field: Optional[np.ndarray] = None
+    primary_peak_mag: Optional[np.ndarray] = None
+    b_mask: Optional[np.ndarray] = None
+    window_size: Optional[tuple[int, int]] = None
+    win_ctrs_x: Optional[np.ndarray] = None
+    win_ctrs_y: Optional[np.ndarray] = None
+
+
+@dataclass
+class PIVResult:
+    passes: List[PIVPassResult] = field(default_factory=list)
+
+    def add_pass(self, pass_result: PIVPassResult):
+        self.passes.append(pass_result)
+
+    def summary(self) -> str:
+        s = f"PIVResult with {len(self.passes)} passes:\n"
+        for i, p in enumerate(self.passes):
+            s += (
+                f"  Pass {i + 1}: ux.shape="
+                f"{None if p.ux_mat is None else p.ux_mat.shape}, "
+            )
+            s += (
+                f"uy.shape={None if p.uy_mat is None else p.uy_mat.shape}\n"
+            )
+        return s