gratinglab 0.1.0__py3-none-any.whl

gratinglab/metrology/stats/icc.py

@@ -0,0 +1,128 @@
+"""
+Intraclass correlation for row-group measurements
+
+Row-group analysis produces many blaze-angle measurements per image, but they
+re-measure the same physical grooves from different horizontal bands of one scan.
+The rest of the statistics in this package divide by sqrt(N) over that full count,
+which is only valid if the measurements are independent.
+
+The intraclass correlation coefficient (ICC) is how that assumption gets tested:
+the fraction of total variance that sits *between* row groups rather than within
+them.
+
+    ICC = 0    groups are indistinguishable; measurements behave independently
+               and dividing by sqrt(N) is roughly right
+    ICC = 1    all variance is between groups; the measurements within a group
+               carry no new information, and the effective sample size is closer
+               to the number of groups than the number of measurements
+
+This module only measures. The correction that this measurement sizes is applied
+in analyzer.py, which uses effective_sample_size() below to compute a
+row-group-corrected SEM.
+"""
+import numpy as np
+
+# Interpretation thresholds, carried over from an earlier prototype
+# (correlation_source_analysis.py, see git history prior to 2026-08-09).
+ICC_NEGLIGIBLE = 0.1
+ICC_MODERATE = 0.2
+
+
+def interpret_icc(icc):
+    """One-line reading of an ICC value"""
+    if icc < ICC_NEGLIGIBLE:
+        return "negligible - simple statistics defensible"
+    if icc < ICC_MODERATE:
+        return "moderate - hierarchical statistics recommended"
+    return "substantial - hierarchical statistics needed"
+
+
+def compute_icc(values, group_labels):
+    """
+    One-way ICC of `values` grouped by `group_labels`.
+
+    Uses the same variance-ratio form as the original quick_icc_check:
+    between-group variance over total variance, where within-group variance is
+    the mean of the per-group variances.
+
+    Parameters:
+        values: measurements (e.g. blaze angles)
+        group_labels: which group each measurement belongs to, same length
+
+    Returns:
+        dict with icc, between_var, within_var, n_groups, n_measurements,
+        mean_group_size and interpretation. Returns icc = nan when there are
+        fewer than two groups, since between-group variance is undefined - a
+        caller must not read that as "no correlation".
+    """
+    values = np.asarray(values, dtype=float)
+    labels = np.asarray(group_labels)
+
+    if len(values) != len(labels):
+        raise ValueError(f"values and group_labels must be the same length, "
+                         f"got {len(values)} and {len(labels)}")
+
+    unique = np.unique(labels)
+    n_groups = len(unique)
+
+    result = {
+        'n_measurements': len(values),
+        'n_groups': n_groups,
+        'mean_group_size': len(values) / n_groups if n_groups else np.nan,
+    }
+
+    if n_groups < 2 or len(values) < 2:
+        result.update({'icc': np.nan, 'between_var': np.nan,
+                       'within_var': np.nan,
+                       'interpretation': 'undefined - need at least 2 groups'})
+        return result
+
+    group_means = np.array([values[labels == g].mean() for g in unique])
+    # Groups of size 1 contribute no within-group variance information
+    group_vars = np.array([values[labels == g].var(ddof=1)
+                           if (labels == g).sum() > 1 else np.nan
+                           for g in unique])
+
+    between_var = float(group_means.var(ddof=1))
+    within_var = float(np.nanmean(group_vars)) if not np.all(np.isnan(group_vars)) else 0.0
+
+    total = between_var + within_var
+    icc = float(between_var / total) if total > 0 else 0.0
+
+    result.update({
+        'icc': icc,
+        'between_var': between_var,
+        'within_var': within_var,
+        'interpretation': interpret_icc(icc),
+    })
+    return result
+
+
+def effective_sample_size(n_measurements, mean_group_size, icc):
+    """
+    Sample size the measurements are actually worth, given their correlation.
+
+    n_eff = n / (1 + (m - 1) * ICC), the standard design-effect correction, where
+    m is the mean measurements per group. At ICC = 0 this returns n; as ICC rises
+    it falls toward the number of groups.
+
+    This is the number that shows how far the current SEMs are off: they are
+    computed with n, and dividing by sqrt(n) instead of sqrt(n_eff) understates
+    the standard error by sqrt(n / n_eff).
+    """
+    if not np.isfinite(icc):
+        return np.nan
+    design_effect = 1.0 + (mean_group_size - 1.0) * icc
+    return n_measurements / design_effect if design_effect > 0 else np.nan
+
+
+def sem_inflation_factor(mean_group_size, icc):
+    """
+    How much the reported SEM should grow: sqrt(design effect).
+
+    A factor of 2 means the true standard error is twice what is currently
+    reported, and every confidence interval is half the width it should be.
+    """
+    if not np.isfinite(icc):
+        return np.nan
+    return float(np.sqrt(1.0 + (mean_group_size - 1.0) * icc))

gratinglab/metrology/visualization/__init__.py

@@ -0,0 +1,16 @@
+"""
+Visualization and plotting functions
+Diagnostic plots, statistics, and AFM profiles
+"""
+
+from .diagnostics import plot_analyzed_regions_overlay, plot_flattening_diagnostic
+from .statistics import plot_summary_statistics, plot_multi_file_comparison
+from .profiles import plot_sample_profiles_by_temperature
+
+__all__ = [
+    'plot_analyzed_regions_overlay',
+    'plot_flattening_diagnostic',
+    'plot_summary_statistics',
+    'plot_multi_file_comparison',
+    'plot_sample_profiles_by_temperature'
+]

gratinglab/metrology/visualization/diagnostics.py

@@ -0,0 +1,111 @@
+"""
+Diagnostic plotting functions
+Shows analysis details like flattening and analyzed regions
+"""
+import matplotlib.pyplot as plt
+from scipy.signal import find_peaks
+from matplotlib.lines import Line2D
+
+
+def plot_analyzed_regions_overlay(raw_x, flat_y, groove_centers, period_nm, quality_list, facet_trim):
+    """
+    Create an overlay plot showing which portions of each groove are actually analyzed.
+    
+    Parameters:
+        raw_x: x-axis positions (µm)
+        flat_y: flattened height profile (nm)
+        groove_centers: indices of detected groove centers
+        period_nm: groove period (nm)
+        quality_list: list of quality dictionaries with region info
+        facet_trim: fraction of facet trimmed from edges
+    """
+    fig, ax = plt.subplots(figsize=(14, 6))
+    
+    ax.plot(raw_x, flat_y, 'k-', linewidth=1, alpha=0.4, label='Flattened profile', zorder=1)
+    ax.plot(raw_x[groove_centers], flat_y[groove_centers], 'ro', 
+            markersize=6, label='Groove centers', zorder=3, alpha=0.7)
+    
+    for i, qual in enumerate(quality_list):
+        if 'regions' in qual:
+            reg = qual['regions']
+            blaze_x_um = reg['blaze_x_trim'] / 1000
+            blaze_y = reg['blaze_y_trim']
+            ax.plot(blaze_x_um, blaze_y, 'b-', linewidth=3, alpha=0.8, zorder=2)
+            
+            blaze_x_full_um = reg['blaze_x_full'] / 1000
+            blaze_y_full = reg['blaze_y_full']
+            ax.plot(blaze_x_full_um, blaze_y_full, 'b-', linewidth=1.5, alpha=0.3, zorder=1)
+            
+            if len(blaze_x_um) > 0:
+                fit_line = reg['blaze_fit']
+                ax.plot(blaze_x_um, fit_line, 'cyan', linewidth=2, 
+                       linestyle='--', alpha=0.7, zorder=2)
+    
+    custom_lines = [
+        Line2D([0], [0], color='k', linewidth=1, alpha=0.4),
+        Line2D([0], [0], color='b', linewidth=1.5, alpha=0.3),
+        Line2D([0], [0], color='b', linewidth=3, alpha=0.8),
+        Line2D([0], [0], color='cyan', linewidth=2, linestyle='--', alpha=0.7),
+        Line2D([0], [0], marker='o', color='w', markerfacecolor='r', markersize=6)
+    ]
+    ax.legend(custom_lines, 
+             ['Full profile (flattened)', 'Full blaze facets', 
+              'Analyzed regions (trimmed)', 'Linear fits', 'Groove centers'],
+             loc='best', fontsize=10)
+    
+    ax.set_xlabel('Position (µm)', fontsize=12)
+    ax.set_ylabel('Height (nm)', fontsize=12)
+    ax.set_title(f'Analyzed Regions Overlay - Trim fraction: {facet_trim:.0%} (2.5x extra trough trimming)', fontsize=13)
+    ax.grid(True, alpha=0.3)
+    plt.tight_layout()
+
+
+def plot_flattening_diagnostic(raw_x, raw_y, flat_y, background, flatten_method, 
+                               flatten_feature, period_nm_est):
+    """
+    Show before/after flattening comparison
+    
+    Parameters:
+        raw_x: x-axis positions (µm)
+        raw_y: raw height profile (nm)
+        flat_y: flattened height profile (nm)
+        background: fitted background curve (nm)
+        flatten_method: method used for flattening
+        flatten_feature: feature used (for level_grooves method)
+        period_nm_est: estimated period (nm)
+    """
+    fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(12, 6))
+    
+    ax1.plot(raw_x, raw_y, 'b-', linewidth=1, label='Raw profile', alpha=0.7)
+    ax1.plot(raw_x, background, 'r--', linewidth=2, label=f'Background ({flatten_method})')
+    
+    if flatten_method == 'level_grooves':
+        if flatten_feature in ['peaks', 'both']:
+            dx_nm = (raw_x[1] - raw_x[0]) * 1000
+            min_distance = int(0.3 * period_nm_est / dx_nm)
+            peaks, _ = find_peaks(raw_y, distance=min_distance)
+            ax1.plot(raw_x[peaks], raw_y[peaks], 'go', markersize=8, 
+                    label=f'Peaks used ({len(peaks)})', zorder=5)
+        if flatten_feature in ['troughs', 'both']:
+            y_inv = -raw_y
+            dx_nm = (raw_x[1] - raw_x[0]) * 1000
+            min_distance = int(0.3 * period_nm_est / dx_nm)
+            troughs, _ = find_peaks(y_inv, distance=min_distance)
+            ax1.plot(raw_x[troughs], raw_y[troughs], 'mo', markersize=8, 
+                    label=f'Troughs used ({len(troughs)})', zorder=5)
+    
+    ax1.set_xlabel('Displacement (µm)', fontsize=11)
+    ax1.set_ylabel('Height (nm)', fontsize=11)
+    ax1.set_title('Before Flattening: Raw Profile + Background Fit', fontsize=12)
+    ax1.legend(fontsize=10)
+    ax1.grid(True, alpha=0.3)
+    
+    ax2.plot(raw_x, flat_y, 'g-', linewidth=1, label='Flattened profile')
+    ax2.axhline(y=0, color='k', linestyle='--', linewidth=0.5, alpha=0.5)
+    ax2.set_xlabel('Displacement (µm)', fontsize=11)
+    ax2.set_ylabel('Height (nm)', fontsize=11)
+    ax2.set_title('After Flattening: Background Removed', fontsize=12)
+    ax2.legend(fontsize=10)
+    ax2.grid(True, alpha=0.3)
+    
+    plt.tight_layout()

gratinglab/metrology/visualization/profiles.py

@@ -0,0 +1,234 @@
+"""
+AFM profile plotting functions
+Complex multi-panel layouts organized by temperature
+"""
+import matplotlib.pyplot as plt
+import os
+
+
+def plot_sample_profiles_by_temperature(grouped_results, grouped_labels, temp_order):
+    """
+    Plot AFM profiles organized by temperature (one figure per temperature)
+    
+    Parameters:
+        grouped_results: dict mapping temperature key to result
+        grouped_labels: dict mapping temperature key to label
+        temp_order: list of temperature keys in order
+    """
+    
+    for temp_key in temp_order:
+        result = grouped_results[temp_key]
+        label = grouped_labels[temp_key]
+        
+        # Check if this is a combined result with multiple scans
+        if 'individual_scans' in result:
+            # Multiple scans at this temperature
+            _plot_multiple_scans(result, label)
+        else:
+            # Single scan at this temperature
+            _plot_single_scan(result, label)
+
+
+def _plot_multiple_scans(result, label):
+    """
+    Plot multiple AFM scans on separate subplots (for same temperature)
+    
+    Parameters:
+        result: combined result dictionary with 'individual_scans' key
+        label: sample label
+    """
+    scans = result['individual_scans']
+    n_scans = len(scans)
+    
+    # Create figure with one subplot per scan
+    subplot_height = 3
+    total_height = min(subplot_height * n_scans, 11)
+    fig, axes = plt.subplots(n_scans, 1, figsize=(14, total_height))
+    
+    if n_scans == 1:
+        axes = [axes]
+    
+    # Overall figure title
+    fig.suptitle(f'{label} - Combined: {result["mean_angle"]:.2f}° ± {result["std_angle"]:.2f}° '
+                f'(N={result["n_grooves"]} grooves from {n_scans} scans)', 
+                fontsize=12, fontweight='bold', y=0.995)
+    
+    for i, scan in enumerate(scans):
+        ax = axes[i]
+        
+        if 'raw_x' in scan and 'flat_y' in scan:
+            # Plot profile
+            ax.plot(scan['raw_x'], scan['flat_y'], 'k-', linewidth=0.8, alpha=0.5, 
+                   label='Flattened profile')
+            
+            # Plot groove centers
+            if 'groove_centers' in scan:
+                ax.plot(scan['raw_x'][scan['groove_centers']], 
+                       scan['flat_y'][scan['groove_centers']], 
+                       'ro', markersize=4, alpha=0.7, label='Groove centers')
+            
+            # Plot analyzed regions and fits
+            if 'quality' in scan:
+                for j, qual in enumerate(scan['quality']):
+                    if 'regions' in qual:
+                        reg = qual['regions']
+                        blaze_x_um = reg['blaze_x_trim'] / 1000
+                        blaze_y = reg['blaze_y_trim']
+                        fit_line = reg['blaze_fit']
+                        
+                        if j == 0:
+                            ax.plot(blaze_x_um, blaze_y, 'b-', linewidth=2.5, alpha=0.8,
+                                   label='Analyzed regions', zorder=3)
+                            ax.plot(blaze_x_um, fit_line, 'cyan', linewidth=1.5, 
+                                   linestyle='--', alpha=0.7, label='Linear fits', zorder=4)
+                        else:
+                            ax.plot(blaze_x_um, blaze_y, 'b-', linewidth=2.5, alpha=0.8, zorder=3)
+                            ax.plot(blaze_x_um, fit_line, 'cyan', linewidth=1.5, 
+                                   linestyle='--', alpha=0.7, zorder=4)
+            
+            ax.set_ylabel('Height (nm)', fontsize=9)
+            
+            # Scan-specific title
+            scan_title = f'Scan {i+1}: {os.path.basename(scan["filename"])}\n'
+            scan_title += f'Mean = {scan["mean_angle"]:.2f}° ± {scan["std_angle"]:.2f}° '
+            scan_title += f'(N={scan["n_grooves"]} grooves) | Spacing: {scan["period_nm"]:.2f} nm'
+            ax.set_title(scan_title, fontsize=9)
+            ax.legend(loc='best', fontsize=7)
+            ax.grid(True, alpha=0.3)
+            
+            if i == n_scans - 1:
+                ax.set_xlabel('Position (µm)', fontsize=9)
+        else:
+            ax.text(0.5, 0.5, f'Scan {i+1}\nProfile data not available', 
+                   ha='center', va='center', transform=ax.transAxes)
+    
+    plt.tight_layout(rect=[0, 0, 1, 0.99])
+
+
+def _plot_single_scan(result, label):
+    """
+    Plot a single AFM scan
+    
+    Parameters:
+        result: result dictionary
+        label: sample label
+    """
+    fig, ax = plt.subplots(1, 1, figsize=(14, 4))
+    
+    if 'raw_x' in result and 'flat_y' in result:
+        # Plot profile
+        ax.plot(result['raw_x'], result['flat_y'], 'k-', linewidth=0.8, alpha=0.5, 
+               label='Flattened profile')
+        
+        # Plot groove centers
+        if 'groove_centers' in result:
+            ax.plot(result['raw_x'][result['groove_centers']], 
+                   result['flat_y'][result['groove_centers']], 
+                   'ro', markersize=4, alpha=0.7, label='Groove centers')
+        
+        # Plot analyzed regions and fits
+        if 'quality' in result:
+            for j, qual in enumerate(result['quality']):
+                if 'regions' in qual:
+                    reg = qual['regions']
+                    blaze_x_um = reg['blaze_x_trim'] / 1000
+                    blaze_y = reg['blaze_y_trim']
+                    fit_line = reg['blaze_fit']
+                    
+                    if j == 0:
+                        ax.plot(blaze_x_um, blaze_y, 'b-', linewidth=2.5, alpha=0.8,
+                               label='Analyzed regions', zorder=3)
+                        ax.plot(blaze_x_um, fit_line, 'cyan', linewidth=1.5, 
+                               linestyle='--', alpha=0.7, label='Linear fits', zorder=4)
+                    else:
+                        ax.plot(blaze_x_um, blaze_y, 'b-', linewidth=2.5, alpha=0.8, zorder=3)
+                        ax.plot(blaze_x_um, fit_line, 'cyan', linewidth=1.5, 
+                               linestyle='--', alpha=0.7, zorder=4)
+        
+        ax.set_ylabel('Height (nm)', fontsize=10)
+        ax.set_xlabel('Position (µm)', fontsize=10)
+        
+        # Title
+        title = f'{label}: Mean = {result["mean_angle"]:.2f}° ± {result["std_angle"]:.2f}° '
+        title += f'(N={result["n_grooves"]} grooves) | Spacing: {result["period_nm"]:.2f} nm'
+        ax.set_title(title, fontsize=11, fontweight='bold')
+        ax.legend(loc='best', fontsize=9)
+        ax.grid(True, alpha=0.3)
+    else:
+        ax.text(0.5, 0.5, f'{label}\nProfile data not available', 
+               ha='center', va='center', transform=ax.transAxes)
+    
+    plt.tight_layout()
+
+
+# Legacy function for backwards compatibility
+def plot_sample_profiles_comparison(results, labels):
+    """
+    DEPRECATED: Use plot_sample_profiles_by_temperature instead
+    
+    Plot AFM profiles for all samples in a single multi-panel figure
+    This is kept for backwards compatibility but not recommended for many samples
+    """
+    import warnings
+    warnings.warn("plot_sample_profiles_comparison is deprecated. "
+                 "Use plot_sample_profiles_by_temperature instead.", 
+                 DeprecationWarning)
+    
+    n_samples = len(results)
+    
+    if n_samples <= 3:
+        subplot_height = 3
+    elif n_samples == 4:
+        subplot_height = 2.5
+    else:
+        subplot_height = 2.0
+    
+    total_height = min(subplot_height * n_samples, 11)
+    fig, axes = plt.subplots(n_samples, 1, figsize=(14, total_height))
+    
+    if n_samples == 1:
+        axes = [axes]
+    
+    for i, (r, label) in enumerate(zip(results, labels)):
+        if 'raw_x' in r and 'flat_y' in r:
+            axes[i].plot(r['raw_x'], r['flat_y'], 'k-', linewidth=0.8, alpha=0.5, 
+                        label='Flattened profile')
+            
+            if 'groove_centers' in r:
+                axes[i].plot(r['raw_x'][r['groove_centers']], r['flat_y'][r['groove_centers']], 
+                           'ro', markersize=4, alpha=0.7, label='Groove centers')
+            
+            if 'quality' in r:
+                for j, qual in enumerate(r['quality']):
+                    if 'regions' in qual:
+                        reg = qual['regions']
+                        blaze_x_um = reg['blaze_x_trim'] / 1000
+                        blaze_y = reg['blaze_y_trim']
+                        fit_line = reg['blaze_fit']
+                        
+                        if j == 0:
+                            axes[i].plot(blaze_x_um, blaze_y, 'b-', linewidth=2.5, alpha=0.8,
+                                       label='Analyzed regions', zorder=3)
+                            axes[i].plot(blaze_x_um, fit_line, 'cyan', linewidth=1.5, 
+                                       linestyle='--', alpha=0.7, label='Linear fits', zorder=4)
+                        else:
+                            axes[i].plot(blaze_x_um, blaze_y, 'b-', linewidth=2.5, alpha=0.8, zorder=3)
+                            axes[i].plot(blaze_x_um, fit_line, 'cyan', linewidth=1.5, 
+                                       linestyle='--', alpha=0.7, zorder=4)
+            
+            axes[i].set_ylabel('Height (nm)', fontsize=9)
+            
+            # Include spacing in the title
+            spacing_str = f"Spacing: {r['period_nm']:.2f} nm"
+            axes[i].set_title(f'{label}: Mean = {r["mean_angle"]:.2f}° ± {r["std_angle"]:.2f}° (N={r["n_grooves"]}) | {spacing_str}', 
+                            fontsize=10)
+            axes[i].legend(loc='best', fontsize=7)
+            axes[i].grid(True, alpha=0.3)
+            
+            if i == n_samples - 1:
+                axes[i].set_xlabel('Position (µm)', fontsize=9)
+        else:
+            axes[i].text(0.5, 0.5, f'{label}\nProfile data not available', 
+                       ha='center', va='center', transform=axes[i].transAxes)
+    
+    plt.tight_layout()