microstructure-tpw 0.3.0__py3-none-any.whl

odas_tpw/scor160/compare.py

@@ -0,0 +1,484 @@
+"""Comparison utilities for validating computed results against reference."""
+
+from __future__ import annotations
+
+import numpy as np
+
+from odas_tpw.scor160.io import L2Data, L3Data, L4Data
+
+
+def _log_spectral_metrics(comp: np.ndarray, ref: np.ndarray) -> dict:
+    """Compute log10 comparison metrics between two positive arrays."""
+    valid = np.isfinite(comp) & np.isfinite(ref) & (comp > 0) & (ref > 0)
+    if not valid.any():
+        return {"n_valid": 0}
+
+    log_diff = np.log10(comp[valid]) - np.log10(ref[valid])
+    abs_log_diff = np.abs(log_diff)
+
+    return {
+        "n_valid": int(valid.sum()),
+        "mean_log_diff": float(np.mean(log_diff)),
+        "rms_log_diff": float(np.sqrt(np.mean(log_diff**2))),
+        "median_ratio": float(10 ** np.median(log_diff)),
+        "q0_abs_log": float(np.min(abs_log_diff)),
+        "q50_abs_log": float(np.median(abs_log_diff)),
+        "q100_abs_log": float(np.max(abs_log_diff)),
+    }
+
+
+def _format_report_header(level: str, filename: str) -> list[str]:
+    """Build report header lines for comparison output."""
+    header = f"{level} comparison: {filename}" if filename else f"{level} comparison"
+    return [header, "=" * len(header)]
+
+
+def compare_l2(computed: L2Data, reference: L2Data) -> dict:
+    """Compare computed L2 against reference L2.
+
+    Returns a dict of comparison metrics.
+    """
+    results: dict = {}
+
+    # Section agreement
+    ref_sec = reference.section_number
+    comp_sec = computed.section_number
+    ref_in = ref_sec > 0
+    comp_in = comp_sec > 0
+    both_in = ref_in & comp_in
+
+    # Guard the .max() reductions: np.max on a zero-size array raises
+    # ValueError; an empty/degenerate L2 should report 0 sections, not crash.
+    results["section"] = {
+        "ref_n_sections": int(ref_sec.max()) if ref_sec.size else 0,
+        "comp_n_sections": int(comp_sec.max()) if comp_sec.size else 0,
+        "ref_n_selected": int(ref_in.sum()),
+        "comp_n_selected": int(comp_in.sum()),
+        "overlap_frac": float(int(both_in.sum()) / max(int(ref_in.sum()), 1)),
+    }
+
+    # Speed comparison (within reference sections)
+    if both_in.any():
+        w_ref = reference.pspd_rel[both_in]
+        w_comp = computed.pspd_rel[both_in]
+        finite = np.isfinite(w_ref) & np.isfinite(w_comp)
+        if finite.any():
+            diff = w_comp[finite] - w_ref[finite]
+            results["speed"] = {
+                "mean_diff": float(np.mean(diff)),
+                "rms_diff": float(np.sqrt(np.mean(diff**2))),
+                "max_abs_diff": float(np.max(np.abs(diff))),
+                "ref_mean": float(np.mean(w_ref[finite])),
+                "relative_rms": float(np.sqrt(np.mean(diff**2)) / np.mean(np.abs(w_ref[finite]))),
+            }
+
+    # Shear comparison (within reference sections)
+    n_sh = min(computed.shear.shape[0], reference.shear.shape[0])
+    shear_stats = []
+    for i in range(n_sh):
+        ref_sh = reference.shear[i, both_in]
+        comp_sh = computed.shear[i, both_in]
+        finite = np.isfinite(ref_sh) & np.isfinite(comp_sh)
+        if finite.any():
+            diff = comp_sh[finite] - ref_sh[finite]
+            ref_rms = np.sqrt(np.mean(ref_sh[finite] ** 2))
+            shear_stats.append(
+                {
+                    "probe": i + 1,
+                    "rms_diff": float(np.sqrt(np.mean(diff**2))),
+                    "ref_rms": float(ref_rms),
+                    "relative_rms": float(
+                        np.sqrt(np.mean(diff**2)) / ref_rms if ref_rms > 0 else np.inf
+                    ),
+                    "corr": float(np.corrcoef(ref_sh[finite], comp_sh[finite])[0, 1]),
+                }
+            )
+    results["shear"] = shear_stats
+
+    # Vibration comparison
+    n_vib = min(computed.vib.shape[0], reference.vib.shape[0])
+    vib_stats = []
+    for i in range(n_vib):
+        ref_v = reference.vib[i, both_in]
+        comp_v = computed.vib[i, both_in]
+        finite = np.isfinite(ref_v) & np.isfinite(comp_v)
+        if finite.any():
+            diff = comp_v[finite] - ref_v[finite]
+            ref_rms = np.sqrt(np.mean(ref_v[finite] ** 2))
+            vib_stats.append(
+                {
+                    "sensor": i + 1,
+                    "rms_diff": float(np.sqrt(np.mean(diff**2))),
+                    "ref_rms": float(ref_rms),
+                    "relative_rms": float(
+                        np.sqrt(np.mean(diff**2)) / ref_rms if ref_rms > 0 else np.inf
+                    ),
+                    "corr": float(np.corrcoef(ref_v[finite], comp_v[finite])[0, 1]),
+                }
+            )
+    results["vibration"] = vib_stats
+
+    return results
+
+
+def format_l2_report(metrics: dict, filename: str = "") -> str:
+    """Format L2 comparison metrics as a human-readable report."""
+    lines = _format_report_header("L2", filename)
+
+    # Section selection
+    sec = metrics.get("section", {})
+    lines.append("")
+    lines.append("Section selection:")
+    lines.append(f"  Reference sections:   {sec.get('ref_n_sections', '?')}")
+    lines.append(f"  Computed sections:    {sec.get('comp_n_sections', '?')}")
+    lines.append(f"  Reference selected:   {sec.get('ref_n_selected', '?')} samples")
+    lines.append(f"  Computed selected:    {sec.get('comp_n_selected', '?')} samples")
+    lines.append(f"  Overlap fraction:     {sec.get('overlap_frac', 0):.4f}")
+
+    # Speed
+    spd = metrics.get("speed")
+    if spd:
+        lines.append("")
+        lines.append("Profiling speed (within overlapping sections):")
+        lines.append(f"  Reference mean:       {spd['ref_mean']:.4f} m/s")
+        lines.append(f"  Mean difference:      {spd['mean_diff']:.6f} m/s")
+        lines.append(f"  RMS difference:       {spd['rms_diff']:.6f} m/s")
+        lines.append(f"  Max |difference|:     {spd['max_abs_diff']:.6f} m/s")
+        lines.append(f"  Relative RMS:         {spd['relative_rms']:.4f}")
+
+    # Shear
+    for sh in metrics.get("shear", []):
+        lines.append("")
+        lines.append(f"Shear probe {sh['probe']}:")
+        lines.append(f"  Reference RMS:        {sh['ref_rms']:.6f} s⁻¹")
+        lines.append(f"  RMS difference:       {sh['rms_diff']:.6f} s⁻¹")
+        lines.append(f"  Relative RMS diff:    {sh['relative_rms']:.4f}")
+        lines.append(f"  Correlation:          {sh['corr']:.6f}")
+
+    # Vibration
+    for vb in metrics.get("vibration", []):
+        lines.append("")
+        lines.append(f"Vibration sensor {vb['sensor']}:")
+        lines.append(f"  Reference RMS:        {vb['ref_rms']:.6f}")
+        lines.append(f"  RMS difference:       {vb['rms_diff']:.6f}")
+        lines.append(f"  Relative RMS diff:    {vb['relative_rms']:.4f}")
+        lines.append(f"  Correlation:          {vb['corr']:.6f}")
+
+    return "\n".join(lines)
+
+
+# ---------------------------------------------------------------------------
+# L3 comparison
+# ---------------------------------------------------------------------------
+
+
+def compare_l3(computed: L3Data, reference: L3Data) -> dict:
+    """Compare computed L3 spectra against reference L3.
+
+    Compares wavenumber grids, spectral shapes, and per-window metadata.
+    """
+    results: dict = {}
+
+    results["n_spectra"] = {
+        "ref": reference.n_spectra,
+        "comp": computed.n_spectra,
+    }
+
+    n_common = min(computed.n_spectra, reference.n_spectra)
+    if n_common == 0:
+        return results
+
+    # Speed comparison
+    spd_diff = computed.pspd_rel[:n_common] - reference.pspd_rel[:n_common]
+    finite = np.isfinite(spd_diff)
+    if finite.any():
+        results["speed"] = {
+            "rms_diff": float(np.sqrt(np.mean(spd_diff[finite] ** 2))),
+            "ref_mean": float(np.nanmean(reference.pspd_rel[:n_common])),
+        }
+
+    # Pressure comparison
+    pres_diff = computed.pres[:n_common] - reference.pres[:n_common]
+    finite = np.isfinite(pres_diff)
+    if finite.any():
+        results["pressure"] = {
+            "rms_diff": float(np.sqrt(np.mean(pres_diff[finite] ** 2))),
+            "ref_mean": float(np.nanmean(reference.pres[:n_common])),
+        }
+
+    # Spectral comparison — compare log10(spectrum) to handle dynamic range
+    n_sh = min(computed.n_shear, reference.n_shear)
+    n_wn = min(computed.n_wavenumber, reference.n_wavenumber)
+
+    spec_stats = []
+    for label, comp_spec, ref_spec in [
+        ("raw", computed.sh_spec, reference.sh_spec),
+        ("clean", computed.sh_spec_clean, reference.sh_spec_clean),
+    ]:
+        for i in range(n_sh):
+            c = comp_spec[i, :n_wn, :n_common]
+            r = ref_spec[i, :n_wn, :n_common]
+
+            m = _log_spectral_metrics(c, r)
+            if m["n_valid"] == 0:
+                continue
+
+            spec_stats.append(
+                {
+                    "type": label,
+                    "probe": i + 1,
+                    "mean_log_diff": m["mean_log_diff"],
+                    "rms_log_diff": m["rms_log_diff"],
+                    "median_ratio": m["median_ratio"],
+                    "q0_abs_log": m["q0_abs_log"],
+                    "q50_abs_log": m["q50_abs_log"],
+                    "q100_abs_log": m["q100_abs_log"],
+                    "n_values": m["n_valid"],
+                }
+            )
+    results["spectra"] = spec_stats
+
+    return results
+
+
+def format_l3_report(metrics: dict, filename: str = "") -> str:
+    """Format L3 comparison metrics as a human-readable report."""
+    lines = _format_report_header("L3", filename)
+
+    ns = metrics.get("n_spectra", {})
+    lines.append(f"\n  Reference spectra:  {ns.get('ref', '?')}")
+    lines.append(f"  Computed spectra:   {ns.get('comp', '?')}")
+
+    spd = metrics.get("speed")
+    if spd:
+        lines.append(
+            f"\n  Speed RMS diff:     {spd['rms_diff']:.6f} m/s (ref mean {spd['ref_mean']:.4f})"
+        )
+
+    prs = metrics.get("pressure")
+    if prs:
+        lines.append(
+            f"  Pressure RMS diff:  {prs['rms_diff']:.4f} dbar (ref mean {prs['ref_mean']:.1f})"
+        )
+
+    for ss in metrics.get("spectra", []):
+        lines.append(f"\n  Shear probe {ss['probe']} ({ss['type']}):")
+        lines.append(f"    Mean log10 diff:    {ss['mean_log_diff']:+.4f}")
+        lines.append(f"    RMS log10 diff:     {ss['rms_log_diff']:.4f}")
+        lines.append(f"    Median ratio:       {ss['median_ratio']:.4f}")
+        lines.append(
+            f"    |log10 diff| Q0/Q50/Q100: "
+            f"{ss['q0_abs_log']:.4f} / {ss['q50_abs_log']:.4f} / "
+            f"{ss['q100_abs_log']:.4f}"
+        )
+        lines.append(f"    N values compared:  {ss['n_values']}")
+
+    return "\n".join(lines)
+
+
+# ---------------------------------------------------------------------------
+# L4 comparison
+# ---------------------------------------------------------------------------
+
+
+def compare_l4(computed: L4Data, reference: L4Data) -> dict:
+    """Compare computed L4 against reference L4.
+
+    Returns a dict of comparison metrics.
+    """
+    results: dict = {}
+
+    results["n_spectra"] = {
+        "ref": reference.n_spectra,
+        "comp": computed.n_spectra,
+    }
+
+    n_common = min(computed.n_spectra, reference.n_spectra)
+    if n_common == 0:
+        return results
+
+    n_sh = min(computed.n_shear, reference.n_shear)
+
+    # Per-probe epsilon comparison (log10 space — epsilon spans decades)
+    probe_stats = []
+    for i in range(n_sh):
+        ref_e = reference.epsi[i, :n_common]
+        comp_e = computed.epsi[i, :n_common]
+
+        m = _log_spectral_metrics(comp_e, ref_e)
+        if m["n_valid"] == 0:
+            continue
+
+        # Additional quantile metrics beyond the base helper
+        valid = np.isfinite(ref_e) & np.isfinite(comp_e) & (ref_e > 0) & (comp_e > 0)
+        abs_log_diff = np.abs(np.log10(comp_e[valid]) - np.log10(ref_e[valid]))
+
+        probe_stats.append(
+            {
+                "probe": i + 1,
+                "n_valid": m["n_valid"],
+                "mean_log_diff": m["mean_log_diff"],
+                "rms_log_diff": m["rms_log_diff"],
+                "median_ratio": m["median_ratio"],
+                "q0_abs_log": m["q0_abs_log"],
+                "q25_abs_log": float(np.percentile(abs_log_diff, 25)),
+                "q50_abs_log": m["q50_abs_log"],
+                "q75_abs_log": float(np.percentile(abs_log_diff, 75)),
+                "q100_abs_log": m["q100_abs_log"],
+                "within_factor2": float(np.mean(abs_log_diff < np.log10(2))),
+                "within_factor3": float(np.mean(abs_log_diff < np.log10(3))),
+            }
+        )
+    results["probes"] = probe_stats
+
+    # EPSI_FINAL comparison
+    ref_ef = reference.epsi_final[:n_common]
+    comp_ef = computed.epsi_final[:n_common]
+    m = _log_spectral_metrics(comp_ef, ref_ef)
+    if m["n_valid"] > 0:
+        valid = np.isfinite(ref_ef) & np.isfinite(comp_ef) & (ref_ef > 0) & (comp_ef > 0)
+        abs_log_diff = np.abs(np.log10(comp_ef[valid]) - np.log10(ref_ef[valid]))
+        results["epsi_final"] = {
+            "n_valid": m["n_valid"],
+            "mean_log_diff": m["mean_log_diff"],
+            "rms_log_diff": m["rms_log_diff"],
+            "median_ratio": m["median_ratio"],
+            "q0_abs_log": m["q0_abs_log"],
+            "q50_abs_log": m["q50_abs_log"],
+            "q100_abs_log": m["q100_abs_log"],
+            "within_factor2": float(np.mean(abs_log_diff < np.log10(2))),
+        }
+
+    # FOM comparison.  The benchmark reference FOM is the MAD-based Lueck
+    # (2022) FM statistic, so compare the computed FM against it when
+    # available; the variance-ratio ``fom`` is a different quantity.
+    fom_stats = []
+    comp_fom_source = computed.FM if computed.FM is not None else computed.fom
+    for i in range(n_sh):
+        ref_f = reference.fom[i, :n_common]
+        comp_f = comp_fom_source[i, :n_common]
+        valid = np.isfinite(ref_f) & np.isfinite(comp_f)
+        if valid.any():
+            diff = comp_f[valid] - ref_f[valid]
+            fom_stats.append(
+                {
+                    "probe": i + 1,
+                    "ref_mean": float(np.mean(ref_f[valid])),
+                    "comp_mean": float(np.mean(comp_f[valid])),
+                    "rms_diff": float(np.sqrt(np.mean(diff**2))),
+                    "corr": float(np.corrcoef(ref_f[valid], comp_f[valid])[0, 1])
+                    if len(ref_f[valid]) > 1
+                    else np.nan,
+                }
+            )
+    results["fom"] = fom_stats
+
+    # Method agreement
+    method_stats = []
+    for i in range(n_sh):
+        ref_m = reference.method[i, :n_common]
+        comp_m = computed.method[i, :n_common]
+        valid = np.isfinite(ref_m) & np.isfinite(comp_m)
+        if valid.any():
+            agree = (ref_m[valid] == comp_m[valid]).sum()
+            method_stats.append(
+                {
+                    "probe": i + 1,
+                    "agreement": float(agree / valid.sum()),
+                    "ref_isr_frac": float((ref_m[valid] == 1).mean()),
+                    "comp_isr_frac": float((comp_m[valid] == 1).mean()),
+                }
+            )
+    results["method"] = method_stats
+
+    # QC flag agreement (exact match of the full bitfield, plus per-bit
+    # agreement for the ATOMIX bits)
+    flag_stats = []
+    for i in range(n_sh):
+        ref_f = reference.epsi_flags[i, :n_common]
+        comp_f = computed.epsi_flags[i, :n_common]
+        valid = np.isfinite(ref_f) & np.isfinite(comp_f) & (ref_f != 255) & (comp_f != 255)
+        if valid.any():
+            rf = ref_f[valid].astype(np.int64)
+            cf = comp_f[valid].astype(np.int64)
+            entry = {
+                "probe": i + 1,
+                "exact": float((rf == cf).mean()),
+                "n_ref_flagged": int((rf > 0).sum()),
+                "n_comp_flagged": int((cf > 0).sum()),
+            }
+            for bit, name in ((1, "fom"), (2, "despike"), (4, "ratio"), (8, "passes"), (16, "var")):
+                entry[f"bit_{name}"] = float(((rf & bit) == (cf & bit)).mean())
+            flag_stats.append(entry)
+    results["flags"] = flag_stats
+
+    return results
+
+
+def format_l4_report(metrics: dict, filename: str = "") -> str:
+    """Format L4 comparison metrics as a human-readable report."""
+    lines = _format_report_header("L4", filename)
+
+    ns = metrics.get("n_spectra", {})
+    lines.append(f"\n  Reference spectra:  {ns.get('ref', '?')}")
+    lines.append(f"  Computed spectra:   {ns.get('comp', '?')}")
+
+    for ps in metrics.get("probes", []):
+        lines.append(f"\n  Shear probe {ps['probe']}:")
+        lines.append(f"    N valid:            {ps['n_valid']}")
+        lines.append(f"    Mean log10 diff:    {ps['mean_log_diff']:+.4f}")
+        lines.append(f"    RMS log10 diff:     {ps['rms_log_diff']:.4f}")
+        lines.append(f"    Median ratio:       {ps['median_ratio']:.4f}")
+        lines.append(
+            f"    |log10 diff| Q0/Q25/Q50/Q75/Q100: "
+            f"{ps['q0_abs_log']:.4f} / {ps['q25_abs_log']:.4f} / "
+            f"{ps['q50_abs_log']:.4f} / {ps['q75_abs_log']:.4f} / "
+            f"{ps['q100_abs_log']:.4f}"
+        )
+        lines.append(f"    Within factor 2:    {ps['within_factor2']:.1%}")
+        lines.append(f"    Within factor 3:    {ps['within_factor3']:.1%}")
+
+    ef = metrics.get("epsi_final")
+    if ef:
+        lines.append("\n  EPSI_FINAL (combined):")
+        lines.append(f"    N valid:            {ef['n_valid']}")
+        lines.append(f"    Mean log10 diff:    {ef['mean_log_diff']:+.4f}")
+        lines.append(f"    RMS log10 diff:     {ef['rms_log_diff']:.4f}")
+        lines.append(f"    Median ratio:       {ef['median_ratio']:.4f}")
+        lines.append(
+            f"    |log10 diff| Q0/Q50/Q100: "
+            f"{ef['q0_abs_log']:.4f} / {ef['q50_abs_log']:.4f} / "
+            f"{ef['q100_abs_log']:.4f}"
+        )
+        lines.append(f"    Within factor 2:    {ef['within_factor2']:.1%}")
+
+    for fs in metrics.get("fom", []):
+        lines.append(f"\n  FOM probe {fs['probe']}:")
+        lines.append(
+            f"    Ref mean: {fs['ref_mean']:.4f}  Comp mean: {fs['comp_mean']:.4f}"
+            f"  RMS diff: {fs['rms_diff']:.4f}  Corr: {fs['corr']:.4f}"
+        )
+
+    for ms in metrics.get("method", []):
+        lines.append(f"\n  Method probe {ms['probe']}:")
+        lines.append(
+            f"    Agreement: {ms['agreement']:.1%}"
+            f"  Ref ISR: {ms['ref_isr_frac']:.1%}"
+            f"  Comp ISR: {ms['comp_isr_frac']:.1%}"
+        )
+
+    for fl in metrics.get("flags", []):
+        lines.append(f"\n  QC flags probe {fl['probe']}:")
+        lines.append(
+            f"    Exact match: {fl['exact']:.1%}"
+            f"  (ref flagged: {fl['n_ref_flagged']}, comp flagged: {fl['n_comp_flagged']})"
+        )
+        lines.append(
+            f"    Per-bit agreement: fom {fl['bit_fom']:.1%}"
+            f"  despike {fl['bit_despike']:.1%}"
+            f"  ratio {fl['bit_ratio']:.1%}"
+            f"  passes {fl['bit_passes']:.1%}"
+            f"  var {fl['bit_var']:.1%}"
+        )
+
+    return "\n".join(lines)