electropycal 0.9.0__py3-none-any.whl

electropycal/__init__.py

@@ -0,0 +1,7 @@
+"""electropycal — data processing and recalibration for electrochemical sensors."""
+__version__ = "0.9.0"
+
+from .features.catalog import (FEATURE_DEFINITIONS, catalog_frame, feature_catalog,
+                               print_feature_catalog)
+
+__all__ = ["FEATURE_DEFINITIONS", "catalog_frame", "feature_catalog", "print_feature_catalog"]

electropycal/_demo.py

@@ -0,0 +1,145 @@
+"""Locate the shipped demo tree, from wherever a notebook happens to be running.
+
+**Private on purpose.** The leading underscore keeps this out of the 0.9.0 public API
+(same convention as :mod:`electropycal._parallel`): it is scaffolding so the shipped
+notebooks have something to open, not a path API anyone should build on. It lives
+*inside the package* rather than next to the notebooks because in Colab the notebook is
+uploaded on its own and the pip-installed wheel is the only thing present — a sibling
+``notebooks/_demo.py`` would not be importable there.
+
+The problem it solves: every public notebook used to hardcode ``ROOT =
+"demo/in_vitro/input"``, a path relative to the repo root. A Jupyter kernel's working
+directory is the *notebook's own* directory, so opening ``notebooks/x.ipynb`` and
+running it gave ``FileNotFoundError: 'demo\\in_vitro\\input'`` several cells in. Swapping
+in ``"../demo/..."`` just moves the breakage: it would then work from ``notebooks/`` and
+fail from the repo root, from VS Code with a workspace-root cwd, and in Colab.
+
+So nothing here counts parent directories. Candidate roots are probed in order and each
+is accepted only if the tree it should contain is really there:
+
+1. ``$ELECTROPYCAL_DEMO`` — explicit override, for CI or an unpacked release.
+2. the current working directory and each of its ancestors — covers the repo root,
+   ``notebooks/``, ``publish/``, and anywhere else inside a checkout.
+3. the installed package's own location — covers an editable/src-layout install driven
+   from an unrelated cwd. A wheel install has no demo tree next to it, so this simply
+   does not match, which is the correct answer rather than a wrong guess.
+
+If none match — the ordinary Colab case, where there is no checkout at all —
+:func:`demo_input` synthesizes an equivalent tree instead of failing.
+"""
+
+from __future__ import annotations
+
+import os
+import tempfile
+from pathlib import Path
+
+__all__ = ["DemoTreeNotFound", "demo_input", "find_demo_tree", "require_dir"]
+
+#: Demo kinds and the generator that can stand in for each when nothing is on disk.
+_KINDS = {"in_vitro": "write_synthetic_pstrace_dir", "in_vivo": "write_synthetic_invivo_dir"}
+
+
+class DemoTreeNotFound(FileNotFoundError):
+    """Raised when the demo tree cannot be located and synthesis was not wanted."""
+
+
+def _candidate_roots() -> list[Path]:
+    """Roots that might contain a ``demo/`` directory, most explicit first."""
+    roots: list[Path] = []
+
+    env = os.environ.get("ELECTROPYCAL_DEMO")
+    if env:
+        p = Path(env).expanduser()
+        # Accept either the tree root (containing demo/) or demo/ itself.
+        roots += [p, p.parent]
+
+    cwd = Path.cwd().resolve()
+    roots += [cwd, *cwd.parents]
+
+    # src-layout editable install: .../<root>/src/electropycal/_demo.py -> parents[2] is <root>.
+    pkg = Path(__file__).resolve()
+    roots += [pkg.parents[2], pkg.parents[1]] if len(pkg.parents) > 2 else []
+
+    seen, uniq = set(), []
+    for r in roots:
+        if r not in seen:
+            seen.add(r)
+            uniq.append(r)
+    return uniq
+
+
+def _relative_targets(kind: str) -> list[Path]:
+    """Sub-paths, relative to a candidate root, that would BE the demo input tree."""
+    return [Path("demo") / kind / "input", Path(kind) / "input"]
+
+
+def find_demo_tree(kind: str = "in_vitro") -> Path | None:
+    """Return the shipped demo *input* tree for ``kind``, or ``None`` if unreachable.
+
+    ``None`` is a normal outcome, not an error: it means no checkout is present (Colab),
+    and the caller should synthesize a tree instead.
+    """
+    if kind not in _KINDS:
+        raise ValueError(f"unknown demo kind {kind!r}; expected one of {sorted(_KINDS)}")
+    for root in _candidate_roots():
+        for rel in _relative_targets(kind):
+            cand = root / rel
+            if cand.is_dir() and any(cand.iterdir()):
+                return cand.resolve()
+    return None
+
+
+def _searched_report(kind: str) -> str:
+    """Human-readable account of what was looked for and where — never a bare errno."""
+    rels = " or ".join(str(r) for r in _relative_targets(kind))
+    lines = [f"could not find the shipped demo tree for kind={kind!r}.",
+             f"  looked for : {rels}",
+             f"  under      : {len(_candidate_roots())} candidate root(s):"]
+    lines += [f"               {r}" for r in _candidate_roots()[:12]]
+    lines += ["",
+              "  Fixes, in order of least surprise:",
+              "    - run scripts/build_demo_dataset.py --out demo   (builds it, ~15 s)",
+              "    - set ELECTROPYCAL_DEMO=/path/to/the/tree/containing/demo",
+              "    - point ROOT at your own PSTrace export directory instead"]
+    return "\n".join(lines)
+
+
+def require_dir(path, what: str = "directory") -> str:
+    """Validate a caller-supplied path, reporting what was expected rather than an errno."""
+    p = Path(path).expanduser()
+    if p.is_dir():
+        return str(p.resolve())
+    raise DemoTreeNotFound(
+        f"{what} not found: {p!r}\n"
+        f"  resolved to : {p.resolve() if not p.is_absolute() else p}\n"
+        f"  cwd         : {Path.cwd()}\n"
+        f"  A relative path is resolved against the KERNEL's working directory, which for a\n"
+        f"  notebook is the notebook's own folder -- not the repository root.")
+
+
+def demo_input(kind: str = "in_vitro", user_path=None, *,
+               synthesize: bool = True, **synth_kwargs) -> str:
+    """One call that every public notebook can use for its input tree.
+
+    ``user_path``  -- if given, validated and returned (clear error if absent).
+    otherwise      -- the shipped demo tree if any candidate root has one,
+    otherwise      -- a freshly synthesized equivalent, unless ``synthesize=False``.
+
+    Returns a string path, because the review classes take ``str | Path`` and notebooks
+    print it.
+    """
+    if user_path is not None:
+        return require_dir(user_path, f"{kind} input directory")
+
+    found = find_demo_tree(kind)
+    if found is not None:
+        return str(found)
+
+    if not synthesize:
+        raise DemoTreeNotFound(_searched_report(kind))
+
+    from . import data  # noqa: F401  (ensure the subpackage is importable before getattr)
+    from .data import synthetic
+    writer = getattr(synthetic, _KINDS[kind])
+    return str(writer(tempfile.mkdtemp(prefix=f"electropycal-demo-{kind}-"), **synth_kwargs))

electropycal/_parallel.py

@@ -0,0 +1,40 @@
+"""Shared joblib helper for the library's parallel work units.
+
+Two callers: the session-parallel raw walks (extraction, QC, reliability), one unit per
+device-timepoint session, and discovery's fold loop, one unit per CV fold. Both are
+embarrassingly parallel, so they parallelize cleanly. The one hazard for **non-notebook / direct-API** callers is
+BLAS/OpenMP oversubscription: without the thread pin the notebooks set, each worker process would
+spin up a full multi-threaded BLAS pool, so ``n_jobs`` workers × ``cores`` threads fight over the
+cores and the "speedup" can be a slowdown. :func:`map_sessions` closes that hole by capping every
+worker to a single inner thread (``inner_max_num_threads=1``) itself — so a plain
+``extract_dataset(n_jobs=8)`` from a script is safe with no environment setup required.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from typing import Any
+
+
+def should_parallelize(n_jobs: int, n_items: int) -> bool:
+    """True when a joblib pool is worth spawning: more than one worker requested (``n_jobs`` not
+    0/1; negatives like -1 mean "all cores") and more than one item to spread over it. Below that,
+    the worker spawn + pickling overhead outweighs the gain, so the caller runs serially."""
+    return n_jobs not in (0, 1) and n_items > 1
+
+
+def map_sessions(n_jobs: int, tasks: Iterable[Any]) -> list:
+    """Run ``tasks`` (an iterable of ``joblib.delayed(...)`` calls) across worker processes, with
+    **each worker capped to one BLAS/OpenMP thread** so parallel sessions never oversubscribe the
+    cores — even when the caller has not pinned BLAS in its environment. joblib preserves input
+    order, so the returned list is in ``tasks`` order (the raw walks rely on this for deterministic,
+    serial-identical output). The caller decides *whether* to parallelize (see
+    :func:`should_parallelize`); this only runs the pool once that decision is made.
+
+    Named for its first caller; ``tasks`` are any independent work units — raw-walk sessions or
+    discovery CV folds."""
+    from joblib import Parallel, parallel_config
+    # inner_max_num_threads=1: the library defends itself against oversubscription regardless of the
+    # caller's environment (notebook pin, script with nothing set, CI). loky = process backend.
+    with parallel_config(backend="loky", inner_max_num_threads=1):
+        return list(Parallel(n_jobs=n_jobs)(tasks))

electropycal/analysis_config.py

@@ -0,0 +1,168 @@
+"""Shared analysis configuration for the review/QC/discovery notebooks.
+
+The three pre-made notebooks — ``raw_spectra_review``, ``quality_filtering_dashboard``,
+and ``discovery_checkpointed`` — run as **separate** Colab kernels, so a config set in
+one cannot be seen by the others. To keep the EIS ``band`` and the QC / extraction
+parameters from drifting apart between them, this module persists a single
+``electropycal_analysis_config.json`` **inside the data ``ROOT``**. Each notebook loads it
+(falling back to defaults if absent) and may still override any field in its setup cell.
+
+Workflow:
+  1. In ``raw_spectra_review`` set the band + params and call ``save_analysis_config(ROOT, cfg)``.
+  2. ``quality_filtering_dashboard`` and ``discovery_checkpointed`` call
+     ``load_analysis_config(ROOT)`` and get exactly those values.
+
+The library primitives (``extract_dataset``, ``channel_quality_report``, the CLI) still take
+explicit parameters — this is a convenience layer for the notebooks, not a new gate.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+
+CONFIG_FILENAME = "electropycal_analysis_config.json"
+QC_STATS_FILENAME = "electropycal_qc_stats.json"
+
+
+@dataclass
+class AnalysisConfig:
+    """Analysis **parameters** shared by the raw-spectra / QC-dashboard / discovery notebooks.
+
+    This holds only processing/QC knobs that must agree across notebooks (band, peak method,
+    acceptance, gates). It deliberately does **not** hold data *selection* (device types / ids /
+    channels / timepoints): selection is per-notebook-run (raw_spectra_review is often run one
+    device at a time, while discovery runs across many), so each notebook keeps its own selection
+    and only the parameters are shared.
+    """
+
+    band: tuple[float, float] | str = (2.0, 2000.0)   # (lo, hi) Hz; "auto" for data-driven. Study
+    #: default 2–2000 Hz: the f-dependent drift-alignment review (diagnostics §3.1) shows aligned
+    #: drift concentrated ~2 Hz–2 kHz, while retention is flat below the ~10 kHz inductive-onset cliff,
+    #: so 2 kHz captures the signal at full retention and 2 Hz reaches the low-f aligned band.
+    peak_method: str = "direct"                   # "direct" | "chord"
+    acceptance: str = "monotonic"                 # extract_dataset acceptance test
+    mono_tol: float = 0.10                         # EIS.2 |Z|-monotonicity tolerance
+    min_norm_snr: float = 3.0                      # per-dose reproducibility-SNR cutoff
+    monotonic_r_min: float = 0.6                   # dose-monotonicity min log-conc correlation
+    mono_method: str = "pearson"                   # dose-response corr: "pearson" | "spearman" (rank)
+    max_reps: int | None = 3                       # first N FSCV replicate cycles
+    gate_on: dict = field(default_factory=lambda: {
+        "eis": True, "monotonic": True, "snr_all": False, "peak_in_window": False})
+
+
+def config_path(root: str | Path) -> Path:
+    """Where the shared config lives for a given data ``root``."""
+    return Path(root) / CONFIG_FILENAME
+
+
+def load_analysis_config(root: str | Path, **overrides) -> AnalysisConfig:
+    """Load the shared config from ``<root>/electropycal_analysis_config.json``.
+
+    Returns defaults when the file is absent. Any keyword in ``overrides`` that is not
+    ``None`` replaces the loaded value (so a notebook cell can override a field inline).
+    Unknown keys in the file are ignored, so old configs keep loading as fields are added.
+    """
+    data: dict = {}
+    p = config_path(root)
+    if p.exists():
+        data = json.loads(p.read_text())
+    known = {f for f in AnalysisConfig().__dataclass_fields__}
+    cfg = AnalysisConfig(**{k: v for k, v in data.items() if k in known})
+    if isinstance(cfg.band, list):                # JSON has no tuples
+        cfg.band = tuple(cfg.band)
+    for k, v in overrides.items():
+        if v is not None and k in known:
+            setattr(cfg, k, v)
+    return cfg
+
+
+def save_analysis_config(root: str | Path, cfg: AnalysisConfig) -> Path:
+    """Write ``cfg`` to ``<root>/electropycal_analysis_config.json`` and return the path."""
+    p = config_path(root)
+    p.write_text(json.dumps(asdict(cfg), indent=2, default=list))
+    return p
+
+
+def qc_stats_path(root: str | Path) -> Path:
+    """Where the QC-stats companion artifact lives for a given data ``root``."""
+    return Path(root) / QC_STATS_FILENAME
+
+
+def save_qc_stats(root: str | Path, stats: dict) -> Path:
+    """Write the QC-statistics companion to ``<root>/electropycal_qc_stats.json``.
+
+    This is the *evidence* alongside the config (``save_analysis_config``): the config records
+    **what** QC parameters were used, this records **what they did** on the data (yields, per-gate
+    dropout, the stringency-sweep numbers) so a finalized run documents both the knobs and their
+    effect. ``stats`` should embed the config that produced it (e.g. under a ``"config"`` key).
+    Non-JSON scalars (numpy types) are coerced via ``default=str``.
+    """
+    p = qc_stats_path(root)
+    p.write_text(json.dumps(stats, indent=2, default=str))
+    return p
+
+
+def load_qc_stats(root: str | Path) -> dict | None:
+    """Load the QC-stats companion, or ``None`` if it has not been written yet."""
+    p = qc_stats_path(root)
+    return json.loads(p.read_text()) if p.exists() else None
+
+
+def format_provenance(root: str | Path, target: str | None = None,
+                      cfg: AnalysisConfig | None = None) -> str:
+    """One human-readable block: the pre-processing **parameters**, the **decisions**, and (if the
+    QC-stats companion exists) their **impact** on the dataset. Printed at the start of every discovery
+    run — whatever the entry point (notebook / script / CLI) — so a run always states what
+    pre-processing produced the featureset it trained on, and why."""
+    cfg = cfg or load_analysis_config(root)
+    stats = load_qc_stats(root)
+    b = cfg.band if isinstance(cfg.band, str) else f"({cfg.band[0]:g}, {cfg.band[1]:g}) Hz"
+    L = ["=" * 74,
+         "ElectroPyCal — run provenance  (pre-processing parameters · decisions · impact)",
+         "=" * 74,
+         "config (electropycal_analysis_config.json):",
+         f"  band = {b}   peak_method = {cfg.peak_method}   acceptance = {cfg.acceptance}",
+         f"  EIS.2 mono_tol = {cfg.mono_tol}   FSCV.1 monotonic_r_min = {cfg.monotonic_r_min} "
+         f"({cfg.mono_method})   min_norm_snr = {cfg.min_norm_snr}   max_reps = {cfg.max_reps}",
+         f"  gate_on = {cfg.gate_on}",
+         "decisions / conventions:",
+         f"  target = {target or 'NormIpeak'}",
+         "  D0-normalization: ON (per-sensor drift-from-baseline; additive for phase/bounded types, "
+         "multiplicative for magnitudes)",
+         "  leakage-safe predictors: EIS + 0 nM-background FSCV only; faradaic peak features are "
+         "RESERVED targets",
+         "  NormIpeak < 0 dose rows dropped (drop_negative, non-physical)"]
+    if stats:
+        t = stats.get("totals", {})
+        L.append("QC impact (electropycal_qc_stats.json):")
+        L.append(f"  paired = {t.get('channel_timepoints_paired')}   retained = "
+                 f"{t.get('overall_valid')} ({t.get('overall_valid_pct')}%)")
+        for g in (stats.get("gate_impact") or []):
+            L.append(f"    {g.get('gate')}: {g.get('n_channeltimepoints_failed')} failed "
+                     f"({g.get('failed_%')}%)")
+    else:
+        L.append("QC impact: electropycal_qc_stats.json not found "
+                 "(run quality_filtering_dashboard §7 to generate it)")
+    L += ["see docs/DESIGN.md for the method rationale.", "=" * 74]
+    return "\n".join(L)
+
+
+def print_provenance(root: str | Path, target: str | None = None,
+                     cfg: AnalysisConfig | None = None) -> None:
+    """Print :func:`format_provenance` (safe: never raises — provenance is informational)."""
+    try:
+        print(format_provenance(root, target=target, cfg=cfg), flush=True)
+    except Exception as e:                                     # never let provenance break a run
+        print(f"(provenance unavailable: {e})", flush=True)
+
+
+def resolve_band(root: str | Path, cfg: AnalysisConfig) -> tuple[float, float]:
+    """Resolve ``cfg.band`` to a concrete ``(lo, hi)`` — ``"auto"`` → ``recommended_band(root)``."""
+    if isinstance(cfg.band, str):
+        if cfg.band != "auto":
+            raise ValueError(f"band must be a (lo, hi) tuple or 'auto', got {cfg.band!r}")
+        from .data.inventory import recommended_band
+        return recommended_band(root)
+    return (float(cfg.band[0]), float(cfg.band[1]))