eemkit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eemkit/__init__.py +51 -0
- eemkit/absorbance.py +216 -0
- eemkit/campaigns.py +276 -0
- eemkit/core/__init__.py +1 -0
- eemkit/core/dataset.py +91 -0
- eemkit/core/grid.py +100 -0
- eemkit/core/history.py +78 -0
- eemkit/corrections/__init__.py +44 -0
- eemkit/corrections/_common.py +58 -0
- eemkit/corrections/blank.py +66 -0
- eemkit/corrections/crop.py +33 -0
- eemkit/corrections/inner_filter.py +123 -0
- eemkit/corrections/raman.py +107 -0
- eemkit/corrections/scatter.py +191 -0
- eemkit/corrections/spectral.py +117 -0
- eemkit/errors.py +37 -0
- eemkit/gui/__init__.py +30 -0
- eemkit/gui/__main__.py +5 -0
- eemkit/gui/campaigns_tab.py +275 -0
- eemkit/gui/common.py +395 -0
- eemkit/gui/corrections_tab.py +334 -0
- eemkit/gui/export_tab.py +137 -0
- eemkit/gui/indices_tab.py +228 -0
- eemkit/gui/main_window.py +258 -0
- eemkit/gui/parafac_tab.py +399 -0
- eemkit/gui/project_tab.py +332 -0
- eemkit/gui/state.py +149 -0
- eemkit/gui/theme.py +125 -0
- eemkit/indices.py +256 -0
- eemkit/io/__init__.py +1 -0
- eemkit/io/_cary_csv.py +174 -0
- eemkit/io/cary60.py +120 -0
- eemkit/io/cary_eclipse.py +140 -0
- eemkit/io/discover.py +176 -0
- eemkit/io/loader.py +158 -0
- eemkit/io/pairing.py +115 -0
- eemkit/parafac/__init__.py +28 -0
- eemkit/parafac/_cp.py +125 -0
- eemkit/parafac/diagnostics.py +56 -0
- eemkit/parafac/export.py +54 -0
- eemkit/parafac/model.py +191 -0
- eemkit/parafac/validation.py +111 -0
- eemkit/pipeline.py +560 -0
- eemkit/plot.py +437 -0
- eemkit-0.1.0.dist-info/METADATA +195 -0
- eemkit-0.1.0.dist-info/RECORD +49 -0
- eemkit-0.1.0.dist-info/WHEEL +4 -0
- eemkit-0.1.0.dist-info/entry_points.txt +2 -0
- eemkit-0.1.0.dist-info/licenses/LICENSE +674 -0
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
"""Removal of Rayleigh and Raman scatter, and interpolation of the gaps.
|
|
2
|
+
|
|
3
|
+
Four bands are handled, each centred on a line of the (ex, em) plane:
|
|
4
|
+
|
|
5
|
+
============== ===========================================
|
|
6
|
+
band centre (nm)
|
|
7
|
+
============== ===========================================
|
|
8
|
+
``rayleigh1`` em = ex
|
|
9
|
+
``raman1`` em = 1e7 / (1e7 / ex - shift)
|
|
10
|
+
``rayleigh2`` em = 2 ex
|
|
11
|
+
``raman2`` em = 2 * raman1 centre
|
|
12
|
+
============== ===========================================
|
|
13
|
+
|
|
14
|
+
``shift`` is the O-H stretching Raman shift of water in cm^-1. Each band is
|
|
15
|
+
removed between ``centre - below`` and ``centre + above``. Optionally the
|
|
16
|
+
region below first-order Rayleigh (em < ex - offset), where no fluorescence
|
|
17
|
+
can occur, is set to zero; Thygesen et al. (2004, Chemometr. Intell. Lab.
|
|
18
|
+
Syst. 71:97-106) showed that such zeros stabilise PARAFAC.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
from collections.abc import Mapping
|
|
24
|
+
|
|
25
|
+
import numpy as np
|
|
26
|
+
import xarray as xr
|
|
27
|
+
|
|
28
|
+
from eemkit.core.history import has_step, with_history
|
|
29
|
+
from eemkit.corrections._common import require_not_done, require_vars
|
|
30
|
+
from eemkit.errors import CorrectionError
|
|
31
|
+
|
|
32
|
+
BANDS = ("rayleigh1", "raman1", "rayleigh2", "raman2")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def raman_emission(ex: np.ndarray, shift_cm1: float) -> np.ndarray:
|
|
36
|
+
"""Emission wavelength (nm) of the water Raman line excited at ``ex`` (nm)."""
|
|
37
|
+
ex = np.asarray(ex, dtype=float)
|
|
38
|
+
return 1e7 / (1e7 / ex - shift_cm1)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def scatter_mask(
|
|
42
|
+
ex: np.ndarray,
|
|
43
|
+
em: np.ndarray,
|
|
44
|
+
widths: Mapping[str, tuple[float, float]],
|
|
45
|
+
raman_shift_cm1: float | None = None,
|
|
46
|
+
) -> xr.DataArray:
|
|
47
|
+
"""Boolean (ex, em) mask, True inside any of the bands listed in ``widths``.
|
|
48
|
+
|
|
49
|
+
``widths`` maps band names to ``(below, above)`` in nm. A Raman band
|
|
50
|
+
requires ``raman_shift_cm1``.
|
|
51
|
+
"""
|
|
52
|
+
unknown = set(widths) - set(BANDS)
|
|
53
|
+
if unknown:
|
|
54
|
+
raise ValueError(f"unknown scatter band(s) {sorted(unknown)}; use {BANDS}")
|
|
55
|
+
if any(b.startswith("raman") for b in widths) and raman_shift_cm1 is None:
|
|
56
|
+
raise ValueError("raman_shift_cm1 is required to remove Raman scatter")
|
|
57
|
+
ex = np.asarray(ex, dtype=float)
|
|
58
|
+
em = np.asarray(em, dtype=float)
|
|
59
|
+
centres = {"rayleigh1": ex, "rayleigh2": 2 * ex}
|
|
60
|
+
if raman_shift_cm1 is not None:
|
|
61
|
+
r = raman_emission(ex, raman_shift_cm1)
|
|
62
|
+
centres.update(raman1=r, raman2=2 * r)
|
|
63
|
+
mask = np.zeros((ex.size, em.size), dtype=bool)
|
|
64
|
+
for band, (below, above) in widths.items():
|
|
65
|
+
if below < 0 or above < 0:
|
|
66
|
+
raise ValueError(f"{band}: widths must be non-negative")
|
|
67
|
+
c = centres[band][:, None]
|
|
68
|
+
mask |= (em[None, :] >= c - below) & (em[None, :] <= c + above)
|
|
69
|
+
return xr.DataArray(mask, dims=("ex", "em"), coords={"ex": ex, "em": em})
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def remove_scatter(
|
|
73
|
+
ds: xr.Dataset,
|
|
74
|
+
*,
|
|
75
|
+
widths: Mapping[str, tuple[float, float]],
|
|
76
|
+
raman_shift_cm1: float | None = None,
|
|
77
|
+
zero_below_rayleigh: float | None = None,
|
|
78
|
+
variables: tuple[str, ...] = ("intensity",),
|
|
79
|
+
) -> xr.Dataset:
|
|
80
|
+
"""Set scatter bands to NaN (and optionally the region below Rayleigh to 0).
|
|
81
|
+
|
|
82
|
+
Parameters
|
|
83
|
+
----------
|
|
84
|
+
widths
|
|
85
|
+
``{band: (below_nm, above_nm)}`` for the bands to remove; see module doc.
|
|
86
|
+
raman_shift_cm1
|
|
87
|
+
Raman shift of water, needed for Raman bands.
|
|
88
|
+
zero_below_rayleigh
|
|
89
|
+
If given, intensities at ``em < ex - zero_below_rayleigh`` are set to 0
|
|
90
|
+
instead of NaN (applied after the bands, so it never overrides a band
|
|
91
|
+
lying above that limit).
|
|
92
|
+
variables
|
|
93
|
+
Data variables to process.
|
|
94
|
+
|
|
95
|
+
Notes
|
|
96
|
+
-----
|
|
97
|
+
The removed region is stored as ``scatter_mask(ex, em)``.
|
|
98
|
+
"""
|
|
99
|
+
step = "remove_scatter"
|
|
100
|
+
require_vars(ds, *variables, step=step)
|
|
101
|
+
require_not_done(ds, step)
|
|
102
|
+
mask = scatter_mask(ds["ex"].values, ds["em"].values, widths, raman_shift_cm1)
|
|
103
|
+
out = ds.copy()
|
|
104
|
+
zero = None
|
|
105
|
+
if zero_below_rayleigh is not None:
|
|
106
|
+
zero = ds["em"] < ds["ex"] - zero_below_rayleigh
|
|
107
|
+
for name in variables:
|
|
108
|
+
v = ds[name].where(~mask)
|
|
109
|
+
if zero is not None:
|
|
110
|
+
v = xr.where(zero, 0.0, v)
|
|
111
|
+
out[name] = v.transpose(*ds[name].dims).assign_attrs(ds[name].attrs)
|
|
112
|
+
out["scatter_mask"] = mask.assign_attrs(long_name="removed scatter region")
|
|
113
|
+
return with_history(out, step, widths={k: list(v) for k, v in widths.items()},
|
|
114
|
+
raman_shift_cm1=raman_shift_cm1,
|
|
115
|
+
zero_below_rayleigh=zero_below_rayleigh, variables=list(variables))
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
INTERP_METHODS = ("em", "ex", "both")
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _fill_rows(a: np.ndarray, x: np.ndarray) -> np.ndarray:
|
|
122
|
+
"""Linear interpolation of interior NaN gaps along the last axis (no extrapolation)."""
|
|
123
|
+
out = a.copy()
|
|
124
|
+
for row in out.reshape(-1, a.shape[-1]):
|
|
125
|
+
ok = np.isfinite(row)
|
|
126
|
+
if ok.sum() < 2 or ok.all():
|
|
127
|
+
continue
|
|
128
|
+
lo, hi = np.flatnonzero(ok)[[0, -1]]
|
|
129
|
+
gap = ~ok
|
|
130
|
+
gap[:lo] = gap[hi + 1:] = False
|
|
131
|
+
row[gap] = np.interp(x[gap], x[ok], row[ok])
|
|
132
|
+
return out
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def interpolate_missing(
|
|
136
|
+
ds: xr.Dataset,
|
|
137
|
+
*,
|
|
138
|
+
method: str,
|
|
139
|
+
variables: tuple[str, ...] = ("intensity",),
|
|
140
|
+
) -> xr.Dataset:
|
|
141
|
+
"""Fill NaN gaps in each EEM by linear interpolation between measured points.
|
|
142
|
+
|
|
143
|
+
Parameters
|
|
144
|
+
----------
|
|
145
|
+
method
|
|
146
|
+
``"em"``: along emission within each excitation scan; ``"ex"``: along
|
|
147
|
+
excitation at each emission wavelength; ``"both"``: mean of the two
|
|
148
|
+
estimates where both exist, otherwise whichever exists.
|
|
149
|
+
|
|
150
|
+
Notes
|
|
151
|
+
-----
|
|
152
|
+
Only gaps bracketed by measured values are filled; gaps at the edge of
|
|
153
|
+
the EEM stay NaN. The coordinates ``n_interpolated(sample)`` and
|
|
154
|
+
``n_still_missing(sample)`` count filled and remaining values.
|
|
155
|
+
"""
|
|
156
|
+
step = "interpolate_missing"
|
|
157
|
+
require_vars(ds, *variables, step=step)
|
|
158
|
+
if not has_step(ds, "remove_scatter"):
|
|
159
|
+
raise CorrectionError(f"{step}: nothing to interpolate; run remove_scatter first")
|
|
160
|
+
if method not in INTERP_METHODS:
|
|
161
|
+
raise ValueError(f"method must be one of {INTERP_METHODS}")
|
|
162
|
+
ex, em = ds["ex"].values, ds["em"].values
|
|
163
|
+
out = ds.copy()
|
|
164
|
+
counts: dict[str, list[int]] = {}
|
|
165
|
+
for name in variables:
|
|
166
|
+
da = ds[name].transpose(..., "ex", "em")
|
|
167
|
+
vals = da.values
|
|
168
|
+
along_em = _fill_rows(vals, em)
|
|
169
|
+
along_ex = np.swapaxes(_fill_rows(np.swapaxes(vals, -1, -2), ex), -1, -2)
|
|
170
|
+
if method == "em":
|
|
171
|
+
new = along_em
|
|
172
|
+
elif method == "ex":
|
|
173
|
+
new = along_ex
|
|
174
|
+
else:
|
|
175
|
+
with np.errstate(invalid="ignore"):
|
|
176
|
+
new = np.where(np.isfinite(along_em) & np.isfinite(along_ex),
|
|
177
|
+
(along_em + along_ex) / 2,
|
|
178
|
+
np.where(np.isfinite(along_em), along_em, along_ex))
|
|
179
|
+
gap_before = np.isnan(vals).reshape(-1, ex.size * em.size).sum(axis=1)
|
|
180
|
+
gap_after = np.isnan(new).reshape(-1, ex.size * em.size).sum(axis=1)
|
|
181
|
+
vals = new
|
|
182
|
+
filled = [int(v) for v in gap_before - gap_after]
|
|
183
|
+
left = [int(v) for v in gap_after]
|
|
184
|
+
out[name] = (da.dims, vals, da.attrs)
|
|
185
|
+
out[name] = out[name].transpose(*ds[name].dims)
|
|
186
|
+
counts[name] = filled
|
|
187
|
+
if name == "intensity" and "sample" in da.dims:
|
|
188
|
+
out = out.assign_coords(n_interpolated=("sample", filled),
|
|
189
|
+
n_still_missing=("sample", left))
|
|
190
|
+
return with_history(out, step, method=method, variables=list(variables),
|
|
191
|
+
total_interpolated={k: int(sum(v)) for k, v in counts.items()})
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
"""Instrument spectral correction (excitation and emission correction factors).
|
|
2
|
+
|
|
3
|
+
The Cary Eclipse was operated with 'Corrected spectra OFF', so intensities
|
|
4
|
+
contain the wavelength response of lamp, monochromators and detector.
|
|
5
|
+
Correction factors are instrument-specific and must be measured (e.g. with
|
|
6
|
+
certified reference fluorophores); eemkit ships none and never invents them.
|
|
7
|
+
|
|
8
|
+
Factors are read from two-column text files (wavelength in nm, factor) and
|
|
9
|
+
applied **multiplicatively** by default: ``F_corr = F_obs * Xc(ex) * Mc(em)``.
|
|
10
|
+
If your files hold divisors (e.g. an instrument response or lamp reference
|
|
11
|
+
spectrum), pass ``convention="divide"``: ``F_corr = F_obs / Xr(ex) / Mr(em)``.
|
|
12
|
+
Until applied, ``ds.attrs["spectral_correction"]`` is ``"none"``; data
|
|
13
|
+
recorded with the instrument's own correction ON are refused.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
import numpy as np
|
|
21
|
+
import xarray as xr
|
|
22
|
+
|
|
23
|
+
from eemkit.core.history import with_history
|
|
24
|
+
from eemkit.corrections._common import require_not_after, require_not_done, require_vars
|
|
25
|
+
from eemkit.errors import CorrectionError, FileFormatError
|
|
26
|
+
|
|
27
|
+
NOT_CORRECTED = "none"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def read_correction_file(path: str | Path) -> tuple[np.ndarray, np.ndarray]:
|
|
31
|
+
"""Read ``wavelength, factor`` pairs; header lines and ``,;`` or whitespace separators ok."""
|
|
32
|
+
path = Path(path)
|
|
33
|
+
wl, f = [], []
|
|
34
|
+
for n, line in enumerate(path.read_text(encoding="utf-8-sig").splitlines(), 1):
|
|
35
|
+
toks = [t for t in line.replace(";", ",").replace("\t", ",").replace(" ", ",")
|
|
36
|
+
.split(",") if t]
|
|
37
|
+
if not toks:
|
|
38
|
+
continue
|
|
39
|
+
try:
|
|
40
|
+
w, v = float(toks[0]), float(toks[1])
|
|
41
|
+
except (ValueError, IndexError):
|
|
42
|
+
if wl:
|
|
43
|
+
raise FileFormatError(f"{path}, line {n}: expected 'wavelength, factor'") \
|
|
44
|
+
from None
|
|
45
|
+
continue # header
|
|
46
|
+
wl.append(w)
|
|
47
|
+
f.append(v)
|
|
48
|
+
if len(wl) < 2:
|
|
49
|
+
raise FileFormatError(f"{path}: fewer than two numeric rows")
|
|
50
|
+
order = np.argsort(wl)
|
|
51
|
+
return np.asarray(wl)[order], np.asarray(f)[order]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def apply_spectral_correction(
|
|
55
|
+
ds: xr.Dataset,
|
|
56
|
+
*,
|
|
57
|
+
ex_factors: str | Path | tuple[np.ndarray, np.ndarray],
|
|
58
|
+
em_factors: str | Path | tuple[np.ndarray, np.ndarray],
|
|
59
|
+
variables: tuple[str, ...] = ("intensity", "blank_intensity"),
|
|
60
|
+
convention: str = "multiply",
|
|
61
|
+
) -> xr.Dataset:
|
|
62
|
+
"""Multiply (or divide) EEMs by excitation and emission correction factors.
|
|
63
|
+
|
|
64
|
+
Factors are linearly interpolated onto the ``ex`` and ``em`` grids, which
|
|
65
|
+
they must cover (no extrapolation). ``convention`` says what the files
|
|
66
|
+
hold: ``"multiply"`` (correction factors) or ``"divide"`` (response
|
|
67
|
+
spectra). The stored ``spectral_correction_ex/em`` variables are always
|
|
68
|
+
the multipliers actually applied. Must precede every other correction.
|
|
69
|
+
"""
|
|
70
|
+
step = "apply_spectral_correction"
|
|
71
|
+
if convention not in ("multiply", "divide"):
|
|
72
|
+
raise ValueError(f"{step}: convention must be 'multiply' or 'divide', not {convention!r}")
|
|
73
|
+
variables = tuple(v for v in variables if v in ds.data_vars)
|
|
74
|
+
require_vars(ds, "intensity", step=step)
|
|
75
|
+
require_not_done(ds, step)
|
|
76
|
+
if spectral_correction_status(ds) != NOT_CORRECTED:
|
|
77
|
+
raise CorrectionError(f"{step}: the data are already spectrally corrected "
|
|
78
|
+
f"({spectral_correction_status(ds)})")
|
|
79
|
+
for later in ("correct_inner_filter", "subtract_blank", "normalize_raman", "remove_scatter"):
|
|
80
|
+
require_not_after(ds, step, later)
|
|
81
|
+
|
|
82
|
+
sources = {}
|
|
83
|
+
curves = {}
|
|
84
|
+
for dim, spec in (("ex", ex_factors), ("em", em_factors)):
|
|
85
|
+
if isinstance(spec, (str, Path)):
|
|
86
|
+
sources[dim] = str(spec)
|
|
87
|
+
spec = read_correction_file(spec)
|
|
88
|
+
else:
|
|
89
|
+
sources[dim] = "array"
|
|
90
|
+
wl, f = (np.asarray(a, dtype=float) for a in spec)
|
|
91
|
+
grid = ds[dim].values
|
|
92
|
+
if grid.min() < wl.min() - 1e-6 or grid.max() > wl.max() + 1e-6:
|
|
93
|
+
raise CorrectionError(f"{step}: {dim} factors cover {wl.min()}-{wl.max()} nm, "
|
|
94
|
+
f"grid needs {grid.min()}-{grid.max()} nm")
|
|
95
|
+
on_grid = np.interp(grid, wl, f)
|
|
96
|
+
if not np.all(np.isfinite(on_grid) & (on_grid > 0)):
|
|
97
|
+
raise CorrectionError(f"{step}: {dim} factors must be positive over "
|
|
98
|
+
f"{grid.min()}-{grid.max()} nm")
|
|
99
|
+
if convention == "divide":
|
|
100
|
+
on_grid = 1.0 / on_grid
|
|
101
|
+
curves[dim] = xr.DataArray(on_grid, dims=dim, coords={dim: grid})
|
|
102
|
+
|
|
103
|
+
out = ds.copy()
|
|
104
|
+
for name in variables:
|
|
105
|
+
out[name] = (ds[name] * curves["ex"] * curves["em"]).transpose(*ds[name].dims) \
|
|
106
|
+
.assign_attrs(ds[name].attrs)
|
|
107
|
+
out["spectral_correction_ex"] = curves["ex"]
|
|
108
|
+
out["spectral_correction_em"] = curves["em"]
|
|
109
|
+
out.attrs["spectral_correction"] = (f"applied (ex: {sources['ex']}; em: {sources['em']}; "
|
|
110
|
+
f"{convention})")
|
|
111
|
+
return with_history(out, step, ex_factors=sources["ex"], em_factors=sources["em"],
|
|
112
|
+
convention=convention)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def spectral_correction_status(ds: xr.Dataset) -> str:
|
|
116
|
+
"""'none' or a description of the applied correction."""
|
|
117
|
+
return ds.attrs.get("spectral_correction", NOT_CORRECTED)
|
eemkit/errors.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""Exceptions and warnings raised by eemkit."""
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class EEMKitError(Exception):
|
|
5
|
+
"""Base class for eemkit errors."""
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class FileFormatError(EEMKitError, ValueError):
|
|
9
|
+
"""An instrument file does not have the expected layout."""
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class PairingError(EEMKitError, ValueError):
|
|
13
|
+
"""The pairing table is malformed or points to missing files."""
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class GridError(EEMKitError, ValueError):
|
|
17
|
+
"""Spectra cannot be placed on, or combined into, a common wavelength grid."""
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class CorrectionError(EEMKitError, ValueError):
|
|
21
|
+
"""A correction cannot be applied: missing input, wrong order or applied twice."""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class IndexComputationError(EEMKitError, ValueError):
|
|
25
|
+
"""A fluorescence index cannot be computed with the requested settings."""
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class SettingsError(EEMKitError, ValueError):
|
|
29
|
+
"""A processing setting is missing or invalid (e.g. no cuvette path length)."""
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class Cancelled(EEMKitError):
|
|
33
|
+
"""A long computation was cancelled by the caller."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class EEMKitWarning(UserWarning):
|
|
37
|
+
"""Something unusual that does not prevent processing but should be reviewed."""
|
eemkit/gui/__init__.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Desktop application for eemkit (requires ``pip install eemkit[gui]``).
|
|
2
|
+
|
|
3
|
+
Start it with ``eemkit-gui [project.toml]`` or ``python -m eemkit.gui``.
|
|
4
|
+
The application is a front end to :mod:`eemkit.pipeline`: all processing
|
|
5
|
+
is done by the library, and the project file it saves is the same settings
|
|
6
|
+
TOML that scripts can run.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import sys
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def main(argv: list[str] | None = None) -> int:
|
|
15
|
+
try:
|
|
16
|
+
from PySide6.QtWidgets import QApplication
|
|
17
|
+
except ImportError as exc: # pragma: no cover
|
|
18
|
+
print(f"The eemkit desktop application needs PySide6 (pip install eemkit[gui]): {exc}",
|
|
19
|
+
file=sys.stderr)
|
|
20
|
+
return 1
|
|
21
|
+
from eemkit.gui import theme
|
|
22
|
+
from eemkit.gui.main_window import MainWindow
|
|
23
|
+
|
|
24
|
+
argv = sys.argv if argv is None else argv
|
|
25
|
+
app = QApplication.instance() or QApplication(argv)
|
|
26
|
+
app.setApplicationName("eemkit")
|
|
27
|
+
theme.apply_theme(app, theme.saved_theme())
|
|
28
|
+
window = MainWindow(argv[1] if len(argv) > 1 else None)
|
|
29
|
+
window.show()
|
|
30
|
+
return app.exec()
|
eemkit/gui/__main__.py
ADDED
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
"""Tab 5: differences between campaigns, along sites and with covariates."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import pandas as pd
|
|
6
|
+
from PySide6.QtCore import Qt
|
|
7
|
+
from PySide6.QtWidgets import (
|
|
8
|
+
QComboBox,
|
|
9
|
+
QFileDialog,
|
|
10
|
+
QFormLayout,
|
|
11
|
+
QGroupBox,
|
|
12
|
+
QHBoxLayout,
|
|
13
|
+
QLabel,
|
|
14
|
+
QLineEdit,
|
|
15
|
+
QMessageBox,
|
|
16
|
+
QPushButton,
|
|
17
|
+
QSplitter,
|
|
18
|
+
QTableView,
|
|
19
|
+
QTabWidget,
|
|
20
|
+
QVBoxLayout,
|
|
21
|
+
QWidget,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
from eemkit import pipeline, plot
|
|
25
|
+
from eemkit.gui.common import DataFrameModel, PlotPanel, TaskBar, note, pair, primary
|
|
26
|
+
from eemkit.gui.state import Project
|
|
27
|
+
|
|
28
|
+
NONE = "(none)"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _table_view(model: DataFrameModel) -> QTableView:
|
|
32
|
+
v = QTableView()
|
|
33
|
+
v.setModel(model)
|
|
34
|
+
return v
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class CampaignsTab(QWidget):
|
|
38
|
+
def __init__(self, project: Project, tasks: TaskBar, parent=None):
|
|
39
|
+
super().__init__(parent)
|
|
40
|
+
self.project, self.tasks = project, tasks
|
|
41
|
+
|
|
42
|
+
self.value = QComboBox()
|
|
43
|
+
self.group = QComboBox()
|
|
44
|
+
self.time = QComboBox()
|
|
45
|
+
self.site = QComboBox()
|
|
46
|
+
self.position = QComboBox()
|
|
47
|
+
self.sites_file = QLineEdit()
|
|
48
|
+
self.cov_file = QLineEdit()
|
|
49
|
+
for e in (self.sites_file, self.cov_file):
|
|
50
|
+
e.setReadOnly(True)
|
|
51
|
+
e.setPlaceholderText("optional")
|
|
52
|
+
b_sites = QPushButton("Choose…")
|
|
53
|
+
b_sites.clicked.connect(lambda: self.choose_table("sites_table"))
|
|
54
|
+
b_cov = QPushButton("Choose…")
|
|
55
|
+
b_cov.clicked.connect(lambda: self.choose_table("covariates_table"))
|
|
56
|
+
b_clear = QPushButton("Clear tables")
|
|
57
|
+
b_clear.clicked.connect(self.clear_tables)
|
|
58
|
+
|
|
59
|
+
g_cols = QGroupBox("Sample columns")
|
|
60
|
+
f = QFormLayout(g_cols)
|
|
61
|
+
f.addRow("Variable", self.value)
|
|
62
|
+
f.addRow("Group (e.g. river)", self.group)
|
|
63
|
+
f.addRow("Time (e.g. campaign)", self.time)
|
|
64
|
+
f.addRow("Site", self.site)
|
|
65
|
+
f.addRow(note("Replicates are averaged per site and time before any test, so they "
|
|
66
|
+
"are never counted as independent samples."))
|
|
67
|
+
g_opt = QGroupBox("Optional tables")
|
|
68
|
+
f = QFormLayout(g_opt)
|
|
69
|
+
f.addRow("Site table", pair(self.sites_file, b_sites))
|
|
70
|
+
f.addRow("Position along the river", self.position)
|
|
71
|
+
f.addRow("Covariate table", pair(self.cov_file, b_cov))
|
|
72
|
+
f.addRow("", b_clear)
|
|
73
|
+
f.addRow(note("Site table: CSV with the group and site columns plus e.g. latitude, "
|
|
74
|
+
"longitude, distance_km. Covariate table: CSV with the group and time "
|
|
75
|
+
"columns, written exactly as in the samples (e.g. 2024-05), plus e.g. "
|
|
76
|
+
"discharge, rainfall. Rows are joined only on exact matches; missing "
|
|
77
|
+
"rows are reported."))
|
|
78
|
+
|
|
79
|
+
self.b_run = primary(QPushButton("Analyse"))
|
|
80
|
+
self.b_run.clicked.connect(self.analyse)
|
|
81
|
+
self.b_save = QPushButton("Save to output folder")
|
|
82
|
+
self.b_save.clicked.connect(self.save)
|
|
83
|
+
|
|
84
|
+
self.warn = QLabel("")
|
|
85
|
+
self.warn.setWordWrap(True)
|
|
86
|
+
self.warn.setProperty("role", "warning")
|
|
87
|
+
self.warn.hide()
|
|
88
|
+
self.models = {k: DataFrameModel() for k in ("kruskal", "pairwise", "spatial",
|
|
89
|
+
"covariates", "design", "units")}
|
|
90
|
+
self.plots = {k: PlotPanel() for k in ("time", "sites", "composition")}
|
|
91
|
+
views = QTabWidget()
|
|
92
|
+
views.addTab(self.plots["time"], "Over time")
|
|
93
|
+
views.addTab(self.plots["sites"], "Along sites")
|
|
94
|
+
views.addTab(self.plots["composition"], "PARAFAC composition")
|
|
95
|
+
tests = QWidget()
|
|
96
|
+
tl = QVBoxLayout(tests)
|
|
97
|
+
for title, key in (("Kruskal–Wallis between times, within each group", "kruskal"),
|
|
98
|
+
("Dunn's pairwise comparisons (Holm-adjusted p)", "pairwise")):
|
|
99
|
+
tl.addWidget(QLabel(f"<b>{title}</b>"))
|
|
100
|
+
tl.addWidget(_table_view(self.models[key]), 1)
|
|
101
|
+
views.addTab(tests, "Tests")
|
|
102
|
+
corr = QWidget()
|
|
103
|
+
cl = QVBoxLayout(corr)
|
|
104
|
+
for title, key in (("Spearman correlation with position", "spatial"),
|
|
105
|
+
("Spearman correlation with covariates", "covariates")):
|
|
106
|
+
cl.addWidget(QLabel(f"<b>{title}</b>"))
|
|
107
|
+
cl.addWidget(_table_view(self.models[key]), 1)
|
|
108
|
+
views.addTab(corr, "Correlations")
|
|
109
|
+
views.addTab(_table_view(self.models["design"]), "Design")
|
|
110
|
+
views.addTab(_table_view(self.models["units"]), "Site means")
|
|
111
|
+
self.views = views
|
|
112
|
+
|
|
113
|
+
left = QWidget()
|
|
114
|
+
ll = QVBoxLayout(left)
|
|
115
|
+
ll.addWidget(g_cols)
|
|
116
|
+
ll.addWidget(g_opt)
|
|
117
|
+
buttons = QHBoxLayout()
|
|
118
|
+
buttons.addWidget(self.b_run)
|
|
119
|
+
buttons.addWidget(self.b_save)
|
|
120
|
+
ll.addLayout(buttons)
|
|
121
|
+
ll.addStretch(1)
|
|
122
|
+
right = QWidget()
|
|
123
|
+
rl = QVBoxLayout(right)
|
|
124
|
+
rl.addWidget(self.warn)
|
|
125
|
+
rl.addWidget(views, 1)
|
|
126
|
+
split = QSplitter(Qt.Horizontal)
|
|
127
|
+
split.addWidget(left)
|
|
128
|
+
split.addWidget(right)
|
|
129
|
+
split.setStretchFactor(1, 1)
|
|
130
|
+
QVBoxLayout(self).addWidget(split)
|
|
131
|
+
project.changed.connect(self._on_changed)
|
|
132
|
+
tasks.busy_changed.connect(lambda b: self.b_run.setEnabled(not b))
|
|
133
|
+
self.refresh_columns()
|
|
134
|
+
|
|
135
|
+
# columns ------------------------------------------------------------- #
|
|
136
|
+
|
|
137
|
+
def table(self) -> pd.DataFrame | None:
|
|
138
|
+
p = self.project
|
|
139
|
+
if p.indices is None and p.model is None:
|
|
140
|
+
return None
|
|
141
|
+
return pipeline.sample_table(p.indices, p.model)
|
|
142
|
+
|
|
143
|
+
def _site_table_columns(self) -> list[str]:
|
|
144
|
+
path = self.project.resolve("sites_table", "campaigns")
|
|
145
|
+
if path is None or not path.is_file():
|
|
146
|
+
return []
|
|
147
|
+
try:
|
|
148
|
+
return list(pipeline.read_table(path).columns)
|
|
149
|
+
except Exception: # noqa: BLE001 - an unreadable table offers no columns
|
|
150
|
+
return []
|
|
151
|
+
|
|
152
|
+
def refresh_columns(self) -> None:
|
|
153
|
+
df = self.table()
|
|
154
|
+
c = self.project.settings["campaigns"]
|
|
155
|
+
cols = [] if df is None else list(df.reset_index().columns)
|
|
156
|
+
numeric = [] if df is None else [k for k in df.columns
|
|
157
|
+
if pd.api.types.is_numeric_dtype(df[k])]
|
|
158
|
+
labels = [k for k in cols if k not in numeric and k not in ("sample", "cdom_notes")]
|
|
159
|
+
for box, items, want in ((self.value, numeric, self.value.currentText()),
|
|
160
|
+
(self.group, labels, c.get("group")),
|
|
161
|
+
(self.time, labels, c.get("time")),
|
|
162
|
+
(self.site, [NONE] + labels, c.get("site") or NONE),
|
|
163
|
+
(self.position, [NONE] + self._site_table_columns() + numeric,
|
|
164
|
+
c.get("position") or NONE)):
|
|
165
|
+
box.blockSignals(True)
|
|
166
|
+
box.clear()
|
|
167
|
+
box.addItems(list(dict.fromkeys(items)))
|
|
168
|
+
if want:
|
|
169
|
+
box.setCurrentText(want)
|
|
170
|
+
box.blockSignals(False)
|
|
171
|
+
self.sites_file.setText(c.get("sites_table", ""))
|
|
172
|
+
self.cov_file.setText(c.get("covariates_table", ""))
|
|
173
|
+
|
|
174
|
+
def _on_changed(self, what: str) -> None:
|
|
175
|
+
if what in ("settings", "indices", "final"):
|
|
176
|
+
self.refresh_columns()
|
|
177
|
+
if what in ("indices", "final", "campaigns") and self.project.campaign is None:
|
|
178
|
+
self._clear_results("Choose the columns and press Analyse.")
|
|
179
|
+
|
|
180
|
+
def choose_table(self, key: str) -> None:
|
|
181
|
+
path, _ = QFileDialog.getOpenFileName(self, "Choose table", str(self.project.base),
|
|
182
|
+
"Tables (*.csv *.txt)")
|
|
183
|
+
if path:
|
|
184
|
+
self.set_table(key, path)
|
|
185
|
+
|
|
186
|
+
def set_table(self, key: str, path: str) -> None:
|
|
187
|
+
self.project.set_path_setting(key, path, "campaigns")
|
|
188
|
+
self.project.invalidate("campaigns")
|
|
189
|
+
self.refresh_columns()
|
|
190
|
+
|
|
191
|
+
def clear_tables(self) -> None:
|
|
192
|
+
self.project.set_settings("campaigns", {"sites_table": "", "covariates_table": ""},
|
|
193
|
+
"campaigns")
|
|
194
|
+
self.refresh_columns()
|
|
195
|
+
|
|
196
|
+
# analysis ------------------------------------------------------------ #
|
|
197
|
+
|
|
198
|
+
def _store(self) -> None:
|
|
199
|
+
def val(box):
|
|
200
|
+
t = box.currentText()
|
|
201
|
+
return "" if t == NONE else t
|
|
202
|
+
self.project.set_settings("campaigns", {
|
|
203
|
+
"group": val(self.group), "time": val(self.time), "site": val(self.site),
|
|
204
|
+
"position": val(self.position)}, "campaigns")
|
|
205
|
+
|
|
206
|
+
def analyse(self) -> None:
|
|
207
|
+
df = self.table()
|
|
208
|
+
if df is None:
|
|
209
|
+
QMessageBox.information(self, "eemkit", "Compute the indices (or fit the final "
|
|
210
|
+
"PARAFAC model) first.")
|
|
211
|
+
return
|
|
212
|
+
self._store()
|
|
213
|
+
value, settings, base = self.value.currentText(), self.project.settings, self.project.base
|
|
214
|
+
|
|
215
|
+
def work(progress, cancelled):
|
|
216
|
+
return pipeline.campaign_analysis(df, value, settings, base)
|
|
217
|
+
|
|
218
|
+
def done(res):
|
|
219
|
+
self.project.campaign = res
|
|
220
|
+
self.project.changed.emit("campaigns")
|
|
221
|
+
self.show_results()
|
|
222
|
+
|
|
223
|
+
if not value:
|
|
224
|
+
QMessageBox.information(self, "eemkit", "Choose a variable.")
|
|
225
|
+
return
|
|
226
|
+
self.tasks.run(work, done, "Comparing campaigns…")
|
|
227
|
+
|
|
228
|
+
def _clear_results(self, message: str) -> None:
|
|
229
|
+
for m in self.models.values():
|
|
230
|
+
m.set_frame(pd.DataFrame())
|
|
231
|
+
for p in self.plots.values():
|
|
232
|
+
p.clear(message)
|
|
233
|
+
self.warn.hide()
|
|
234
|
+
|
|
235
|
+
def show_results(self) -> None:
|
|
236
|
+
res = self.project.campaign
|
|
237
|
+
if res is None:
|
|
238
|
+
return
|
|
239
|
+
c = self.project.settings["campaigns"]
|
|
240
|
+
self.warn.setText("⚠ " + "\n⚠ ".join(res.warnings) if res.warnings else "")
|
|
241
|
+
self.warn.setVisible(bool(res.warnings))
|
|
242
|
+
for key, df in (("kruskal", res.kruskal), ("pairwise", res.pairwise),
|
|
243
|
+
("spatial", res.spatial), ("covariates", res.covariates),
|
|
244
|
+
("design", res.design.reset_index() if res.design is not None else None),
|
|
245
|
+
("units", res.units)):
|
|
246
|
+
self.models[key].set_frame(df if df is not None else pd.DataFrame())
|
|
247
|
+
self.plots["time"].show_figure(plot.campaign_series(res.units, res.value,
|
|
248
|
+
group=c["group"], time=c["time"]))
|
|
249
|
+
if res.spatial is not None:
|
|
250
|
+
self.plots["sites"].show_figure(plot.along_sites(res.units, res.value, c["position"],
|
|
251
|
+
group=c["group"]))
|
|
252
|
+
else:
|
|
253
|
+
self.plots["sites"].clear("Choose a position column (e.g. distance_km from a site "
|
|
254
|
+
"table) to see the variable along the sites.")
|
|
255
|
+
pct = [k for k in res.units.columns if k.endswith("_pct") and not k.endswith("cv_pct")]
|
|
256
|
+
if pct:
|
|
257
|
+
self.plots["composition"].show_figure(plot.composition(
|
|
258
|
+
res.units, pct, group=c["group"], time=c["time"]))
|
|
259
|
+
else:
|
|
260
|
+
self.plots["composition"].clear("Fit the final PARAFAC model to see the share of "
|
|
261
|
+
"each component.")
|
|
262
|
+
|
|
263
|
+
def save(self) -> None:
|
|
264
|
+
res, out = self.project.campaign, self.project.resolve("output_dir")
|
|
265
|
+
if res is None:
|
|
266
|
+
QMessageBox.information(self, "eemkit", "Run the analysis first.")
|
|
267
|
+
return
|
|
268
|
+
if out is None or self.project.in_data_root(out):
|
|
269
|
+
QMessageBox.information(self, "eemkit", "Choose an output folder outside the data "
|
|
270
|
+
"folder (Project tab).")
|
|
271
|
+
return
|
|
272
|
+
written = pipeline.export_campaigns(out / "campaigns" / res.value, res,
|
|
273
|
+
self.project.settings)
|
|
274
|
+
QMessageBox.information(self, "eemkit", f"Wrote {len(written)} files to "
|
|
275
|
+
f"{out / 'campaigns' / res.value}.")
|