eemkit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eemkit/__init__.py +51 -0
- eemkit/absorbance.py +216 -0
- eemkit/campaigns.py +276 -0
- eemkit/core/__init__.py +1 -0
- eemkit/core/dataset.py +91 -0
- eemkit/core/grid.py +100 -0
- eemkit/core/history.py +78 -0
- eemkit/corrections/__init__.py +44 -0
- eemkit/corrections/_common.py +58 -0
- eemkit/corrections/blank.py +66 -0
- eemkit/corrections/crop.py +33 -0
- eemkit/corrections/inner_filter.py +123 -0
- eemkit/corrections/raman.py +107 -0
- eemkit/corrections/scatter.py +191 -0
- eemkit/corrections/spectral.py +117 -0
- eemkit/errors.py +37 -0
- eemkit/gui/__init__.py +30 -0
- eemkit/gui/__main__.py +5 -0
- eemkit/gui/campaigns_tab.py +275 -0
- eemkit/gui/common.py +395 -0
- eemkit/gui/corrections_tab.py +334 -0
- eemkit/gui/export_tab.py +137 -0
- eemkit/gui/indices_tab.py +228 -0
- eemkit/gui/main_window.py +258 -0
- eemkit/gui/parafac_tab.py +399 -0
- eemkit/gui/project_tab.py +332 -0
- eemkit/gui/state.py +149 -0
- eemkit/gui/theme.py +125 -0
- eemkit/indices.py +256 -0
- eemkit/io/__init__.py +1 -0
- eemkit/io/_cary_csv.py +174 -0
- eemkit/io/cary60.py +120 -0
- eemkit/io/cary_eclipse.py +140 -0
- eemkit/io/discover.py +176 -0
- eemkit/io/loader.py +158 -0
- eemkit/io/pairing.py +115 -0
- eemkit/parafac/__init__.py +28 -0
- eemkit/parafac/_cp.py +125 -0
- eemkit/parafac/diagnostics.py +56 -0
- eemkit/parafac/export.py +54 -0
- eemkit/parafac/model.py +191 -0
- eemkit/parafac/validation.py +111 -0
- eemkit/pipeline.py +560 -0
- eemkit/plot.py +437 -0
- eemkit-0.1.0.dist-info/METADATA +195 -0
- eemkit-0.1.0.dist-info/RECORD +49 -0
- eemkit-0.1.0.dist-info/WHEEL +4 -0
- eemkit-0.1.0.dist-info/entry_points.txt +2 -0
- eemkit-0.1.0.dist-info/licenses/LICENSE +674 -0
eemkit/core/grid.py
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""Wavelength grids and resampling of spectra onto them."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Iterable
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
import xarray as xr
|
|
9
|
+
|
|
10
|
+
from eemkit.core.history import with_history
|
|
11
|
+
from eemkit.errors import GridError
|
|
12
|
+
|
|
13
|
+
#: Grid points this far (nm) outside the measured range take the edge value
|
|
14
|
+
#: instead of NaN. Absorbs the ~0.02 nm jitter of instrument wavelength axes.
|
|
15
|
+
DEFAULT_TOLERANCE = 0.1
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def regular_grid(start: float, stop: float, step: float) -> np.ndarray:
|
|
19
|
+
"""Evenly spaced grid from ``start`` to ``stop`` inclusive.
|
|
20
|
+
|
|
21
|
+
``stop - start`` must be a whole number of steps.
|
|
22
|
+
"""
|
|
23
|
+
if step <= 0 or stop <= start:
|
|
24
|
+
raise GridError(f"invalid grid {start}..{stop} step {step}")
|
|
25
|
+
n = (stop - start) / step
|
|
26
|
+
if not np.isclose(n, round(n), atol=1e-6):
|
|
27
|
+
raise GridError(f"{start}..{stop} is not a whole number of {step} nm steps")
|
|
28
|
+
return np.round(start + step * np.arange(round(n) + 1), 10)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def interp_on_grid(
|
|
32
|
+
x: np.ndarray, y: np.ndarray, grid: np.ndarray, tolerance: float = DEFAULT_TOLERANCE
|
|
33
|
+
) -> np.ndarray:
|
|
34
|
+
"""Linearly interpolate the spectrum ``y(x)`` onto ``grid``.
|
|
35
|
+
|
|
36
|
+
NaN pairs are dropped and ``x`` may be in any order. Grid points farther
|
|
37
|
+
than ``tolerance`` outside the measured range are NaN (no extrapolation).
|
|
38
|
+
"""
|
|
39
|
+
x = np.asarray(x, dtype=float)
|
|
40
|
+
y = np.asarray(y, dtype=float)
|
|
41
|
+
grid = np.asarray(grid, dtype=float)
|
|
42
|
+
ok = np.isfinite(x) & np.isfinite(y)
|
|
43
|
+
if ok.sum() < 2:
|
|
44
|
+
return np.full(grid.shape, np.nan)
|
|
45
|
+
xs, ys = x[ok], y[ok]
|
|
46
|
+
order = np.argsort(xs)
|
|
47
|
+
xs, ys = xs[order], ys[order]
|
|
48
|
+
if np.any(np.diff(xs) == 0):
|
|
49
|
+
raise GridError("duplicate wavelengths in spectrum")
|
|
50
|
+
out = np.interp(grid, xs, ys)
|
|
51
|
+
out[(grid < xs[0] - tolerance) | (grid > xs[-1] + tolerance)] = np.nan
|
|
52
|
+
return out
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def resample(
|
|
56
|
+
ds: xr.Dataset, dim: str, grid: np.ndarray, tolerance: float = DEFAULT_TOLERANCE
|
|
57
|
+
) -> xr.Dataset:
|
|
58
|
+
"""Resample every data variable of ``ds`` along ``dim`` onto ``grid``.
|
|
59
|
+
|
|
60
|
+
Linear interpolation; recorded in the processing history.
|
|
61
|
+
"""
|
|
62
|
+
grid = np.asarray(grid, dtype=float)
|
|
63
|
+
old = ds[dim].values
|
|
64
|
+
out = ds.drop_dims(dim)
|
|
65
|
+
for name, var in ds.data_vars.items():
|
|
66
|
+
if dim not in var.dims:
|
|
67
|
+
out[name] = var
|
|
68
|
+
continue
|
|
69
|
+
moved = np.moveaxis(var.values, var.dims.index(dim), -1)
|
|
70
|
+
flat = moved.reshape(-1, moved.shape[-1])
|
|
71
|
+
res = np.stack([interp_on_grid(old, row, grid, tolerance) for row in flat])
|
|
72
|
+
res = np.moveaxis(res.reshape(moved.shape[:-1] + (grid.size,)), -1, var.dims.index(dim))
|
|
73
|
+
out[name] = (var.dims, res, var.attrs)
|
|
74
|
+
out = out.assign_coords({dim: (dim, grid, ds[dim].attrs)})
|
|
75
|
+
out.attrs = dict(ds.attrs)
|
|
76
|
+
# the native axis is left out on purpose: it is in the source file, and keeping
|
|
77
|
+
# the entry identical across samples lets a stacked dataset hold one history
|
|
78
|
+
return with_history(out, "resample", dim=dim, grid=grid, method="linear",
|
|
79
|
+
tolerance_nm=tolerance)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def infer_common_grid(axes: Iterable[np.ndarray], decimals: int = 0) -> np.ndarray:
|
|
83
|
+
"""Propose a regular grid covered by every axis in ``axes``.
|
|
84
|
+
|
|
85
|
+
The step is the coarsest of the median native steps (rounded to
|
|
86
|
+
``decimals``), so no axis is resampled to a finer resolution than it was
|
|
87
|
+
measured at. Start and stop are the overlap of all axes, snapped inward
|
|
88
|
+
onto multiples of the step.
|
|
89
|
+
"""
|
|
90
|
+
axes = [np.asarray(a, dtype=float) for a in axes]
|
|
91
|
+
if not axes:
|
|
92
|
+
raise GridError("no axes given")
|
|
93
|
+
step = max(round(float(np.median(np.abs(np.diff(a)))), decimals) for a in axes)
|
|
94
|
+
if step <= 0:
|
|
95
|
+
raise GridError("cannot infer a positive step")
|
|
96
|
+
lo = max(np.nanmin(a) for a in axes)
|
|
97
|
+
hi = min(np.nanmax(a) for a in axes)
|
|
98
|
+
start = np.ceil(round(lo / step - DEFAULT_TOLERANCE / step, 6)) * step
|
|
99
|
+
stop = np.floor(round(hi / step + DEFAULT_TOLERANCE / step, 6)) * step
|
|
100
|
+
return regular_grid(start, stop, step)
|
eemkit/core/history.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Processing history (provenance) stored in dataset attributes.
|
|
2
|
+
|
|
3
|
+
Every function that changes data appends one entry to ``ds.attrs["eemkit_history"]``,
|
|
4
|
+
a JSON-encoded list, so the record survives a round trip through netCDF.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
from datetime import UTC, datetime
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
import numpy as np
|
|
14
|
+
import xarray as xr
|
|
15
|
+
|
|
16
|
+
HISTORY_ATTR = "eemkit_history"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def get_history(ds: xr.Dataset | xr.DataArray) -> list[dict[str, Any]]:
|
|
20
|
+
"""Return the list of processing steps applied to ``ds``, oldest first."""
|
|
21
|
+
return json.loads(ds.attrs.get(HISTORY_ATTR, "[]"))
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def make_entry(step: str, **params: Any) -> dict[str, Any]:
|
|
25
|
+
"""One history record: step name, parameters, eemkit version, UTC time."""
|
|
26
|
+
from eemkit import __version__
|
|
27
|
+
|
|
28
|
+
return {
|
|
29
|
+
"step": step,
|
|
30
|
+
"params": _jsonable(params),
|
|
31
|
+
"eemkit_version": __version__,
|
|
32
|
+
"time": datetime.now(UTC).isoformat(timespec="seconds"),
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def with_history(ds: xr.Dataset, step: str, **params: Any) -> xr.Dataset:
|
|
37
|
+
"""Return a shallow copy of ``ds`` with one history entry appended."""
|
|
38
|
+
out = ds.copy()
|
|
39
|
+
out.attrs[HISTORY_ATTR] = json.dumps(get_history(ds) + [make_entry(step, **params)])
|
|
40
|
+
return out
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def has_step(ds: xr.Dataset | xr.DataArray, step: str) -> bool:
|
|
44
|
+
"""True if ``step`` appears in the processing history of ``ds``."""
|
|
45
|
+
return any(e["step"] == step for e in get_history(ds))
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def same_history(a: list[dict], b: list[dict]) -> bool:
|
|
49
|
+
"""Compare two histories ignoring timestamps."""
|
|
50
|
+
strip = [{k: v for k, v in e.items() if k != "time"} for e in a]
|
|
51
|
+
return strip == [{k: v for k, v in e.items() if k != "time"} for e in b]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def describe_grid(grid: np.ndarray) -> dict[str, Any]:
|
|
55
|
+
"""Compact, JSON-friendly description of a wavelength grid."""
|
|
56
|
+
g = np.asarray(grid, dtype=float)
|
|
57
|
+
steps = np.diff(g)
|
|
58
|
+
regular = steps.size > 0 and np.allclose(steps, steps[0])
|
|
59
|
+
return {
|
|
60
|
+
"start": float(g[0]),
|
|
61
|
+
"stop": float(g[-1]),
|
|
62
|
+
"n": int(g.size),
|
|
63
|
+
"step": float(steps[0]) if regular else None,
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _jsonable(obj: Any) -> Any:
|
|
68
|
+
if isinstance(obj, dict):
|
|
69
|
+
return {str(k): _jsonable(v) for k, v in obj.items()}
|
|
70
|
+
if isinstance(obj, (list, tuple)):
|
|
71
|
+
return [_jsonable(v) for v in obj]
|
|
72
|
+
if isinstance(obj, np.ndarray):
|
|
73
|
+
return describe_grid(obj) if obj.ndim == 1 and obj.size > 1 else obj.tolist()
|
|
74
|
+
if isinstance(obj, np.generic):
|
|
75
|
+
return obj.item()
|
|
76
|
+
if obj is None or isinstance(obj, (str, int, float, bool)):
|
|
77
|
+
return obj
|
|
78
|
+
return str(obj)
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""EEM corrections. Each is a separate, optional function recorded in the history.
|
|
2
|
+
|
|
3
|
+
Recommended order (each function refuses to run after a step that must follow it):
|
|
4
|
+
|
|
5
|
+
1. :func:`apply_spectral_correction` (instrument response; optional)
|
|
6
|
+
2. :func:`subtract_absorbance_blank`
|
|
7
|
+
3. :func:`correct_inner_filter`
|
|
8
|
+
4. :func:`subtract_blank`
|
|
9
|
+
5. :func:`normalize_raman`
|
|
10
|
+
6. :func:`remove_scatter`, then optionally :func:`interpolate_missing`
|
|
11
|
+
7. :func:`crop`
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from eemkit.corrections.blank import subtract_absorbance_blank, subtract_blank
|
|
15
|
+
from eemkit.corrections.crop import crop
|
|
16
|
+
from eemkit.corrections.inner_filter import correct_inner_filter
|
|
17
|
+
from eemkit.corrections.raman import normalize_raman, raman_area
|
|
18
|
+
from eemkit.corrections.scatter import (
|
|
19
|
+
interpolate_missing,
|
|
20
|
+
raman_emission,
|
|
21
|
+
remove_scatter,
|
|
22
|
+
scatter_mask,
|
|
23
|
+
)
|
|
24
|
+
from eemkit.corrections.spectral import (
|
|
25
|
+
apply_spectral_correction,
|
|
26
|
+
read_correction_file,
|
|
27
|
+
spectral_correction_status,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
__all__ = [
|
|
31
|
+
"apply_spectral_correction",
|
|
32
|
+
"correct_inner_filter",
|
|
33
|
+
"crop",
|
|
34
|
+
"interpolate_missing",
|
|
35
|
+
"normalize_raman",
|
|
36
|
+
"raman_area",
|
|
37
|
+
"raman_emission",
|
|
38
|
+
"read_correction_file",
|
|
39
|
+
"remove_scatter",
|
|
40
|
+
"scatter_mask",
|
|
41
|
+
"spectral_correction_status",
|
|
42
|
+
"subtract_absorbance_blank",
|
|
43
|
+
"subtract_blank",
|
|
44
|
+
]
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""Helpers shared by the correction functions."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
import xarray as xr
|
|
7
|
+
|
|
8
|
+
from eemkit.core.dataset import SAMPLE_DIM
|
|
9
|
+
from eemkit.core.history import has_step
|
|
10
|
+
from eemkit.errors import CorrectionError
|
|
11
|
+
|
|
12
|
+
MISSING_POLICIES = ("error", "keep", "drop")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def require_vars(ds: xr.Dataset, *names: str, step: str) -> None:
|
|
16
|
+
missing = [n for n in names if n not in ds.data_vars]
|
|
17
|
+
if missing:
|
|
18
|
+
raise CorrectionError(f"{step}: dataset has no {missing}")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def require_not_done(ds: xr.Dataset, step: str) -> None:
|
|
22
|
+
if has_step(ds, step):
|
|
23
|
+
raise CorrectionError(f"{step} has already been applied to this dataset")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def require_not_after(ds: xr.Dataset, step: str, later: str) -> None:
|
|
27
|
+
if has_step(ds, later):
|
|
28
|
+
raise CorrectionError(f"{step} must be applied before {later}")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def resolve_missing(
|
|
32
|
+
ds: xr.Dataset, available: xr.DataArray, policy: str, step: str, what: str
|
|
33
|
+
) -> tuple[xr.Dataset, xr.DataArray]:
|
|
34
|
+
"""Apply the policy for samples that lack an input (``available`` is False).
|
|
35
|
+
|
|
36
|
+
'error' raises listing the samples, 'drop' removes them, 'keep' leaves
|
|
37
|
+
them uncorrected. Returns the (possibly reduced) dataset and the mask of
|
|
38
|
+
samples that will be corrected.
|
|
39
|
+
"""
|
|
40
|
+
if policy not in MISSING_POLICIES:
|
|
41
|
+
raise ValueError(f"missing must be one of {MISSING_POLICIES}")
|
|
42
|
+
lacking = list(ds[SAMPLE_DIM].values[~available.values])
|
|
43
|
+
if lacking and policy == "error":
|
|
44
|
+
raise CorrectionError(
|
|
45
|
+
f"{step}: {len(lacking)} sample(s) have no {what}: {lacking[:10]}"
|
|
46
|
+
+ (" ..." if len(lacking) > 10 else "")
|
|
47
|
+
+ "; fix the pairing table or pass missing='keep' or 'drop'"
|
|
48
|
+
)
|
|
49
|
+
if lacking and policy == "drop":
|
|
50
|
+
ds = ds.sel({SAMPLE_DIM: available.values})
|
|
51
|
+
available = available.sel({SAMPLE_DIM: available.values})
|
|
52
|
+
return ds, available
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def has_data(da: xr.DataArray) -> xr.DataArray:
|
|
56
|
+
"""Per-sample flag: True where the array is not entirely NaN."""
|
|
57
|
+
dims = [d for d in da.dims if d != SAMPLE_DIM]
|
|
58
|
+
return np.isfinite(da).any(dims)
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""Blank subtraction for EEMs and absorbance spectra."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import xarray as xr
|
|
6
|
+
|
|
7
|
+
from eemkit.core.history import with_history
|
|
8
|
+
from eemkit.corrections._common import (
|
|
9
|
+
has_data,
|
|
10
|
+
require_not_after,
|
|
11
|
+
require_not_done,
|
|
12
|
+
require_vars,
|
|
13
|
+
resolve_missing,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def subtract_blank(ds: xr.Dataset, missing: str = "error") -> xr.Dataset:
|
|
18
|
+
"""Subtract the paired blank EEM from each sample EEM.
|
|
19
|
+
|
|
20
|
+
``intensity`` becomes ``intensity - blank_intensity``; ``blank_intensity``
|
|
21
|
+
is kept unchanged (it is still needed for Raman normalisation). The
|
|
22
|
+
coordinate ``blank_subtracted(sample)`` records which samples were
|
|
23
|
+
corrected.
|
|
24
|
+
|
|
25
|
+
Parameters
|
|
26
|
+
----------
|
|
27
|
+
missing
|
|
28
|
+
What to do with samples that have no blank: ``"error"`` (default),
|
|
29
|
+
``"drop"`` them, or ``"keep"`` them uncorrected.
|
|
30
|
+
"""
|
|
31
|
+
step = "subtract_blank"
|
|
32
|
+
require_vars(ds, "intensity", "blank_intensity", step=step)
|
|
33
|
+
require_not_done(ds, step)
|
|
34
|
+
require_not_after(ds, step, "normalize_raman")
|
|
35
|
+
require_not_after(ds, step, "remove_scatter")
|
|
36
|
+
ds, ok = resolve_missing(ds, has_data(ds["blank_intensity"]), missing, step, "blank EEM")
|
|
37
|
+
|
|
38
|
+
out = ds.copy()
|
|
39
|
+
out["intensity"] = xr.where(ok, ds["intensity"] - ds["blank_intensity"], ds["intensity"],
|
|
40
|
+
keep_attrs=True)
|
|
41
|
+
out = out.assign_coords(blank_subtracted=ok)
|
|
42
|
+
return with_history(out, step, missing=missing, n_corrected=int(ok.sum()),
|
|
43
|
+
n_uncorrected=int((~ok).sum()))
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def subtract_absorbance_blank(ds: xr.Dataset, missing: str = "error") -> xr.Dataset:
|
|
47
|
+
"""Subtract the paired absorbance blank: ``absorbance - blank_absorbance``.
|
|
48
|
+
|
|
49
|
+
Coordinate ``absorbance_blank_subtracted(sample)`` records which samples
|
|
50
|
+
were corrected. ``missing`` as in :func:`subtract_blank`; samples without
|
|
51
|
+
any absorbance are left alone.
|
|
52
|
+
"""
|
|
53
|
+
step = "subtract_absorbance_blank"
|
|
54
|
+
require_vars(ds, "absorbance", "blank_absorbance", step=step)
|
|
55
|
+
require_not_done(ds, step)
|
|
56
|
+
require_not_after(ds, step, "correct_inner_filter")
|
|
57
|
+
has_abs = has_data(ds["absorbance"])
|
|
58
|
+
ds, ok = resolve_missing(ds, has_data(ds["blank_absorbance"]) | ~has_abs, missing, step,
|
|
59
|
+
"absorbance blank")
|
|
60
|
+
ok = ok & has_data(ds["absorbance"])
|
|
61
|
+
|
|
62
|
+
out = ds.copy()
|
|
63
|
+
out["absorbance"] = xr.where(ok, ds["absorbance"] - ds["blank_absorbance"],
|
|
64
|
+
ds["absorbance"], keep_attrs=True)
|
|
65
|
+
out = out.assign_coords(absorbance_blank_subtracted=ok)
|
|
66
|
+
return with_history(out, step, missing=missing, n_corrected=int(ok.sum()))
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""Restricting EEMs to a wavelength window."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import xarray as xr
|
|
6
|
+
|
|
7
|
+
from eemkit.core.history import with_history
|
|
8
|
+
from eemkit.errors import CorrectionError
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def crop(
|
|
12
|
+
ds: xr.Dataset,
|
|
13
|
+
*,
|
|
14
|
+
ex: tuple[float | None, float | None] | None = None,
|
|
15
|
+
em: tuple[float | None, float | None] | None = None,
|
|
16
|
+
) -> xr.Dataset:
|
|
17
|
+
"""Keep ``ex`` and ``em`` within the given inclusive ``(min, max)`` limits.
|
|
18
|
+
|
|
19
|
+
``None`` leaves a limit (or a whole axis) unchanged. Absorbance, which
|
|
20
|
+
lives on its own ``wavelength`` axis, is not affected.
|
|
21
|
+
"""
|
|
22
|
+
sel = {}
|
|
23
|
+
for dim, lim in (("ex", ex), ("em", em)):
|
|
24
|
+
if lim is None:
|
|
25
|
+
continue
|
|
26
|
+
lo, hi = lim
|
|
27
|
+
sel[dim] = slice(lo, hi)
|
|
28
|
+
out = ds.sel(sel)
|
|
29
|
+
for dim in sel:
|
|
30
|
+
if out.sizes[dim] == 0:
|
|
31
|
+
raise CorrectionError(f"crop: no {dim} wavelength within {sel[dim]}")
|
|
32
|
+
return with_history(out, "crop", ex=list(ex) if ex else None, em=list(em) if em else None,
|
|
33
|
+
n_ex=out.sizes["ex"], n_em=out.sizes["em"])
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"""Absorbance-based inner-filter-effect (IFE) correction.
|
|
2
|
+
|
|
3
|
+
For a right-angle geometry with central illumination of a square cuvette,
|
|
4
|
+
|
|
5
|
+
F_corr(ex, em) = F_obs(ex, em) * 10 ** ((A_ex + A_em) / 2)
|
|
6
|
+
|
|
7
|
+
where A_ex and A_em are the sample absorbances at the excitation and
|
|
8
|
+
emission wavelengths over the fluorescence cuvette path length (Lakowicz,
|
|
9
|
+
*Principles of Fluorescence Spectroscopy*, 3rd ed., 2006). Kothawala et al.
|
|
10
|
+
(2013, Limnol. Oceanogr. Methods 11:616-630) showed that this
|
|
11
|
+
absorbance-based approach is reliable for samples with A254 up to about 1.5
|
|
12
|
+
(1 cm path) and recommended dilution above that.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import warnings
|
|
18
|
+
|
|
19
|
+
import numpy as np
|
|
20
|
+
import xarray as xr
|
|
21
|
+
|
|
22
|
+
from eemkit.core.dataset import SAMPLE_DIM
|
|
23
|
+
from eemkit.core.history import has_step, with_history
|
|
24
|
+
from eemkit.corrections._common import (
|
|
25
|
+
has_data,
|
|
26
|
+
require_not_after,
|
|
27
|
+
require_not_done,
|
|
28
|
+
require_vars,
|
|
29
|
+
resolve_missing,
|
|
30
|
+
)
|
|
31
|
+
from eemkit.errors import CorrectionError, EEMKitWarning
|
|
32
|
+
|
|
33
|
+
KOTHAWALA_MAX_A254 = 1.5
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def correct_inner_filter(
|
|
37
|
+
ds: xr.Dataset,
|
|
38
|
+
*,
|
|
39
|
+
absorbance_path_length_cm: float,
|
|
40
|
+
fluorescence_path_length_cm: float,
|
|
41
|
+
missing: str = "error",
|
|
42
|
+
max_a254: float = KOTHAWALA_MAX_A254,
|
|
43
|
+
) -> xr.Dataset:
|
|
44
|
+
"""Correct ``intensity`` for inner-filter effects using the sample absorbance.
|
|
45
|
+
|
|
46
|
+
Parameters
|
|
47
|
+
----------
|
|
48
|
+
absorbance_path_length_cm
|
|
49
|
+
Path length of the cuvette the absorbance was measured in.
|
|
50
|
+
fluorescence_path_length_cm
|
|
51
|
+
Path length of the fluorescence cuvette. Both are required: eemkit
|
|
52
|
+
never assumes a cuvette.
|
|
53
|
+
missing
|
|
54
|
+
Samples without absorbance: ``"error"``, ``"drop"`` or ``"keep"``.
|
|
55
|
+
max_a254
|
|
56
|
+
Samples whose A254 over the fluorescence path exceeds this value get a
|
|
57
|
+
warning (Kothawala et al. 2013 recommend dilution above 1.5).
|
|
58
|
+
|
|
59
|
+
Notes
|
|
60
|
+
-----
|
|
61
|
+
The absorbance must already be blank-corrected
|
|
62
|
+
(:func:`~eemkit.corrections.subtract_absorbance_blank`). The applied
|
|
63
|
+
factor is kept as the variable ``ife_factor(sample, ex, em)``.
|
|
64
|
+
"""
|
|
65
|
+
step = "correct_inner_filter"
|
|
66
|
+
require_vars(ds, "intensity", "absorbance", step=step)
|
|
67
|
+
require_not_done(ds, step)
|
|
68
|
+
# The water Raman signal inside the sample is attenuated like the DOM
|
|
69
|
+
# fluorescence, so IFE must be corrected before the blank is subtracted.
|
|
70
|
+
for later in ("subtract_blank", "normalize_raman", "remove_scatter"):
|
|
71
|
+
require_not_after(ds, step, later)
|
|
72
|
+
if not has_step(ds, "subtract_absorbance_blank"):
|
|
73
|
+
raise CorrectionError(f"{step}: subtract the absorbance blank first")
|
|
74
|
+
for name, val in (("absorbance_path_length_cm", absorbance_path_length_cm),
|
|
75
|
+
("fluorescence_path_length_cm", fluorescence_path_length_cm)):
|
|
76
|
+
if not np.isfinite(val) or val <= 0:
|
|
77
|
+
raise CorrectionError(f"{step}: {name} must be a positive number, got {val}")
|
|
78
|
+
|
|
79
|
+
wl = ds["wavelength"].values
|
|
80
|
+
for dim in ("ex", "em"):
|
|
81
|
+
lo, hi = ds[dim].values.min(), ds[dim].values.max()
|
|
82
|
+
if lo < wl.min() or hi > wl.max():
|
|
83
|
+
raise CorrectionError(f"{step}: absorbance ({wl.min()}-{wl.max()} nm) does not "
|
|
84
|
+
f"cover {dim} {lo}-{hi} nm")
|
|
85
|
+
|
|
86
|
+
ds, ok = resolve_missing(ds, has_data(ds["absorbance"]), missing, step, "absorbance")
|
|
87
|
+
a_fl = ds["absorbance"] / absorbance_path_length_cm * fluorescence_path_length_cm
|
|
88
|
+
a_ex = _at(a_fl, ds["ex"])
|
|
89
|
+
a_em = _at(a_fl, ds["em"])
|
|
90
|
+
factor = 10.0 ** ((a_ex + a_em) / 2.0)
|
|
91
|
+
factor = xr.where(ok, factor, 1.0).transpose(SAMPLE_DIM, "ex", "em")
|
|
92
|
+
|
|
93
|
+
if 254 >= wl.min() and 254 <= wl.max():
|
|
94
|
+
a254 = _at(a_fl, xr.DataArray([254.0], dims="wavelength")).squeeze(drop=True)
|
|
95
|
+
high = list(ds[SAMPLE_DIM].values[(a254 > max_a254).values & ok.values])
|
|
96
|
+
if high:
|
|
97
|
+
warnings.warn(f"{step}: A254 > {max_a254} for {high}; Kothawala et al. (2013) "
|
|
98
|
+
"recommend diluting such samples", EEMKitWarning, stacklevel=2)
|
|
99
|
+
|
|
100
|
+
out = ds.copy()
|
|
101
|
+
out["intensity"] = (ds["intensity"] * factor).assign_attrs(ds["intensity"].attrs)
|
|
102
|
+
out["ife_factor"] = factor.assign_attrs(
|
|
103
|
+
long_name="inner-filter correction factor 10**((A_ex+A_em)/2)", units="1")
|
|
104
|
+
out = out.assign_coords(ife_corrected=ok)
|
|
105
|
+
return with_history(
|
|
106
|
+
out, step, method="absorbance-based (Kothawala et al. 2013)",
|
|
107
|
+
absorbance_path_length_cm=absorbance_path_length_cm,
|
|
108
|
+
fluorescence_path_length_cm=fluorescence_path_length_cm, missing=missing,
|
|
109
|
+
n_corrected=int(ok.sum()), max_factor=float(factor.max()),
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _at(a: xr.DataArray, wavelengths: xr.DataArray) -> xr.DataArray:
|
|
114
|
+
"""Absorbance at the given wavelengths, linear interpolation, renamed to their dim."""
|
|
115
|
+
dim = wavelengths.dims[0]
|
|
116
|
+
vals = np.apply_along_axis(
|
|
117
|
+
lambda row: np.interp(wavelengths.values, a["wavelength"].values, row),
|
|
118
|
+
a.get_axis_num("wavelength"), a.values,
|
|
119
|
+
)
|
|
120
|
+
dims = tuple(dim if d == "wavelength" else d for d in a.dims)
|
|
121
|
+
coords = {d: a[d].values for d in a.dims if d != "wavelength" and d in a.coords}
|
|
122
|
+
coords[dim] = wavelengths.values
|
|
123
|
+
return xr.DataArray(vals, dims=dims, coords=coords)
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""Normalisation to Raman units (Lawaetz & Stedmon 2009, Appl. Spectrosc. 63:936-940).
|
|
2
|
+
|
|
3
|
+
Intensities are divided by the area of the water Raman peak of a pure-water
|
|
4
|
+
blank measured in the same session, integrated over emission at a fixed
|
|
5
|
+
excitation. Lawaetz & Stedmon used excitation 350 nm and emission 381-426 nm.
|
|
6
|
+
Without instrument spectral correction the result is internally consistent
|
|
7
|
+
but not comparable with Raman units from other instruments.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import numpy as np
|
|
13
|
+
import xarray as xr
|
|
14
|
+
|
|
15
|
+
from eemkit.core.history import with_history
|
|
16
|
+
from eemkit.corrections._common import (
|
|
17
|
+
require_not_after,
|
|
18
|
+
require_not_done,
|
|
19
|
+
require_vars,
|
|
20
|
+
resolve_missing,
|
|
21
|
+
)
|
|
22
|
+
from eemkit.errors import CorrectionError
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def raman_area(
|
|
26
|
+
blank: xr.DataArray,
|
|
27
|
+
*,
|
|
28
|
+
ex: float,
|
|
29
|
+
em_range: tuple[float, float],
|
|
30
|
+
baseline: bool = False,
|
|
31
|
+
) -> xr.DataArray:
|
|
32
|
+
"""Integrated water Raman peak of ``blank`` (dims ``..., ex, em``).
|
|
33
|
+
|
|
34
|
+
The spectrum at excitation ``ex`` (which must be measured) is treated as
|
|
35
|
+
piecewise linear and integrated exactly from ``em_range[0]`` to
|
|
36
|
+
``em_range[1]``, so the limits need not be grid points. With
|
|
37
|
+
``baseline=True`` the straight line joining the spectrum at the two limits
|
|
38
|
+
is subtracted first. Result in intensity units x nm; NaN for an all-NaN
|
|
39
|
+
spectrum (no blank).
|
|
40
|
+
"""
|
|
41
|
+
exs = blank["ex"].values
|
|
42
|
+
if not np.any(np.isclose(exs, ex)):
|
|
43
|
+
raise CorrectionError(f"raman_area: excitation {ex} nm is not measured "
|
|
44
|
+
f"({exs.min()}-{exs.max()} nm)")
|
|
45
|
+
lo, hi = em_range
|
|
46
|
+
em = blank["em"].values
|
|
47
|
+
if not (em.min() <= lo < hi <= em.max()):
|
|
48
|
+
raise CorrectionError(f"raman_area: {lo}-{hi} nm is outside the emission range "
|
|
49
|
+
f"{em.min()}-{em.max()} nm")
|
|
50
|
+
spec = blank.sel(ex=exs[np.isclose(exs, ex)][0])
|
|
51
|
+
inside = (em > lo) & (em < hi)
|
|
52
|
+
x = np.concatenate([[lo], em[inside], [hi]])
|
|
53
|
+
# grid points that enter the integral, including the neighbours of each limit
|
|
54
|
+
used = slice(np.searchsorted(em, lo, side="right") - 1, np.searchsorted(em, hi) + 1)
|
|
55
|
+
|
|
56
|
+
def area_of(row: np.ndarray) -> float:
|
|
57
|
+
if np.isnan(row).all():
|
|
58
|
+
return np.nan
|
|
59
|
+
if np.isnan(row[used]).any():
|
|
60
|
+
raise CorrectionError("raman_area: NaN inside the integration range "
|
|
61
|
+
"(was scatter removed before normalising?)")
|
|
62
|
+
y = np.concatenate([[np.interp(lo, em, row)], row[inside], [np.interp(hi, em, row)]])
|
|
63
|
+
if baseline:
|
|
64
|
+
y = y - (y[0] + (y[-1] - y[0]) * (x - x[0]) / (x[-1] - x[0]))
|
|
65
|
+
return float(np.trapezoid(y, x))
|
|
66
|
+
|
|
67
|
+
axis = spec.get_axis_num("em")
|
|
68
|
+
area = np.apply_along_axis(area_of, axis, spec.values)
|
|
69
|
+
dims = tuple(d for d in spec.dims if d != "em")
|
|
70
|
+
return xr.DataArray(area, dims=dims, coords={d: spec[d].values for d in dims})
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def normalize_raman(
|
|
74
|
+
ds: xr.Dataset,
|
|
75
|
+
*,
|
|
76
|
+
ex: float,
|
|
77
|
+
em_range: tuple[float, float],
|
|
78
|
+
baseline: bool = False,
|
|
79
|
+
missing: str = "error",
|
|
80
|
+
) -> xr.Dataset:
|
|
81
|
+
"""Divide ``intensity`` and ``blank_intensity`` by the blank's Raman area.
|
|
82
|
+
|
|
83
|
+
Each sample is normalised with its own paired blank. The area (a.u. nm)
|
|
84
|
+
is stored as the coordinate ``raman_area(sample)`` and the intensity
|
|
85
|
+
units become ``R.U.``. ``missing`` controls samples without a blank.
|
|
86
|
+
"""
|
|
87
|
+
step = "normalize_raman"
|
|
88
|
+
require_vars(ds, "intensity", "blank_intensity", step=step)
|
|
89
|
+
require_not_done(ds, step)
|
|
90
|
+
require_not_after(ds, step, "remove_scatter")
|
|
91
|
+
area = raman_area(ds["blank_intensity"], ex=ex, em_range=em_range, baseline=baseline)
|
|
92
|
+
ds, ok = resolve_missing(ds, np.isfinite(area), missing, step, "blank for the Raman area")
|
|
93
|
+
area = area.sel(sample=ds["sample"].values)
|
|
94
|
+
if ((area <= 0) & ok).any():
|
|
95
|
+
bad = list(ds["sample"].values[((area <= 0) & ok).values])
|
|
96
|
+
raise CorrectionError(f"{step}: non-positive Raman area for {bad}")
|
|
97
|
+
|
|
98
|
+
scale = xr.where(ok, area, 1.0)
|
|
99
|
+
units = "R.U." if bool(ok.all()) else "R.U. (a.u. where raman_normalized is False)"
|
|
100
|
+
out = ds.copy()
|
|
101
|
+
for name in ("intensity", "blank_intensity"):
|
|
102
|
+
out[name] = (ds[name] / scale).assign_attrs(ds[name].attrs, units=units)
|
|
103
|
+
out = out.assign_coords(raman_area=area.where(ok).assign_attrs(units="a.u. nm"),
|
|
104
|
+
raman_normalized=ok)
|
|
105
|
+
return with_history(out, step, method="Lawaetz & Stedmon (2009)", ex=ex,
|
|
106
|
+
em_range=list(em_range), baseline=baseline, missing=missing,
|
|
107
|
+
n_normalized=int(ok.sum()))
|