chromparser 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- chromparser/__init__.py +183 -0
- chromparser/__main__.py +7 -0
- chromparser/_netcdf3.py +548 -0
- chromparser/_util.py +98 -0
- chromparser/cli.py +192 -0
- chromparser/errors.py +49 -0
- chromparser/hdf5.py +428 -0
- chromparser/model.py +254 -0
- chromparser/py.typed +0 -0
- chromparser/sniff.py +80 -0
- chromparser/vendors/__init__.py +55 -0
- chromparser/vendors/agilent/__init__.py +17 -0
- chromparser/vendors/agilent/ch.py +344 -0
- chromparser/vendors/agilent/dot_d.py +69 -0
- chromparser/vendors/agilent/dx.py +147 -0
- chromparser/vendors/andi/__init__.py +15 -0
- chromparser/vendors/andi/chrom.py +181 -0
- chromparser/vendors/andi/common.py +232 -0
- chromparser/vendors/andi/ms.py +358 -0
- chromparser/vendors/andi/write.py +320 -0
- chromparser/vendors/chromeleon/__init__.py +19 -0
- chromparser/vendors/chromeleon/archive.py +310 -0
- chromparser/vendors/chromeleon/raw.py +407 -0
- chromparser/vendors/chromeleon/sequence.py +177 -0
- chromparser/vendors/chromeleon/txt.py +319 -0
- chromparser/vendors/shimadzu/__init__.py +8 -0
- chromparser/vendors/shimadzu/gcd.py +146 -0
- chromparser/vendors/shimadzu/ole.py +84 -0
- chromparser/vendors/shimadzu/qgd.py +196 -0
- chromparser/vendors/thermo/__init__.py +7 -0
- chromparser/vendors/thermo/raw.py +377 -0
- chromparser-0.1.0.dist-info/METADATA +162 -0
- chromparser-0.1.0.dist-info/RECORD +36 -0
- chromparser-0.1.0.dist-info/WHEEL +4 -0
- chromparser-0.1.0.dist-info/entry_points.txt +3 -0
- chromparser-0.1.0.dist-info/licenses/LICENSE +51 -0
|
@@ -0,0 +1,358 @@
|
|
|
1
|
+
"""ANDI-MS netCDF files: centroided GC-MS runs.
|
|
2
|
+
|
|
3
|
+
One file holds one run: per-scan variables on the scan dimension
|
|
4
|
+
(`scan_acquisition_time` in seconds, `total_intensity`, `scan_index`,
|
|
5
|
+
`point_count`, ...) and the peaks of all scans back to back on the point
|
|
6
|
+
dimension (`mass_values`, `intensity_values`). See `docs/formats/andi.md`.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
import numpy as np
|
|
14
|
+
|
|
15
|
+
from ... import _netcdf3 as nc
|
|
16
|
+
from ...errors import CorruptFile, UnsupportedVariant, warn
|
|
17
|
+
from ...model import Injection, MSData
|
|
18
|
+
from .common import (
|
|
19
|
+
apply_scale,
|
|
20
|
+
attribute_metadata,
|
|
21
|
+
check_variables,
|
|
22
|
+
has_variable,
|
|
23
|
+
injection_of,
|
|
24
|
+
is_fill,
|
|
25
|
+
json_value,
|
|
26
|
+
open_andi,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
KEY = "mass_values"
|
|
30
|
+
CENTROID = "Centroided Mass Spectrum"
|
|
31
|
+
MASS_SCAN = "Mass Scan"
|
|
32
|
+
IONISATION = {"Electron Impact": "EI", "Chemical Ionization": "CI"}
|
|
33
|
+
POLARITY = {"Positive Polarity": "positive", "Negative Polarity": "negative"}
|
|
34
|
+
|
|
35
|
+
#: Per-scan variables of the template that hold no data chromparser keeps
|
|
36
|
+
#: (sentinels such as -9999 or 0 in the files seen so far).
|
|
37
|
+
IGNORED_PER_SCAN = (
|
|
38
|
+
"a_d_sampling_rate",
|
|
39
|
+
"a_d_coaddition_factor",
|
|
40
|
+
"scan_duration",
|
|
41
|
+
"inter_scan_time",
|
|
42
|
+
"resolution",
|
|
43
|
+
"time_range_min",
|
|
44
|
+
"time_range_max",
|
|
45
|
+
)
|
|
46
|
+
PER_SCAN = (
|
|
47
|
+
"scan_acquisition_time",
|
|
48
|
+
"actual_scan_number",
|
|
49
|
+
"total_intensity",
|
|
50
|
+
"scan_index",
|
|
51
|
+
"point_count",
|
|
52
|
+
"flag_count",
|
|
53
|
+
"mass_range_min",
|
|
54
|
+
"mass_range_max",
|
|
55
|
+
*IGNORED_PER_SCAN,
|
|
56
|
+
)
|
|
57
|
+
PER_POINT = ("mass_values", "intensity_values", "time_values")
|
|
58
|
+
#: instrument variable -> metadata key
|
|
59
|
+
INSTRUMENT = {
|
|
60
|
+
"instrument_name": "instrument",
|
|
61
|
+
"instrument_model": "instrument_model",
|
|
62
|
+
"instrument_app_version": "software",
|
|
63
|
+
"instrument_id": "andi_instrument_id",
|
|
64
|
+
"instrument_mfr": "andi_instrument_mfr",
|
|
65
|
+
"instrument_serial_no": "andi_instrument_serial_no",
|
|
66
|
+
"instrument_sw_version": "andi_instrument_sw_version",
|
|
67
|
+
"instrument_fw_version": "andi_instrument_fw_version",
|
|
68
|
+
"instrument_os_version": "andi_instrument_os_version",
|
|
69
|
+
"instrument_comments": "andi_instrument_comments",
|
|
70
|
+
}
|
|
71
|
+
KNOWN = {*PER_SCAN, *PER_POINT, *INSTRUMENT, "error_log"}
|
|
72
|
+
#: Prefixes of global attributes that go to the MS metadata.
|
|
73
|
+
MS_ATTRIBUTES = ("test_", "raw_data_")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _strings(f: nc.File, name: str) -> list[str]:
|
|
77
|
+
if name not in f.variables:
|
|
78
|
+
return []
|
|
79
|
+
return [s for s in nc.strings(f.read(name)) if s]
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _per_scan(f: nc.File, name: str, n_scans: int, file: str) -> np.ndarray | None:
|
|
83
|
+
"""Read a per-scan variable (`None` when absent); unwritten scans raise."""
|
|
84
|
+
if name not in f.variables:
|
|
85
|
+
return None
|
|
86
|
+
variable = f.variables[name]
|
|
87
|
+
data = f.read(name)
|
|
88
|
+
if data.shape != (n_scans,):
|
|
89
|
+
raise CorruptFile(f"{file}: {name} has shape {data.shape}, not ({n_scans},).")
|
|
90
|
+
unwritten = np.flatnonzero(is_fill(data, variable.attributes, variable.nc_type))
|
|
91
|
+
if len(unwritten):
|
|
92
|
+
raise CorruptFile(
|
|
93
|
+
f"{file}: {name} is not written from scan index {unwritten[0]} on "
|
|
94
|
+
f"({len(unwritten)} of {n_scans} scans hold the netCDF fill value); "
|
|
95
|
+
"the export did not finish."
|
|
96
|
+
)
|
|
97
|
+
return data
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _peak_offset(
|
|
101
|
+
scan_index: np.ndarray | None, point_count: np.ndarray | None, file: str
|
|
102
|
+
) -> np.ndarray:
|
|
103
|
+
if point_count is None and scan_index is None:
|
|
104
|
+
raise UnsupportedVariant(f"{file}: neither scan_index nor point_count.")
|
|
105
|
+
if point_count is not None:
|
|
106
|
+
counts = point_count.astype(np.int64)
|
|
107
|
+
if np.any(counts < 0):
|
|
108
|
+
raise CorruptFile(f"{file}: negative point_count.")
|
|
109
|
+
offset = np.zeros(len(counts) + 1, np.int64)
|
|
110
|
+
np.cumsum(counts, out=offset[1:])
|
|
111
|
+
if scan_index is not None and not np.array_equal(scan_index, offset[:-1]):
|
|
112
|
+
bad = int(np.flatnonzero(scan_index != offset[:-1])[0])
|
|
113
|
+
raise CorruptFile(
|
|
114
|
+
f"{file}: scan_index and point_count disagree from scan index {bad} on."
|
|
115
|
+
)
|
|
116
|
+
return offset
|
|
117
|
+
index = scan_index.astype(np.int64)
|
|
118
|
+
if len(index) and (index[0] != 0 or np.any(np.diff(index) < 0)):
|
|
119
|
+
raise CorruptFile(f"{file}: scan_index does not start at 0 and increase.")
|
|
120
|
+
return np.append(index, -1) # the total is filled in from the point dimension
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _ms_metadata(f: nc.File, metadata: dict) -> None:
|
|
124
|
+
for variable, key in INSTRUMENT.items():
|
|
125
|
+
values = _strings(f, variable)
|
|
126
|
+
if values:
|
|
127
|
+
metadata[key] = values[0] if len(values) == 1 else values
|
|
128
|
+
errors = _strings(f, "error_log")
|
|
129
|
+
if errors:
|
|
130
|
+
metadata["andi_error_log"] = errors
|
|
131
|
+
attributes = f.attributes
|
|
132
|
+
mode = json_value(attributes.get("test_ionization_mode"))
|
|
133
|
+
polarity = json_value(attributes.get("test_ionization_polarity"))
|
|
134
|
+
for key, value in attribute_metadata(
|
|
135
|
+
{k: v for k, v in attributes.items() if k.startswith(MS_ATTRIBUTES)}
|
|
136
|
+
).items():
|
|
137
|
+
metadata[key] = value
|
|
138
|
+
if mode in IONISATION:
|
|
139
|
+
metadata["ionisation"] = IONISATION[mode]
|
|
140
|
+
del metadata["andi_test_ionization_mode"]
|
|
141
|
+
if polarity in POLARITY:
|
|
142
|
+
metadata["polarity"] = POLARITY[polarity]
|
|
143
|
+
del metadata["andi_test_ionization_polarity"]
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def decode_andi_ms(f: nc.File, file: str = "andi") -> tuple[MSData, Injection]:
|
|
147
|
+
"""Decode an open ANDI-MS file.
|
|
148
|
+
|
|
149
|
+
Parameters
|
|
150
|
+
----------
|
|
151
|
+
f : chromparser._netcdf3.File
|
|
152
|
+
The open file.
|
|
153
|
+
file : str
|
|
154
|
+
File name for messages.
|
|
155
|
+
|
|
156
|
+
Returns
|
|
157
|
+
-------
|
|
158
|
+
MSData, Injection
|
|
159
|
+
The MS run (named ``"MS"``) and its injection (without the MS run).
|
|
160
|
+
|
|
161
|
+
Raises
|
|
162
|
+
------
|
|
163
|
+
UnsupportedVariant
|
|
164
|
+
For profile or library spectra, SIM, flagged peaks, per-peak times or
|
|
165
|
+
variables on the data dimensions that the reader does not know.
|
|
166
|
+
CorruptFile
|
|
167
|
+
For inconsistent or unfinished files.
|
|
168
|
+
"""
|
|
169
|
+
attributes = f.attributes
|
|
170
|
+
ms_metadata: dict = {}
|
|
171
|
+
experiment = json_value(attributes.get("experiment_type"))
|
|
172
|
+
if experiment is None:
|
|
173
|
+
warn(ms_metadata, f"{file}: no experiment_type; centroided spectra assumed.")
|
|
174
|
+
elif experiment != CENTROID:
|
|
175
|
+
raise UnsupportedVariant(
|
|
176
|
+
f"{file}: experiment_type {experiment!r} (only {CENTROID!r} is read)."
|
|
177
|
+
)
|
|
178
|
+
scan_function = json_value(attributes.get("test_scan_function"))
|
|
179
|
+
if scan_function not in (None, MASS_SCAN):
|
|
180
|
+
raise UnsupportedVariant(
|
|
181
|
+
f"{file}: test_scan_function {scan_function!r} (only {MASS_SCAN!r} is read)."
|
|
182
|
+
)
|
|
183
|
+
for name in ("scan_acquisition_time", "intensity_values"):
|
|
184
|
+
if name not in f.variables:
|
|
185
|
+
raise UnsupportedVariant(f"{file}: ANDI-MS file without {name}.")
|
|
186
|
+
scan_dim = f.variables["scan_acquisition_time"].dimensions
|
|
187
|
+
point_dim = f.variables[KEY].dimensions
|
|
188
|
+
if len(scan_dim) != 1 or len(point_dim) != 1:
|
|
189
|
+
raise UnsupportedVariant(
|
|
190
|
+
f"{file}: scan_acquisition_time or mass_values is not 1D."
|
|
191
|
+
)
|
|
192
|
+
check_variables(f, KNOWN, {scan_dim[0], point_dim[0]}, ms_metadata, file)
|
|
193
|
+
|
|
194
|
+
n_scans = f.variables["scan_acquisition_time"].shape[0]
|
|
195
|
+
seconds = _per_scan(f, "scan_acquisition_time", n_scans, file)
|
|
196
|
+
unit = json_value(f.variables["scan_acquisition_time"].attributes.get("units"))
|
|
197
|
+
if unit not in (None, "Seconds", "seconds", "s"):
|
|
198
|
+
raise UnsupportedVariant(f"{file}: scan_acquisition_time in {unit!r}.")
|
|
199
|
+
scan_index = _per_scan(f, "scan_index", n_scans, file)
|
|
200
|
+
point_count = _per_scan(f, "point_count", n_scans, file)
|
|
201
|
+
offset = _peak_offset(scan_index, point_count, file)
|
|
202
|
+
flag_count = _per_scan(f, "flag_count", n_scans, file)
|
|
203
|
+
if flag_count is not None and np.any(flag_count):
|
|
204
|
+
raise UnsupportedVariant(
|
|
205
|
+
f"{file}: {int(np.sum(flag_count, dtype=np.int64))} flagged peaks; "
|
|
206
|
+
"the ANDI flag encoding is not supported yet (no sample file)."
|
|
207
|
+
)
|
|
208
|
+
|
|
209
|
+
variable = f.variables[KEY]
|
|
210
|
+
numrecs = None
|
|
211
|
+
stored = variable.shape[0]
|
|
212
|
+
if offset[-1] < 0:
|
|
213
|
+
offset[-1] = stored
|
|
214
|
+
if variable.is_record and f.numrecs == 0 and offset[-1] > 0:
|
|
215
|
+
if f.records_in_file() < offset[-1]:
|
|
216
|
+
raise CorruptFile(
|
|
217
|
+
f"{file}: the header says 0 records and the file holds fewer "
|
|
218
|
+
"peaks than point_count says."
|
|
219
|
+
)
|
|
220
|
+
warn(
|
|
221
|
+
ms_metadata,
|
|
222
|
+
f"{file}: the header says 0 records (the file was not closed); "
|
|
223
|
+
f"read the {offset[-1]} peaks point_count says.",
|
|
224
|
+
)
|
|
225
|
+
numrecs = stored = int(offset[-1])
|
|
226
|
+
if stored != offset[-1]:
|
|
227
|
+
raise CorruptFile(
|
|
228
|
+
f"{file}: {stored} peaks stored, point_count adds up to {offset[-1]}."
|
|
229
|
+
)
|
|
230
|
+
if offset[-1] < (offset[-2] if n_scans else 0):
|
|
231
|
+
raise CorruptFile(f"{file}: scan_index points past the peaks.")
|
|
232
|
+
|
|
233
|
+
mz = apply_scale(f, KEY, f.read(KEY, numrecs), ms_metadata, "mass")
|
|
234
|
+
intensity = apply_scale(
|
|
235
|
+
f,
|
|
236
|
+
"intensity_values",
|
|
237
|
+
f.read("intensity_values", numrecs),
|
|
238
|
+
ms_metadata,
|
|
239
|
+
"intensity",
|
|
240
|
+
)
|
|
241
|
+
if "time_values" in f.variables:
|
|
242
|
+
times = f.variables["time_values"]
|
|
243
|
+
data = f.read("time_values", numrecs)
|
|
244
|
+
if not np.all(is_fill(data, times.attributes, times.nc_type)):
|
|
245
|
+
raise UnsupportedVariant(f"{file}: per-peak time_values are not supported.")
|
|
246
|
+
|
|
247
|
+
computed = []
|
|
248
|
+
scan_number = None
|
|
249
|
+
numbers = f.variables.get("actual_scan_number")
|
|
250
|
+
if numbers is not None:
|
|
251
|
+
data = f.read("actual_scan_number")
|
|
252
|
+
usable = not np.any(is_fill(data, numbers.attributes, numbers.nc_type))
|
|
253
|
+
if usable and len(data) > 1 and np.any(np.diff(data.astype(np.int64)) <= 0):
|
|
254
|
+
usable = False
|
|
255
|
+
if usable:
|
|
256
|
+
scan_number = data.astype(np.int32)
|
|
257
|
+
else:
|
|
258
|
+
warn(
|
|
259
|
+
ms_metadata,
|
|
260
|
+
f"{file}: actual_scan_number is not usable; scans numbered from 1.",
|
|
261
|
+
)
|
|
262
|
+
if scan_number is None:
|
|
263
|
+
scan_number = np.arange(1, n_scans + 1, dtype=np.int32)
|
|
264
|
+
computed.append("scan_number")
|
|
265
|
+
|
|
266
|
+
counts = np.diff(offset)
|
|
267
|
+
sums = np.bincount(
|
|
268
|
+
np.repeat(np.arange(n_scans), counts),
|
|
269
|
+
weights=intensity.astype(np.float64),
|
|
270
|
+
minlength=n_scans,
|
|
271
|
+
)
|
|
272
|
+
tic = _per_scan(f, "total_intensity", n_scans, file)
|
|
273
|
+
if tic is None:
|
|
274
|
+
tic = sums
|
|
275
|
+
computed.append("tic")
|
|
276
|
+
else:
|
|
277
|
+
tic = tic.astype(np.float64)
|
|
278
|
+
tolerance = 0 if intensity.dtype.kind in "iu" else 1e-6
|
|
279
|
+
bad = int(np.count_nonzero(~np.isclose(sums, tic, rtol=tolerance, atol=0)))
|
|
280
|
+
if bad:
|
|
281
|
+
warn(ms_metadata, f"{file}: {bad} scans do not add up to total_intensity.")
|
|
282
|
+
if computed:
|
|
283
|
+
ms_metadata["computed"] = computed
|
|
284
|
+
|
|
285
|
+
# a scan range, if every scan has the same one; a minimum of 0 is taken
|
|
286
|
+
# as "not recorded" (all scans of example_ms.cdf say 0-500, its peaks
|
|
287
|
+
# start at m/z 44.95)
|
|
288
|
+
ranges = [
|
|
289
|
+
f.read(n) for n in ("mass_range_min", "mass_range_max") if n in f.variables
|
|
290
|
+
]
|
|
291
|
+
if len(ranges) == 2 and n_scans:
|
|
292
|
+
low, high = ranges
|
|
293
|
+
if (
|
|
294
|
+
low.shape == high.shape == (n_scans,)
|
|
295
|
+
and np.all(low == low[0])
|
|
296
|
+
and np.all(high == high[0])
|
|
297
|
+
and 0 < low[0] < high[0] < nc.FILL[6]
|
|
298
|
+
):
|
|
299
|
+
ms_metadata["mass_range_min"] = float(low[0])
|
|
300
|
+
ms_metadata["mass_range_max"] = float(high[0])
|
|
301
|
+
_ms_metadata(f, ms_metadata)
|
|
302
|
+
|
|
303
|
+
ms = MSData(
|
|
304
|
+
name="MS",
|
|
305
|
+
scan_number=scan_number,
|
|
306
|
+
retention_time=seconds.astype(np.float64) / 60.0,
|
|
307
|
+
tic=tic,
|
|
308
|
+
base_peak_mz=None,
|
|
309
|
+
base_peak_intensity=None,
|
|
310
|
+
filters=None,
|
|
311
|
+
mz=mz,
|
|
312
|
+
intensity=intensity,
|
|
313
|
+
peak_offset=offset,
|
|
314
|
+
metadata=ms_metadata,
|
|
315
|
+
spectrum_type="centroid",
|
|
316
|
+
)
|
|
317
|
+
injection = injection_of(
|
|
318
|
+
f,
|
|
319
|
+
file,
|
|
320
|
+
"experiment_date_time_stamp",
|
|
321
|
+
lambda k: k == "experiment_type" or k.startswith(MS_ATTRIBUTES),
|
|
322
|
+
)
|
|
323
|
+
return ms, injection
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def read_andi_ms(path: str | Path) -> list[Injection]:
|
|
327
|
+
"""Read an ANDI-MS netCDF file (centroided GC-MS run).
|
|
328
|
+
|
|
329
|
+
Parameters
|
|
330
|
+
----------
|
|
331
|
+
path : str or Path
|
|
332
|
+
The `.cdf` file.
|
|
333
|
+
|
|
334
|
+
Returns
|
|
335
|
+
-------
|
|
336
|
+
list of Injection
|
|
337
|
+
One injection with one MS run named ``"MS"``. `inject_time` is
|
|
338
|
+
the ``experiment_date_time_stamp`` (aware when it has an offset).
|
|
339
|
+
|
|
340
|
+
Raises
|
|
341
|
+
------
|
|
342
|
+
UnsupportedFormat
|
|
343
|
+
If the file is not netCDF or has no ``mass_values``.
|
|
344
|
+
UnsupportedVariant
|
|
345
|
+
For netCDF-4 files and the variants `decode_andi_ms` refuses.
|
|
346
|
+
CorruptFile
|
|
347
|
+
For inconsistent or unfinished files.
|
|
348
|
+
"""
|
|
349
|
+
path = Path(path)
|
|
350
|
+
with open_andi(path, KEY, "ANDI-MS") as f:
|
|
351
|
+
ms, injection = decode_andi_ms(f, path.name)
|
|
352
|
+
injection.ms["MS"] = ms
|
|
353
|
+
return [injection]
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def sniff_andi_ms(path: str | Path) -> bool:
|
|
357
|
+
"""Whether `path` is a netCDF file with a ``mass_values`` variable."""
|
|
358
|
+
return has_variable(path, KEY)
|
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
r"""Write injections as ANDI netCDF files (classic format, ``CDF\x01``).
|
|
2
|
+
|
|
3
|
+
One ANDI-MS file per `MSData` and one ANDI chromatography (AIA) file per
|
|
4
|
+
`Signal`, so that tools such as AMDIS, NIST MS Search, MZmine, OpenChrom,
|
|
5
|
+
MS-DIAL and XCMS can open data from any format chromparser reads. ANDI is an
|
|
6
|
+
export: what it does not carry is listed in `docs/formats/andi.md`.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import re
|
|
12
|
+
from collections import Counter
|
|
13
|
+
from collections.abc import Iterable
|
|
14
|
+
from datetime import datetime, timezone
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
import numpy as np
|
|
19
|
+
|
|
20
|
+
from ... import _netcdf3 as nc
|
|
21
|
+
from ...model import Injection, MSData, Signal
|
|
22
|
+
from .common import format_timestamp
|
|
23
|
+
from .ms import CENTROID, INSTRUMENT, IONISATION, MASS_SCAN, POLARITY
|
|
24
|
+
|
|
25
|
+
#: Values the template uses for "not recorded" in the per-scan variables.
|
|
26
|
+
NOT_RECORDED = -9999
|
|
27
|
+
#: Template attributes; the revisions are those of the vendor exports we have.
|
|
28
|
+
MS_TEMPLATE = {
|
|
29
|
+
"dataset_completeness": "C1+C2",
|
|
30
|
+
"ms_template_revision": "1.0.1",
|
|
31
|
+
"netcdf_revision": "2.3.2",
|
|
32
|
+
"languages": "English",
|
|
33
|
+
}
|
|
34
|
+
CHROM_TEMPLATE = {
|
|
35
|
+
"dataset_completeness": "C1+C2",
|
|
36
|
+
"aia_template_revision": "1.0",
|
|
37
|
+
"netcdf_revision": "2.3.2",
|
|
38
|
+
"languages": "English",
|
|
39
|
+
}
|
|
40
|
+
_FORMAT_NAME = {1: "Byte", 3: "Short", 4: "Long", 5: "Float", 6: "Double"}
|
|
41
|
+
_INT32 = np.iinfo(np.int32)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def netcdf_values(data: np.ndarray) -> np.ndarray:
|
|
45
|
+
"""Convert an array to a netCDF-3 type, keeping its dtype where possible.
|
|
46
|
+
|
|
47
|
+
netCDF-3 has no unsigned or 64-bit integers: uint8 becomes int16, uint16
|
|
48
|
+
int32, and wider integers int32 when all values fit, float64 otherwise.
|
|
49
|
+
float16 becomes float32.
|
|
50
|
+
"""
|
|
51
|
+
data = np.asarray(data)
|
|
52
|
+
kind, size = data.dtype.kind, data.dtype.itemsize
|
|
53
|
+
if kind == "f":
|
|
54
|
+
return data.astype(np.float32 if size <= 4 else np.float64, copy=False)
|
|
55
|
+
if kind == "b":
|
|
56
|
+
return data.astype(np.int8)
|
|
57
|
+
if kind not in "iu":
|
|
58
|
+
raise ValueError(f"Cannot write {data.dtype} data to netCDF-3.")
|
|
59
|
+
if kind == "i" and size <= 4:
|
|
60
|
+
return data
|
|
61
|
+
if kind == "u" and size <= 2:
|
|
62
|
+
return data.astype(np.int16 if size == 1 else np.int32)
|
|
63
|
+
if not data.size or (data.min() >= _INT32.min and data.max() <= _INT32.max):
|
|
64
|
+
return data.astype(np.int32)
|
|
65
|
+
return data.astype(np.float64)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _safe(text: str) -> str:
|
|
69
|
+
return re.sub(r"[^A-Za-z0-9._-]+", "_", text).strip("_") or "unnamed"
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _stem(injection: Injection) -> str:
|
|
73
|
+
source = injection.metadata.get("source_file")
|
|
74
|
+
if isinstance(source, str) and source:
|
|
75
|
+
return _safe(Path(source).stem)
|
|
76
|
+
return _safe(injection.name or "injection")
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _common(injection: Injection) -> dict[str, Any]:
|
|
80
|
+
metadata = injection.metadata
|
|
81
|
+
return {
|
|
82
|
+
"experiment_title": injection.name or None,
|
|
83
|
+
"sample_name": _text(metadata.get("sample_name")),
|
|
84
|
+
"operator_name": _text(metadata.get("operator")),
|
|
85
|
+
"source_file_reference": _text(metadata.get("source_file")),
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _stamp(when: datetime | None) -> str | None:
|
|
90
|
+
"""Format `when` (whole seconds) as ANDI timestamp; now (UTC) for `None`."""
|
|
91
|
+
if when is None:
|
|
92
|
+
when = datetime.now(timezone.utc)
|
|
93
|
+
return format_timestamp(when.replace(microsecond=0))
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _injected(injection: Injection, start: datetime | None) -> str | None:
|
|
97
|
+
"""Stamp the injection time, or the acquisition start (UTC) if unknown."""
|
|
98
|
+
when = injection.inject_time or start
|
|
99
|
+
return _stamp(when) if when else None
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _text(value: Any) -> str | None:
|
|
103
|
+
return value if isinstance(value, str) and value else None
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _lookup(key: str, *metadata: dict) -> Any:
|
|
107
|
+
return next((m[key] for m in metadata if key in m), None)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _ms_file(path: Path, injection: Injection, ms: MSData) -> None:
|
|
111
|
+
n_scans, n_peaks = len(ms), int(ms.peak_offset[-1])
|
|
112
|
+
if n_peaks > _INT32.max:
|
|
113
|
+
raise ValueError(f"{ms.name}: {n_peaks} peaks exceed ANDI's int32 scan_index.")
|
|
114
|
+
attributes = dict(MS_TEMPLATE)
|
|
115
|
+
attributes["netcdf_file_date_time_stamp"] = _stamp(None)
|
|
116
|
+
attributes["experiment_date_time_stamp"] = _injected(
|
|
117
|
+
injection, ms.acquisition_start
|
|
118
|
+
)
|
|
119
|
+
attributes.update(_common(injection))
|
|
120
|
+
mz = netcdf_values(ms.mz)
|
|
121
|
+
intensity = netcdf_values(ms.intensity)
|
|
122
|
+
mode = _lookup("ionisation", ms.metadata)
|
|
123
|
+
polarity = _lookup("polarity", ms.metadata)
|
|
124
|
+
attributes.update(
|
|
125
|
+
{
|
|
126
|
+
"experiment_type": (
|
|
127
|
+
CENTROID
|
|
128
|
+
if ms.spectrum_type == "centroid"
|
|
129
|
+
else "Continuum Mass Spectrum"
|
|
130
|
+
),
|
|
131
|
+
"test_ionization_mode": {v: k for k, v in IONISATION.items()}.get(
|
|
132
|
+
mode, _text(ms.metadata.get("andi_test_ionization_mode"))
|
|
133
|
+
),
|
|
134
|
+
"test_ionization_polarity": {v: k for k, v in POLARITY.items()}.get(
|
|
135
|
+
polarity, _text(ms.metadata.get("andi_test_ionization_polarity"))
|
|
136
|
+
),
|
|
137
|
+
"test_scan_function": MASS_SCAN,
|
|
138
|
+
"raw_data_mass_format": _FORMAT_NAME[nc.nc_type_of(mz.dtype)],
|
|
139
|
+
"raw_data_time_format": "Double",
|
|
140
|
+
"raw_data_intensity_format": _FORMAT_NAME[nc.nc_type_of(intensity.dtype)],
|
|
141
|
+
}
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
counts = np.diff(ms.peak_offset)
|
|
145
|
+
mass_range = {}
|
|
146
|
+
for side, reduce in (("min", np.minimum), ("max", np.maximum)):
|
|
147
|
+
stored = ms.metadata.get(f"mass_range_{side}")
|
|
148
|
+
if isinstance(stored, (int, float)) and not isinstance(stored, bool):
|
|
149
|
+
mass_range[side] = np.full(n_scans, float(stored))
|
|
150
|
+
else: # the observed range of each scan, 0 for empty scans
|
|
151
|
+
mass_range[side] = np.zeros(n_scans)
|
|
152
|
+
filled = counts > 0
|
|
153
|
+
if filled.any():
|
|
154
|
+
starts = ms.peak_offset[:-1][filled]
|
|
155
|
+
mass_range[side][filled] = reduce.reduceat(
|
|
156
|
+
mz.astype(np.float64), starts
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
scan = ("scan_number",)
|
|
160
|
+
not_recorded = np.full(n_scans, float(NOT_RECORDED))
|
|
161
|
+
variables = {
|
|
162
|
+
"error_log": nc.NewVariable(
|
|
163
|
+
("error_number", "_64_byte_string"), nc.char_array([""], 64)
|
|
164
|
+
),
|
|
165
|
+
"a_d_sampling_rate": nc.NewVariable(scan, not_recorded),
|
|
166
|
+
"a_d_coaddition_factor": nc.NewVariable(
|
|
167
|
+
scan, np.full(n_scans, NOT_RECORDED, np.int16)
|
|
168
|
+
),
|
|
169
|
+
"scan_acquisition_time": nc.NewVariable(
|
|
170
|
+
scan, np.asarray(ms.retention_time, np.float64) * 60.0, {"units": "Seconds"}
|
|
171
|
+
),
|
|
172
|
+
"scan_duration": nc.NewVariable(scan, not_recorded),
|
|
173
|
+
"inter_scan_time": nc.NewVariable(scan, not_recorded),
|
|
174
|
+
"resolution": nc.NewVariable(scan, not_recorded),
|
|
175
|
+
"actual_scan_number": nc.NewVariable(
|
|
176
|
+
scan, np.asarray(ms.scan_number, np.int32)
|
|
177
|
+
),
|
|
178
|
+
"total_intensity": nc.NewVariable(
|
|
179
|
+
scan, np.asarray(ms.tic, np.float64), {"units": "Arbitrary Intensity Units"}
|
|
180
|
+
),
|
|
181
|
+
"mass_range_min": nc.NewVariable(scan, mass_range["min"]),
|
|
182
|
+
"mass_range_max": nc.NewVariable(scan, mass_range["max"]),
|
|
183
|
+
"time_range_min": nc.NewVariable(scan, not_recorded),
|
|
184
|
+
"time_range_max": nc.NewVariable(scan, not_recorded),
|
|
185
|
+
"scan_index": nc.NewVariable(scan, ms.peak_offset[:-1].astype(np.int32)),
|
|
186
|
+
"point_count": nc.NewVariable(scan, counts.astype(np.int32)),
|
|
187
|
+
"flag_count": nc.NewVariable(scan, np.zeros(n_scans, np.int32)),
|
|
188
|
+
"mass_values": nc.NewVariable(("point_number",), mz, {"units": "M/Z"}),
|
|
189
|
+
"intensity_values": nc.NewVariable(
|
|
190
|
+
("point_number",), intensity, {"units": "Arbitrary Intensity Units"}
|
|
191
|
+
),
|
|
192
|
+
}
|
|
193
|
+
for variable, key in INSTRUMENT.items():
|
|
194
|
+
value = _text(_lookup(key, ms.metadata, injection.metadata)) or ""
|
|
195
|
+
variables[variable] = nc.NewVariable(
|
|
196
|
+
("instrument_number", "_32_byte_string"), nc.char_array([value], 32)
|
|
197
|
+
)
|
|
198
|
+
dimensions = {
|
|
199
|
+
"_32_byte_string": 32,
|
|
200
|
+
"_64_byte_string": 64,
|
|
201
|
+
"error_number": 1,
|
|
202
|
+
"scan_number": n_scans,
|
|
203
|
+
"point_number": n_peaks,
|
|
204
|
+
"instrument_number": 1,
|
|
205
|
+
}
|
|
206
|
+
nc.write(path, dimensions, attributes, variables)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _chrom_file(path: Path, injection: Injection, signal: Signal) -> None:
|
|
210
|
+
values = netcdf_values(signal.values)
|
|
211
|
+
n = len(values)
|
|
212
|
+
interval = float(signal.sampling_interval)
|
|
213
|
+
delay = float(signal.time[0]) * 60.0 if n else 0.0
|
|
214
|
+
attributes = dict(CHROM_TEMPLATE)
|
|
215
|
+
attributes["dataset_date_time_stamp"] = _stamp(None)
|
|
216
|
+
attributes["injection_date_time_stamp"] = _injected(
|
|
217
|
+
injection, signal.acquisition_start
|
|
218
|
+
)
|
|
219
|
+
attributes.update(_common(injection))
|
|
220
|
+
attributes.update(
|
|
221
|
+
{
|
|
222
|
+
"detector_name": signal.channel or None,
|
|
223
|
+
"detector_unit": signal.unit,
|
|
224
|
+
"retention_unit": "Seconds",
|
|
225
|
+
}
|
|
226
|
+
)
|
|
227
|
+
finite = values[np.isfinite(values)] if values.dtype.kind == "f" else values
|
|
228
|
+
variables = {
|
|
229
|
+
"actual_sampling_interval": nc.NewVariable((), np.float64(interval)),
|
|
230
|
+
"actual_delay_time": nc.NewVariable((), np.float64(delay)),
|
|
231
|
+
"actual_run_time_length": nc.NewVariable((), np.float64(delay + n * interval)),
|
|
232
|
+
}
|
|
233
|
+
if finite.size:
|
|
234
|
+
variables["detector_maximum_value"] = nc.NewVariable(
|
|
235
|
+
(), np.float64(finite.max())
|
|
236
|
+
)
|
|
237
|
+
variables["detector_minimum_value"] = nc.NewVariable(
|
|
238
|
+
(), np.float64(finite.min())
|
|
239
|
+
)
|
|
240
|
+
variables["ordinate_values"] = nc.NewVariable(
|
|
241
|
+
("point_number",), values, {"uniform_sampling_flag": "Y"}
|
|
242
|
+
)
|
|
243
|
+
nc.write(path, {"point_number": n}, attributes, variables)
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def write_andi(
|
|
247
|
+
injections: Injection | Iterable[Injection],
|
|
248
|
+
folder: str | Path,
|
|
249
|
+
*,
|
|
250
|
+
overwrite: bool = False,
|
|
251
|
+
) -> list[Path]:
|
|
252
|
+
"""Write injections as ANDI netCDF files.
|
|
253
|
+
|
|
254
|
+
Parameters
|
|
255
|
+
----------
|
|
256
|
+
injections : Injection or iterable of Injection
|
|
257
|
+
What to write.
|
|
258
|
+
folder : str or Path
|
|
259
|
+
Output folder (created if missing).
|
|
260
|
+
overwrite : bool
|
|
261
|
+
Replace existing files.
|
|
262
|
+
|
|
263
|
+
Returns
|
|
264
|
+
-------
|
|
265
|
+
list of Path
|
|
266
|
+
One ANDI-MS file per `MSData` and one ANDI chromatography file per
|
|
267
|
+
`Signal`, named ``<stem>_<channel>.cdf``; ``<stem>`` is the stem of
|
|
268
|
+
``metadata["source_file"]`` (else the injection name), followed by the
|
|
269
|
+
injection number when several injections share it.
|
|
270
|
+
|
|
271
|
+
Raises
|
|
272
|
+
------
|
|
273
|
+
ValueError
|
|
274
|
+
If a signal is not regularly sampled (`sampling_interval` is `None`;
|
|
275
|
+
checked before anything is written), or data does not fit the
|
|
276
|
+
ANDI/netCDF classic format.
|
|
277
|
+
FileExistsError
|
|
278
|
+
If an output file exists and `overwrite` is False. Nothing is written
|
|
279
|
+
then.
|
|
280
|
+
"""
|
|
281
|
+
if isinstance(injections, Injection):
|
|
282
|
+
injections = [injections]
|
|
283
|
+
injections = list(injections)
|
|
284
|
+
folder = Path(folder)
|
|
285
|
+
stems = [_stem(injection) for injection in injections]
|
|
286
|
+
shared = Counter(stems)
|
|
287
|
+
number: Counter = Counter()
|
|
288
|
+
taken: set[str] = set()
|
|
289
|
+
jobs = []
|
|
290
|
+
|
|
291
|
+
def target(base: str, name: str) -> Path:
|
|
292
|
+
candidate, n = f"{base}_{_safe(name)}", 1
|
|
293
|
+
while candidate.lower() in taken:
|
|
294
|
+
n += 1
|
|
295
|
+
candidate = f"{base}_{_safe(name)}_{n}"
|
|
296
|
+
taken.add(candidate.lower())
|
|
297
|
+
return folder / f"{candidate}.cdf"
|
|
298
|
+
|
|
299
|
+
for injection, stem in zip(injections, stems, strict=True):
|
|
300
|
+
number[stem] += 1
|
|
301
|
+
width = len(str(shared[stem]))
|
|
302
|
+
base = f"{stem}_{number[stem]:0{width}d}" if shared[stem] > 1 else stem
|
|
303
|
+
for signal in injection.signals.values():
|
|
304
|
+
if signal.sampling_interval is None:
|
|
305
|
+
raise ValueError(
|
|
306
|
+
f"Signal {signal.channel!r} is not regularly sampled; ANDI "
|
|
307
|
+
"chromatography files need a sampling interval."
|
|
308
|
+
)
|
|
309
|
+
jobs.append((target(base, signal.channel), _chrom_file, injection, signal))
|
|
310
|
+
for ms in injection.ms.values():
|
|
311
|
+
jobs.append((target(base, ms.name), _ms_file, injection, ms))
|
|
312
|
+
existing = [str(path) for path, *_ in jobs if path.exists()]
|
|
313
|
+
if existing and not overwrite:
|
|
314
|
+
raise FileExistsError(
|
|
315
|
+
f"{', '.join(existing)} exist(s), use overwrite=True to replace."
|
|
316
|
+
)
|
|
317
|
+
folder.mkdir(parents=True, exist_ok=True)
|
|
318
|
+
for path, write, injection, data in jobs:
|
|
319
|
+
write(path, injection, data)
|
|
320
|
+
return [path for path, *_ in jobs]
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Thermo Chromeleon 7: `.cmbx` archives, raw signal files and text exports."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from .archive import extract_ms_raw, read_cmbx, sniff_cmbx
|
|
6
|
+
from .raw import decode_signal, read_chromeleon_raw, read_signal, sniff_chromeleon_raw
|
|
7
|
+
from .txt import read_chromeleon_txt, sniff_chromeleon_txt
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"decode_signal",
|
|
11
|
+
"extract_ms_raw",
|
|
12
|
+
"read_chromeleon_raw",
|
|
13
|
+
"read_chromeleon_txt",
|
|
14
|
+
"read_cmbx",
|
|
15
|
+
"read_signal",
|
|
16
|
+
"sniff_chromeleon_raw",
|
|
17
|
+
"sniff_chromeleon_txt",
|
|
18
|
+
"sniff_cmbx",
|
|
19
|
+
]
|