multigas 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- multigas/__init__.py +36 -0
- multigas/config/__init__.py +5 -0
- multigas/core/__init__.py +50 -0
- multigas/core/exceptions.py +215 -0
- multigas/core/io.py +71 -0
- multigas/core/query.py +412 -0
- multigas/core/types.py +290 -0
- multigas/data/__init__.py +18 -0
- multigas/data/loader.py +417 -0
- multigas/logging.py +170 -0
- multigas/utils/__init__.py +13 -0
- multigas/utils/cache.py +171 -0
- multigas/utils/dataframe.py +79 -0
- multigas/utils/path.py +33 -0
- multigas/utils/validation.py +156 -0
- multigas-0.1.0.dist-info/METADATA +175 -0
- multigas-0.1.0.dist-info/RECORD +18 -0
- multigas-0.1.0.dist-info/WHEEL +4 -0
multigas/core/query.py
ADDED
|
@@ -0,0 +1,412 @@
|
|
|
1
|
+
"""Fluent query interface for pandas DataFrames with a datetime index.
|
|
2
|
+
|
|
3
|
+
Defines :class:`Query`, the mixin that :class:`multigas.core.types.MultiGasData`
|
|
4
|
+
inherits from. It normalises the index to a :class:`pandas.DatetimeIndex`,
|
|
5
|
+
tracks the currently selected columns, and exposes chainable helpers for
|
|
6
|
+
column selection and null-checking.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from typing import Self
|
|
10
|
+
|
|
11
|
+
import numpy as np
|
|
12
|
+
import pandas as pd
|
|
13
|
+
|
|
14
|
+
from multigas.logging import logger
|
|
15
|
+
from multigas.utils.dataframe import get_dates, to_dateime_index
|
|
16
|
+
from multigas.utils.validation import check_columns_exist
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class Query:
|
|
20
|
+
"""Fluent column-selection and inspection wrapper around a DataFrame.
|
|
21
|
+
|
|
22
|
+
Wraps a :class:`pandas.DataFrame`, promotes ``index_col`` to a
|
|
23
|
+
:class:`pandas.DatetimeIndex`, and keeps a pristine copy in
|
|
24
|
+
:attr:`df_original` so :meth:`refresh` can restore it. Column selection
|
|
25
|
+
is stored in :attr:`selected_columns` and can be composed via
|
|
26
|
+
:meth:`select_columns` and :meth:`select_numeric_columns`.
|
|
27
|
+
|
|
28
|
+
Attributes:
|
|
29
|
+
df (pd.DataFrame): Working DataFrame with a datetime index.
|
|
30
|
+
df_original (pd.DataFrame): Untouched copy captured at construction.
|
|
31
|
+
index_col (str): Column promoted to the datetime index.
|
|
32
|
+
columns (list[str]): All column names in :attr:`df_original`.
|
|
33
|
+
selected_columns (list[str]): Currently selected column names.
|
|
34
|
+
numeric_columns (list[str]): Numeric-dtype columns in :attr:`df`.
|
|
35
|
+
start_date (pd.Timestamp): Earliest timestamp in :attr:`df`.
|
|
36
|
+
end_date (pd.Timestamp): Latest timestamp in :attr:`df`.
|
|
37
|
+
start_date_str (str): :attr:`start_date` formatted as ``YYYY-MM-DD``.
|
|
38
|
+
end_date_str (str): :attr:`end_date` formatted as ``YYYY-MM-DD``.
|
|
39
|
+
verbose (bool): Whether operations emit informational log messages.
|
|
40
|
+
|
|
41
|
+
Example:
|
|
42
|
+
>>> import pandas as pd
|
|
43
|
+
>>> df = pd.DataFrame(
|
|
44
|
+
... {"TIMESTAMP": ["2025-01-01"], "CO2": [1.2], "SO2": [0.3]}
|
|
45
|
+
... )
|
|
46
|
+
>>> q = Query(df)
|
|
47
|
+
>>> q.select_numeric_columns().selected_columns
|
|
48
|
+
['CO2', 'SO2']
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
def __init__(
|
|
52
|
+
self,
|
|
53
|
+
df: pd.DataFrame,
|
|
54
|
+
index_col: str | None = None,
|
|
55
|
+
verbose: bool = False,
|
|
56
|
+
):
|
|
57
|
+
"""Initialise Query with a DataFrame and a datetime index column.
|
|
58
|
+
|
|
59
|
+
Args:
|
|
60
|
+
df: DataFrame to wrap. Must contain ``index_col`` as a column or
|
|
61
|
+
already have a :class:`pd.DatetimeIndex`.
|
|
62
|
+
index_col: Column name to use as the datetime index. Defaults to
|
|
63
|
+
``"TIMESTAMP"``.
|
|
64
|
+
verbose: Emit informational log messages for each operation.
|
|
65
|
+
Defaults to ``False``.
|
|
66
|
+
|
|
67
|
+
Example:
|
|
68
|
+
>>> import pandas as pd
|
|
69
|
+
>>> df = pd.DataFrame({"TIMESTAMP": ["2025-01-01"], "CO2": [1.2]})
|
|
70
|
+
>>> q = Query(df)
|
|
71
|
+
>>> isinstance(q.df.index, pd.DatetimeIndex)
|
|
72
|
+
True
|
|
73
|
+
"""
|
|
74
|
+
index_col: str = index_col or "TIMESTAMP"
|
|
75
|
+
df = to_dateime_index(df, index_col)
|
|
76
|
+
df_original: pd.DataFrame = df.copy()
|
|
77
|
+
|
|
78
|
+
self.df: pd.DataFrame = df
|
|
79
|
+
self.index_col: str = index_col
|
|
80
|
+
|
|
81
|
+
self.df_original: pd.DataFrame = df_original
|
|
82
|
+
self.columns: list[str] = df_original.columns.tolist()
|
|
83
|
+
self.selected_columns: list[str] = []
|
|
84
|
+
self.numeric_columns: list[str] = (
|
|
85
|
+
df.select_dtypes(include=np.number).keys().tolist()
|
|
86
|
+
)
|
|
87
|
+
self.start_date, self.end_date, self.start_date_str, self.end_date_str = (
|
|
88
|
+
get_dates(df)
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
self.verbose: bool = verbose
|
|
92
|
+
|
|
93
|
+
@property
|
|
94
|
+
def missing_columns(self) -> list[str]:
|
|
95
|
+
"""Names of columns that contain at least one missing value.
|
|
96
|
+
|
|
97
|
+
"Missing" is defined by :meth:`column_has_missing` — ``NaN``/``pd.NA``
|
|
98
|
+
for every dtype and, for string-like dtypes, the empty string ``""``.
|
|
99
|
+
When a selection is active (:attr:`selected_columns` non-empty), only
|
|
100
|
+
selected columns are inspected; otherwise every column is checked.
|
|
101
|
+
Mirrors the scoping of :attr:`empty_columns`.
|
|
102
|
+
|
|
103
|
+
Returns:
|
|
104
|
+
list[str]: Column names that have at least one missing value.
|
|
105
|
+
|
|
106
|
+
Example:
|
|
107
|
+
>>> q.missing_columns
|
|
108
|
+
['CO2', 'H2S']
|
|
109
|
+
"""
|
|
110
|
+
columns_name = self.selected_columns or self.columns
|
|
111
|
+
return [c for c in columns_name if self.column_has_missing(c)]
|
|
112
|
+
|
|
113
|
+
@property
|
|
114
|
+
def empty_columns(self) -> list[str]:
|
|
115
|
+
"""Names of columns whose values sum to zero (all-zero or all-empty).
|
|
116
|
+
|
|
117
|
+
When a selection is active (:attr:`selected_columns` non-empty), only
|
|
118
|
+
selected columns are inspected; otherwise every column is checked.
|
|
119
|
+
|
|
120
|
+
Returns:
|
|
121
|
+
list[str]: Column names that are considered empty.
|
|
122
|
+
|
|
123
|
+
Example:
|
|
124
|
+
>>> q.empty_columns
|
|
125
|
+
['unused_channel']
|
|
126
|
+
"""
|
|
127
|
+
columns_name = self.columns
|
|
128
|
+
columns_selected: int = len(self.selected_columns)
|
|
129
|
+
empty_columns: list[str] = []
|
|
130
|
+
|
|
131
|
+
if columns_selected > 0:
|
|
132
|
+
columns_name = self.selected_columns
|
|
133
|
+
|
|
134
|
+
for column_name in columns_name:
|
|
135
|
+
if self.column_is_empty(column_name):
|
|
136
|
+
empty_columns.append(column_name)
|
|
137
|
+
|
|
138
|
+
return empty_columns
|
|
139
|
+
|
|
140
|
+
@staticmethod
|
|
141
|
+
def intersection(first_list: list[str], second_list: list[str]) -> list[str]:
|
|
142
|
+
"""Return elements of ``first_list`` that also appear in ``second_list``.
|
|
143
|
+
|
|
144
|
+
Order is taken from ``first_list``; duplicates in ``first_list`` are
|
|
145
|
+
preserved. Prefer this over :func:`set.intersection` when the caller
|
|
146
|
+
needs the original ordering.
|
|
147
|
+
|
|
148
|
+
Args:
|
|
149
|
+
first_list (list[str]): Primary list; result order matches this list.
|
|
150
|
+
second_list (list[str]): List whose membership is tested against.
|
|
151
|
+
|
|
152
|
+
Returns:
|
|
153
|
+
list[str]: Elements common to both lists, in ``first_list`` order.
|
|
154
|
+
|
|
155
|
+
Example:
|
|
156
|
+
>>> Query.intersection(["a", "b", "c"], ["b", "c", "d"])
|
|
157
|
+
['b', 'c']
|
|
158
|
+
"""
|
|
159
|
+
intersect_list: list[str] = [
|
|
160
|
+
value for value in first_list if value in second_list
|
|
161
|
+
]
|
|
162
|
+
return intersect_list
|
|
163
|
+
|
|
164
|
+
@staticmethod
|
|
165
|
+
def unique(first_list: list[str], second_list: list[str]) -> list[str]:
|
|
166
|
+
"""Return the deduplicated union of two lists.
|
|
167
|
+
|
|
168
|
+
Args:
|
|
169
|
+
first_list (list[str]): First list of names.
|
|
170
|
+
second_list (list[str]): Second list of names.
|
|
171
|
+
|
|
172
|
+
Returns:
|
|
173
|
+
list[str]: All distinct values from both inputs. Order is not
|
|
174
|
+
guaranteed because a set is used internally.
|
|
175
|
+
|
|
176
|
+
Example:
|
|
177
|
+
>>> sorted(Query.unique(["a", "b"], ["b", "c"]))
|
|
178
|
+
['a', 'b', 'c']
|
|
179
|
+
"""
|
|
180
|
+
return list(set(first_list + second_list))
|
|
181
|
+
|
|
182
|
+
def reset_selected_columns(self) -> Self:
|
|
183
|
+
"""Clear the current column selection.
|
|
184
|
+
|
|
185
|
+
Sets :attr:`selected_columns` back to an empty list. Emits a log line
|
|
186
|
+
when :attr:`verbose` is ``True``.
|
|
187
|
+
|
|
188
|
+
Returns:
|
|
189
|
+
Self: The same instance, to allow method chaining.
|
|
190
|
+
|
|
191
|
+
Example:
|
|
192
|
+
>>> q.select_columns(["CO2"]).reset_selected_columns().selected_columns
|
|
193
|
+
[]
|
|
194
|
+
"""
|
|
195
|
+
self.selected_columns: list[str] = []
|
|
196
|
+
if self.verbose:
|
|
197
|
+
logger.info("Selected columns resetted")
|
|
198
|
+
return self
|
|
199
|
+
|
|
200
|
+
def refresh(self) -> Self:
|
|
201
|
+
"""Restore :attr:`df` to the pristine :attr:`df_original` copy.
|
|
202
|
+
|
|
203
|
+
Also clears the current selection and recomputes
|
|
204
|
+
:attr:`numeric_columns` and the ``start_date`` / ``end_date`` pair.
|
|
205
|
+
|
|
206
|
+
Returns:
|
|
207
|
+
Self: The same instance, to allow method chaining.
|
|
208
|
+
|
|
209
|
+
Example:
|
|
210
|
+
>>> q.select_columns(["CO2"]).refresh().is_filtered()
|
|
211
|
+
False
|
|
212
|
+
"""
|
|
213
|
+
df = self.df_original.copy()
|
|
214
|
+
|
|
215
|
+
self.df = df
|
|
216
|
+
self.reset_selected_columns()
|
|
217
|
+
self.numeric_columns: list[str] = (
|
|
218
|
+
df.select_dtypes(include=np.number).keys().to_list()
|
|
219
|
+
)
|
|
220
|
+
self.start_date, self.end_date, self.start_date_str, self.end_date_str = (
|
|
221
|
+
get_dates(df)
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
if self.verbose:
|
|
225
|
+
logger.info("Dataframe resetted to original data")
|
|
226
|
+
|
|
227
|
+
return self
|
|
228
|
+
|
|
229
|
+
def is_filtered(self) -> bool:
|
|
230
|
+
"""Check whether :attr:`df` differs from :attr:`df_original`.
|
|
231
|
+
|
|
232
|
+
Returns:
|
|
233
|
+
bool: ``True`` when the working DataFrame no longer equals the
|
|
234
|
+
pristine copy, ``False`` otherwise.
|
|
235
|
+
|
|
236
|
+
Example:
|
|
237
|
+
>>> q.is_filtered()
|
|
238
|
+
False
|
|
239
|
+
"""
|
|
240
|
+
return False if self.df_original.equals(self.df) else True
|
|
241
|
+
|
|
242
|
+
def column_has_missing(self, column_name: str) -> bool:
|
|
243
|
+
"""Report whether a column contains any missing value.
|
|
244
|
+
|
|
245
|
+
"Missing" covers ``NaN``/``pd.NA`` for every dtype and, for
|
|
246
|
+
string-like dtypes, the empty string ``""``. The string check uses
|
|
247
|
+
:func:`pandas.api.types.is_string_dtype` so it covers legacy
|
|
248
|
+
``object`` columns and pandas 3.x's default ``StringDtype``.
|
|
249
|
+
``fillna("")`` runs first so ``pd.NA`` does not propagate through
|
|
250
|
+
the ``==`` comparison.
|
|
251
|
+
|
|
252
|
+
Args:
|
|
253
|
+
column_name (str): Name of the column to inspect.
|
|
254
|
+
|
|
255
|
+
Returns:
|
|
256
|
+
bool: ``True`` if the column has at least one NaN, ``pd.NA``, or
|
|
257
|
+
(for string-like dtypes) empty string; ``False`` otherwise.
|
|
258
|
+
|
|
259
|
+
Example:
|
|
260
|
+
>>> q.column_has_missing("CO2")
|
|
261
|
+
False
|
|
262
|
+
"""
|
|
263
|
+
check_columns_exist(column_name, self.columns)
|
|
264
|
+
col = self.df[column_name]
|
|
265
|
+
has_null = col.isna().to_numpy().any()
|
|
266
|
+
has_empty = (
|
|
267
|
+
col.fillna("").eq("").to_numpy().any()
|
|
268
|
+
if pd.api.types.is_string_dtype(col)
|
|
269
|
+
else False
|
|
270
|
+
)
|
|
271
|
+
return bool(has_null or has_empty)
|
|
272
|
+
|
|
273
|
+
def column_is_empty(self, column_name: str) -> bool:
|
|
274
|
+
"""Report whether a column has no meaningful data.
|
|
275
|
+
|
|
276
|
+
A column is treated as empty when every value is NaN, or — depending
|
|
277
|
+
on dtype — additionally when every value is ``0`` (numeric) or ``""``
|
|
278
|
+
(object). Non-numeric, non-object dtypes (e.g. datetime, boolean) are
|
|
279
|
+
empty only when every value is NaN.
|
|
280
|
+
|
|
281
|
+
Args:
|
|
282
|
+
column_name (str): Name of the column to inspect.
|
|
283
|
+
|
|
284
|
+
Returns:
|
|
285
|
+
bool: ``True`` when the column is considered empty.
|
|
286
|
+
|
|
287
|
+
Example:
|
|
288
|
+
>>> q.column_is_empty("unused_channel")
|
|
289
|
+
True
|
|
290
|
+
"""
|
|
291
|
+
check_columns_exist(column_name, self.columns)
|
|
292
|
+
col = self.df[column_name]
|
|
293
|
+
|
|
294
|
+
if col.isna().to_numpy().all():
|
|
295
|
+
return True
|
|
296
|
+
|
|
297
|
+
if pd.api.types.is_numeric_dtype(col):
|
|
298
|
+
return bool(col.fillna(0).eq(0).to_numpy().all())
|
|
299
|
+
|
|
300
|
+
if pd.api.types.is_string_dtype(col):
|
|
301
|
+
return bool(col.fillna("").eq("").to_numpy().all())
|
|
302
|
+
|
|
303
|
+
return False
|
|
304
|
+
|
|
305
|
+
def select_columns(
|
|
306
|
+
self,
|
|
307
|
+
column_names: str | list[str],
|
|
308
|
+
numeric_column_only: bool = False,
|
|
309
|
+
validate: bool = True,
|
|
310
|
+
) -> Self:
|
|
311
|
+
"""Select one or more columns by name.
|
|
312
|
+
|
|
313
|
+
Sets :attr:`selected_columns` to the given names. Optionally restricts
|
|
314
|
+
the selection to numeric columns only.
|
|
315
|
+
|
|
316
|
+
Args:
|
|
317
|
+
column_names: A single column name or a list of column names to
|
|
318
|
+
select.
|
|
319
|
+
numeric_column_only: If ``True``, delegate to
|
|
320
|
+
:meth:`select_numeric_columns` so that non-numeric names are
|
|
321
|
+
dropped from the selection. A log line names the dropped
|
|
322
|
+
columns when :attr:`verbose` is ``True``. Defaults to
|
|
323
|
+
``False``.
|
|
324
|
+
validate: If ``True``, raise :class:`ColumnError` when any name is
|
|
325
|
+
not present in :attr:`columns`. The exception lists every
|
|
326
|
+
missing name in one message. Defaults to ``True``.
|
|
327
|
+
|
|
328
|
+
Returns:
|
|
329
|
+
Self for method chaining.
|
|
330
|
+
|
|
331
|
+
Raises:
|
|
332
|
+
ColumnError: When ``validate=True`` and one or more names are not
|
|
333
|
+
in :attr:`columns`.
|
|
334
|
+
|
|
335
|
+
Example:
|
|
336
|
+
>>> q.select_columns(["CO2", "SO2"]).df.columns.tolist()
|
|
337
|
+
['CO2', 'SO2']
|
|
338
|
+
"""
|
|
339
|
+
if isinstance(column_names, str):
|
|
340
|
+
column_names = [column_names]
|
|
341
|
+
|
|
342
|
+
if validate:
|
|
343
|
+
check_columns_exist(column_names, self.columns)
|
|
344
|
+
|
|
345
|
+
if numeric_column_only:
|
|
346
|
+
return self.select_numeric_columns(column_names, validate=False)
|
|
347
|
+
|
|
348
|
+
self.selected_columns = list(column_names)
|
|
349
|
+
return self
|
|
350
|
+
|
|
351
|
+
def select_numeric_columns(
|
|
352
|
+
self,
|
|
353
|
+
column_names: str | list[str] | None = None,
|
|
354
|
+
validate: bool = True,
|
|
355
|
+
) -> Self:
|
|
356
|
+
"""Select numeric columns, optionally filtered to a given subset.
|
|
357
|
+
|
|
358
|
+
When ``column_names`` is ``None``, all numeric columns in the DataFrame
|
|
359
|
+
are selected. Otherwise, the selection is the intersection of
|
|
360
|
+
``column_names`` and :attr:`numeric_columns`; non-numeric names are
|
|
361
|
+
dropped, and a log line names them when :attr:`verbose` is ``True``.
|
|
362
|
+
|
|
363
|
+
Args:
|
|
364
|
+
column_names: Column name(s) to filter. If ``None``, all numeric
|
|
365
|
+
columns are selected. Defaults to ``None``.
|
|
366
|
+
validate: If ``True``, raise :class:`ColumnError` when any name is
|
|
367
|
+
not present in :attr:`columns` (non-numeric names that *do*
|
|
368
|
+
exist are silently dropped by the intersection instead).
|
|
369
|
+
Defaults to ``True``.
|
|
370
|
+
|
|
371
|
+
Returns:
|
|
372
|
+
Self for method chaining.
|
|
373
|
+
|
|
374
|
+
Raises:
|
|
375
|
+
ColumnError: When ``validate=True`` and one or more names are not
|
|
376
|
+
in :attr:`columns`.
|
|
377
|
+
|
|
378
|
+
Example:
|
|
379
|
+
>>> q.select_numeric_columns().selected_columns
|
|
380
|
+
['CO2', 'SO2', 'H2S']
|
|
381
|
+
"""
|
|
382
|
+
if column_names is None:
|
|
383
|
+
self.selected_columns = list(self.numeric_columns)
|
|
384
|
+
if self.verbose:
|
|
385
|
+
logger.info(
|
|
386
|
+
f"Selected all {len(self.numeric_columns)} numeric "
|
|
387
|
+
f"column(s): {self.numeric_columns}"
|
|
388
|
+
)
|
|
389
|
+
return self
|
|
390
|
+
|
|
391
|
+
if isinstance(column_names, str):
|
|
392
|
+
column_names = [column_names]
|
|
393
|
+
|
|
394
|
+
if validate:
|
|
395
|
+
check_columns_exist(column_names, self.columns)
|
|
396
|
+
|
|
397
|
+
numeric_selection = self.intersection(column_names, self.numeric_columns)
|
|
398
|
+
self.selected_columns = numeric_selection
|
|
399
|
+
|
|
400
|
+
if self.verbose:
|
|
401
|
+
dropped = [c for c in column_names if c not in numeric_selection]
|
|
402
|
+
if dropped:
|
|
403
|
+
logger.info(
|
|
404
|
+
f"Dropped {len(dropped)} non-numeric "
|
|
405
|
+
f"column(s) from selection: {dropped}"
|
|
406
|
+
)
|
|
407
|
+
logger.info(
|
|
408
|
+
f"Selected {len(numeric_selection)} numeric column(s) from "
|
|
409
|
+
f"{len(column_names)} requested: {numeric_selection}"
|
|
410
|
+
)
|
|
411
|
+
|
|
412
|
+
return self
|
multigas/core/types.py
ADDED
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
"""Shared type aliases, enums, and dataclasses for the multigas package.
|
|
2
|
+
|
|
3
|
+
Groups the small, dependency-light types that would otherwise be scattered
|
|
4
|
+
across the package:
|
|
5
|
+
|
|
6
|
+
- Type aliases: :data:`DateLike`, :data:`ColumnName`, :data:`Comparator`.
|
|
7
|
+
- Enums: :class:`DatasetType`, :class:`LogLevel`, :class:`SensorStatus`,
|
|
8
|
+
:class:`FileFormat`.
|
|
9
|
+
- Dataclasses: :class:`MultiGasData` (a DataFrame + provenance wrapper that
|
|
10
|
+
also inherits :class:`multigas.core.query.Query` for fluent operations).
|
|
11
|
+
- TypedDicts: :class:`DatasetMetadataDict`.
|
|
12
|
+
|
|
13
|
+
Values for :class:`DatasetType` are chosen so that they double as pandas
|
|
14
|
+
frequency aliases where applicable (``"1s"``, ``"2s"``, ``"6h"``, ``"1min"``).
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from enum import IntEnum, StrEnum, EnumMeta
|
|
18
|
+
from typing import TypedDict
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from datetime import datetime
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
|
|
23
|
+
import pandas as pd
|
|
24
|
+
|
|
25
|
+
from multigas.core.query import Query
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
DateLike = str | datetime | pd.Timestamp
|
|
29
|
+
"""Anything that :func:`pandas.to_datetime` can interpret as a single date."""
|
|
30
|
+
|
|
31
|
+
ColumnName = str
|
|
32
|
+
"""Alias for a DataFrame column name; kept explicit for readability."""
|
|
33
|
+
|
|
34
|
+
Comparator = str
|
|
35
|
+
"""Alias for a comparison operator string (e.g. ``">="``, ``"=="``)."""
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _raise_missing_value(cls: EnumMeta, value: object) -> None:
|
|
39
|
+
"""Raise a descriptive ValueError for an unrecognised enum value.
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
cls: The enum class that rejected the value.
|
|
43
|
+
value: The value that failed to match any member.
|
|
44
|
+
|
|
45
|
+
Raises:
|
|
46
|
+
ValueError: Always raised with a message listing valid values and
|
|
47
|
+
attribute-access forms.
|
|
48
|
+
|
|
49
|
+
Example:
|
|
50
|
+
>>> _raise_missing_value(DatasetType, "bad")
|
|
51
|
+
ValueError: 'bad' is not a valid DatasetType. ...
|
|
52
|
+
"""
|
|
53
|
+
values = ", ".join(f"`{m.value}`" for m in cls) # ty:ignore[unresolved-attribute]
|
|
54
|
+
attrs = ", ".join(f"`{cls.__name__}.{m.name}`" for m in cls) # ty:ignore[unresolved-attribute]
|
|
55
|
+
raise ValueError(
|
|
56
|
+
f"{value!r} is not a valid {cls.__name__}. Valid values: {values} or {attrs}"
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class DatasetType(StrEnum):
|
|
61
|
+
"""Sampling interval or acquisition mode of a multi-gas dataset.
|
|
62
|
+
|
|
63
|
+
Members whose value is a pandas frequency alias correspond to the four
|
|
64
|
+
fixed sampling rates emitted by the datalogger. The remaining members
|
|
65
|
+
(:attr:`ZERO`, :attr:`SPAN`, :attr:`WX`) tag calibration and weather
|
|
66
|
+
streams that are read side-by-side with the sample data.
|
|
67
|
+
|
|
68
|
+
Example:
|
|
69
|
+
>>> DatasetType("1min")
|
|
70
|
+
<DatasetType.ONE_MINUTE: '1min'>
|
|
71
|
+
>>> DatasetType.SIX_HOURS.value
|
|
72
|
+
'6h'
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
ONE_SECOND = "1s"
|
|
76
|
+
TWO_SECONDS = "2s"
|
|
77
|
+
SIX_HOURS = "6h"
|
|
78
|
+
ONE_MINUTE = "1min"
|
|
79
|
+
ZERO = "zero"
|
|
80
|
+
SPAN = "span"
|
|
81
|
+
WX = "wx"
|
|
82
|
+
|
|
83
|
+
@classmethod
|
|
84
|
+
def _missing_(cls, value: object) -> None:
|
|
85
|
+
"""Raise a descriptive error for unrecognised values.
|
|
86
|
+
|
|
87
|
+
Args:
|
|
88
|
+
value: The value that failed to match any member.
|
|
89
|
+
|
|
90
|
+
Raises:
|
|
91
|
+
ValueError: Always raised with a message listing valid string
|
|
92
|
+
values and attribute-access forms.
|
|
93
|
+
|
|
94
|
+
Example:
|
|
95
|
+
>>> DatasetType("bad")
|
|
96
|
+
ValueError: 'bad' is not a valid DatasetType. ...
|
|
97
|
+
"""
|
|
98
|
+
_raise_missing_value(cls, value)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@dataclass
|
|
102
|
+
class MultiGasData(Query):
|
|
103
|
+
"""A loaded DataFrame together with its provenance metadata and query capabilities.
|
|
104
|
+
|
|
105
|
+
Extends Query to expose fluent column-selection and filtering methods directly
|
|
106
|
+
on the loaded dataset.
|
|
107
|
+
|
|
108
|
+
Attributes:
|
|
109
|
+
df: The loaded (and optionally normalized) DataFrame.
|
|
110
|
+
dataset_type: The type of dataset as declared by the caller.
|
|
111
|
+
source_path: Absolute path to the original source file.
|
|
112
|
+
index_col: Column name used as the datetime index.
|
|
113
|
+
verbose: Whether to emit log messages for each operation.
|
|
114
|
+
|
|
115
|
+
Example:
|
|
116
|
+
>>> result = loader.load(path, DatasetType.ONE_MINUTE)
|
|
117
|
+
>>> result.select_numeric_columns().df.head()
|
|
118
|
+
>>> result.dataset_type
|
|
119
|
+
<DatasetType.ONE_MINUTE: '1min'>
|
|
120
|
+
"""
|
|
121
|
+
|
|
122
|
+
df: pd.DataFrame
|
|
123
|
+
dataset_type: DatasetType
|
|
124
|
+
source_path: Path
|
|
125
|
+
index_col: str = "TIMESTAMP"
|
|
126
|
+
verbose: bool = False
|
|
127
|
+
|
|
128
|
+
def __post_init__(self):
|
|
129
|
+
"""Initialise Query with the loaded DataFrame.
|
|
130
|
+
|
|
131
|
+
Example:
|
|
132
|
+
>>> result = MultiGasData(df, DatasetType.ONE_MINUTE, path)
|
|
133
|
+
>>> result.numeric_columns # populated by Query.__init__
|
|
134
|
+
"""
|
|
135
|
+
Query.__init__(self, self.df, self.index_col, self.verbose)
|
|
136
|
+
|
|
137
|
+
def __repr__(self) -> str:
|
|
138
|
+
"""Return a concise string representation of the dataset.
|
|
139
|
+
|
|
140
|
+
Returns:
|
|
141
|
+
str: A string showing dataset_type, source_path, and DataFrame shape.
|
|
142
|
+
|
|
143
|
+
Example:
|
|
144
|
+
>>> repr(result)
|
|
145
|
+
"MultiGasData(dataset_type=<DatasetType.ONE_MINUTE: '1min'>, ..., index_col='TIMESTAMP', drop_empty_columns=False, verbose=False)"
|
|
146
|
+
"""
|
|
147
|
+
return (
|
|
148
|
+
f"MultiGasData("
|
|
149
|
+
f"dataset_type={self.dataset_type!r}, "
|
|
150
|
+
f"source_path={self.source_path!r}, "
|
|
151
|
+
f"shape={self.df.shape}, "
|
|
152
|
+
f"index_col={self.index_col!r}, "
|
|
153
|
+
f"verbose={self.verbose!r})"
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
class DatasetMetadataDict(TypedDict, total=False):
|
|
158
|
+
"""Metadata extracted from a Campbell Scientific TOA5 header.
|
|
159
|
+
|
|
160
|
+
All keys are optional; only those present in the source file are populated.
|
|
161
|
+
|
|
162
|
+
Attributes:
|
|
163
|
+
station: Station name recorded by the datalogger.
|
|
164
|
+
logger_type: Datalogger model (e.g. ``"CR1000"``).
|
|
165
|
+
firmware: Firmware / OS build identifier.
|
|
166
|
+
program_name: Name of the CRBasic program producing the table.
|
|
167
|
+
file_sampling: Sampling declaration recorded in the file header.
|
|
168
|
+
serial_number: Datalogger serial number.
|
|
169
|
+
os_version: Operating-system version string reported by the logger.
|
|
170
|
+
|
|
171
|
+
Example:
|
|
172
|
+
>>> meta: DatasetMetadataDict = {"station": "SiteA", "logger_type": "CR1000"}
|
|
173
|
+
>>> meta["station"]
|
|
174
|
+
'SiteA'
|
|
175
|
+
"""
|
|
176
|
+
|
|
177
|
+
station: str
|
|
178
|
+
logger_type: str
|
|
179
|
+
firmware: str
|
|
180
|
+
program_name: str
|
|
181
|
+
file_sampling: str
|
|
182
|
+
serial_number: str
|
|
183
|
+
os_version: str
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
class LogLevel(StrEnum):
|
|
187
|
+
"""Verbosity levels understood by the package logger.
|
|
188
|
+
|
|
189
|
+
Values are lowercased strings so they can be passed straight to loguru
|
|
190
|
+
without additional coercion.
|
|
191
|
+
|
|
192
|
+
Example:
|
|
193
|
+
>>> LogLevel.INFO
|
|
194
|
+
<LogLevel.INFO: 'info'>
|
|
195
|
+
>>> LogLevel("debug").value
|
|
196
|
+
'debug'
|
|
197
|
+
"""
|
|
198
|
+
|
|
199
|
+
DEBUG = "debug"
|
|
200
|
+
INFO = "info"
|
|
201
|
+
WARN = "warn"
|
|
202
|
+
ERROR = "error"
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
class SensorStatus(IntEnum):
|
|
206
|
+
"""Campbell Scientific multi-gas sensor operational status codes.
|
|
207
|
+
|
|
208
|
+
Each member maps a numeric status code reported by the datalogger to a
|
|
209
|
+
human-readable description of the sensor's current operating mode.
|
|
210
|
+
|
|
211
|
+
Example:
|
|
212
|
+
>>> SensorStatus(-2)
|
|
213
|
+
<SensorStatus.CHEMICAL_SENSOR_OFF: -2>
|
|
214
|
+
>>> SensorStatus(1).name
|
|
215
|
+
'SAMPLE_ACQUISITION'
|
|
216
|
+
>>> SensorStatus(4).description
|
|
217
|
+
'Standart Gas Measurement for CO2 and SO2'
|
|
218
|
+
"""
|
|
219
|
+
|
|
220
|
+
CHEMICAL_SENSOR_OFF = -2
|
|
221
|
+
WARMING_UP = -1
|
|
222
|
+
ZERO = 0
|
|
223
|
+
SAMPLE_ACQUISITION = 1
|
|
224
|
+
SPAN_CO2_SO2 = 4
|
|
225
|
+
SPAN_H2S = 6
|
|
226
|
+
MANUAL_ZERO = 10
|
|
227
|
+
MANUAL_SAMPLE = 11
|
|
228
|
+
MANUAL_SPAN_CO2_SO2 = 14
|
|
229
|
+
MANUAL_SPAN_H2S = 16
|
|
230
|
+
|
|
231
|
+
@classmethod
|
|
232
|
+
def _missing_(cls, value: object) -> None:
|
|
233
|
+
"""Raise a descriptive error for unrecognised status codes.
|
|
234
|
+
|
|
235
|
+
Args:
|
|
236
|
+
value: The value that failed to match any member.
|
|
237
|
+
|
|
238
|
+
Raises:
|
|
239
|
+
ValueError: Always raised with a message listing valid integer
|
|
240
|
+
values and attribute-access forms.
|
|
241
|
+
|
|
242
|
+
Example:
|
|
243
|
+
>>> SensorStatus(99)
|
|
244
|
+
ValueError: 99 is not a valid SensorStatus. ...
|
|
245
|
+
"""
|
|
246
|
+
_raise_missing_value(cls, value)
|
|
247
|
+
|
|
248
|
+
@property
|
|
249
|
+
def description(self) -> str:
|
|
250
|
+
"""Human-readable description of this status code.
|
|
251
|
+
|
|
252
|
+
Returns:
|
|
253
|
+
str: A plain-English label for the sensor operating mode.
|
|
254
|
+
|
|
255
|
+
Example:
|
|
256
|
+
>>> SensorStatus.WARMING_UP.description
|
|
257
|
+
'Warming Up'
|
|
258
|
+
"""
|
|
259
|
+
_descriptions: dict[int, str] = {
|
|
260
|
+
-2: "Chemical Sensor Off",
|
|
261
|
+
-1: "Warming Up",
|
|
262
|
+
0: "Zero",
|
|
263
|
+
1: "Sample Acquisition",
|
|
264
|
+
4: "Standart Gas Measurement for CO2 and SO2",
|
|
265
|
+
6: "Standart Gas Measurement for H2S",
|
|
266
|
+
10: "Manual Zero Measurement",
|
|
267
|
+
11: "Manual Sample Measurement",
|
|
268
|
+
14: "Manual Standart Gas Measurement for CO2 and SO2",
|
|
269
|
+
16: "Manual Standart Gas Measurement for H2S",
|
|
270
|
+
}
|
|
271
|
+
return _descriptions[self.value]
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
class FileFormat(StrEnum):
|
|
275
|
+
"""Output file formats supported by writers in the multigas package.
|
|
276
|
+
|
|
277
|
+
Values match the tokens accepted by the writer entry points and by
|
|
278
|
+
downstream tooling that inspects the target format.
|
|
279
|
+
|
|
280
|
+
Example:
|
|
281
|
+
>>> FileFormat.CSV
|
|
282
|
+
<FileFormat.CSV: 'csv'>
|
|
283
|
+
>>> FileFormat("parquet").value
|
|
284
|
+
'parquet'
|
|
285
|
+
"""
|
|
286
|
+
|
|
287
|
+
CSV = "csv"
|
|
288
|
+
EXCEL = "excel"
|
|
289
|
+
PARQUET = "parquet"
|
|
290
|
+
JSON = "json"
|