eagleplot 3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eagleplot/Home.py +362 -0
- eagleplot/__init__.py +19 -0
- eagleplot/annotations.py +215 -0
- eagleplot/assets/logo.ico +0 -0
- eagleplot/assets/logo.png +0 -0
- eagleplot/assets/nccu-horiz-logo.png +0 -0
- eagleplot/assets/nccu-wings.png +0 -0
- eagleplot/batch.py +446 -0
- eagleplot/cli.py +719 -0
- eagleplot/data_tools.py +849 -0
- eagleplot/example-data/Artificial/bimodal_distribution.csv +501 -0
- eagleplot/example-data/Artificial/correlation_matrix.npy +0 -0
- eagleplot/example-data/Artificial/gaussian_surface.npy +0 -0
- eagleplot/example-data/Artificial/grouped_categorical.csv +241 -0
- eagleplot/example-data/Artificial/ir_spectrum.csv +1802 -0
- eagleplot/example-data/Artificial/molecular_properties.csv +121 -0
- eagleplot/example-data/Artificial/normal_distribution.csv +401 -0
- eagleplot/example-data/Artificial/reaction_kinetics.csv +121 -0
- eagleplot/example-data/Artificial/scatter_correlation.csv +201 -0
- eagleplot/example-data/Artificial/sine_cosine_waves.csv +301 -0
- eagleplot/gallery/box_plot.png +0 -0
- eagleplot/gallery/contour_plot.png +0 -0
- eagleplot/gallery/distribution_plot.png +0 -0
- eagleplot/gallery/heatmap.png +0 -0
- eagleplot/gallery/histogram.png +0 -0
- eagleplot/gallery/interactive_3d_surface.png +0 -0
- eagleplot/gallery/interactive_heatmap.png +0 -0
- eagleplot/gallery/interactive_histogram.png +0 -0
- eagleplot/gallery/interactive_line.png +0 -0
- eagleplot/gallery/interactive_scatter.png +0 -0
- eagleplot/gallery/line_plot.png +0 -0
- eagleplot/gallery/manifest.json +196 -0
- eagleplot/gallery/regression_plot.png +0 -0
- eagleplot/gallery/scatter_plot.png +0 -0
- eagleplot/io.py +1008 -0
- eagleplot/math.py +1210 -0
- eagleplot/molecular/__init__.py +21 -0
- eagleplot/molecular/atom_data.py +287 -0
- eagleplot/molecular/parsers.py +747 -0
- eagleplot/nist.py +241 -0
- eagleplot/pages/10_Molecular_Viz.py +225 -0
- eagleplot/pages/11_Batch_Analysis.py +325 -0
- eagleplot/pages/12_Help.py +199 -0
- eagleplot/pages/13_Settings.py +388 -0
- eagleplot/pages/1_Data_Upload.py +521 -0
- eagleplot/pages/2_Quick_Plot.py +1693 -0
- eagleplot/pages/3_Analysis_Tools.py +2510 -0
- eagleplot/pages/4_Multi_Plot_Dashboard.py +791 -0
- eagleplot/pages/5_Advanced_Plotting.py +691 -0
- eagleplot/pages/6_Export_Results.py +442 -0
- eagleplot/pages/7_Gallery.py +201 -0
- eagleplot/pages/8_Data_Tools.py +434 -0
- eagleplot/pages/9_Spectroscopy.py +821 -0
- eagleplot/pages/__init__.py +8 -0
- eagleplot/peaks.py +513 -0
- eagleplot/plotting.py +2630 -0
- eagleplot/plugin_loader.py +208 -0
- eagleplot/report.py +448 -0
- eagleplot/signal.py +494 -0
- eagleplot/spectroscopy.py +830 -0
- eagleplot/utils/__init__.py +76 -0
- eagleplot/utils/access.py +77 -0
- eagleplot/utils/data_preview.py +352 -0
- eagleplot/utils/plot_config.py +2721 -0
- eagleplot/utils/session_state.py +535 -0
- eagleplot/utils/theming.py +312 -0
- eagleplot/utils/user_settings.py +294 -0
- eagleplot-3.0.0.dist-info/METADATA +304 -0
- eagleplot-3.0.0.dist-info/RECORD +73 -0
- eagleplot-3.0.0.dist-info/WHEEL +5 -0
- eagleplot-3.0.0.dist-info/entry_points.txt +2 -0
- eagleplot-3.0.0.dist-info/licenses/LICENSE +21 -0
- eagleplot-3.0.0.dist-info/top_level.txt +1 -0
eagleplot/data_tools.py
ADDED
|
@@ -0,0 +1,849 @@
|
|
|
1
|
+
"""Data Manipulation Module.
|
|
2
|
+
|
|
3
|
+
Non-destructive DataFrame transformation operations. Every function accepts a
|
|
4
|
+
``pandas.DataFrame`` and returns a **new** DataFrame; the original is never
|
|
5
|
+
modified in-place.
|
|
6
|
+
|
|
7
|
+
Functions
|
|
8
|
+
---------
|
|
9
|
+
Column transforms:
|
|
10
|
+
add_formula_column -- compute a new column from a safe expression
|
|
11
|
+
normalize_column -- min-max, z-score, percent-of-max, area
|
|
12
|
+
|
|
13
|
+
Shape / structure:
|
|
14
|
+
transpose_dataframe -- rows ↔ columns
|
|
15
|
+
pivot_dataframe -- long → wide (pivot)
|
|
16
|
+
melt_dataframe -- wide → long (melt)
|
|
17
|
+
|
|
18
|
+
Filtering / sorting:
|
|
19
|
+
filter_rows -- keep or drop rows matching a condition string
|
|
20
|
+
sort_dataframe -- sort by one or more columns
|
|
21
|
+
|
|
22
|
+
Merging:
|
|
23
|
+
merge_dataframes -- join two DataFrames on a key column
|
|
24
|
+
|
|
25
|
+
Missing values:
|
|
26
|
+
fill_nan -- fill NaN with a strategy (mean/median/zero/interpolate)
|
|
27
|
+
drop_nan -- drop rows that contain NaN
|
|
28
|
+
|
|
29
|
+
Time-series transforms:
|
|
30
|
+
resample_dataframe -- upsample or downsample to a new x-grid (cubic/linear/nearest)
|
|
31
|
+
rolling_transform -- rolling mean / rolling sum / cumulative sum per column
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
from __future__ import annotations
|
|
35
|
+
|
|
36
|
+
import ast
|
|
37
|
+
import math
|
|
38
|
+
import operator
|
|
39
|
+
from typing import Any, Callable, Dict, List, Optional
|
|
40
|
+
|
|
41
|
+
import numpy as np
|
|
42
|
+
import pandas as pd
|
|
43
|
+
from scipy.interpolate import interp1d
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
# ─── Column transforms ────────────────────────────────────────────────────────
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# Safe built-in namespace made available inside formula expressions.
|
|
50
|
+
_SAFE_MATH = {
|
|
51
|
+
"log": np.log,
|
|
52
|
+
"log2": np.log2,
|
|
53
|
+
"log10": np.log10,
|
|
54
|
+
"exp": np.exp,
|
|
55
|
+
"sqrt": np.sqrt,
|
|
56
|
+
"abs": np.abs,
|
|
57
|
+
"sin": np.sin,
|
|
58
|
+
"cos": np.cos,
|
|
59
|
+
"tan": np.tan,
|
|
60
|
+
"pi": math.pi,
|
|
61
|
+
"e": math.e,
|
|
62
|
+
"mean": np.mean,
|
|
63
|
+
"std": np.std,
|
|
64
|
+
"min": np.min,
|
|
65
|
+
"max": np.max,
|
|
66
|
+
"sum": np.sum,
|
|
67
|
+
"cumsum": np.cumsum,
|
|
68
|
+
"diff": np.diff,
|
|
69
|
+
"nan": np.nan,
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
# ─── Safe expression evaluation ───────────────────────────────────────────────
|
|
74
|
+
#
|
|
75
|
+
# Formula expressions arrive from a text input in the GUI, so they are
|
|
76
|
+
# untrusted. `eval(expr, {"__builtins__": {}}, ns)` is NOT a sandbox — the
|
|
77
|
+
# empty-builtins trick is bypassed by walking the type hierarchy to reach an
|
|
78
|
+
# importer, e.g.
|
|
79
|
+
#
|
|
80
|
+
# [c for c in ().__class__.__bases__[0].__subclasses__()
|
|
81
|
+
# if c.__name__ == "BuiltinImporter"][0]().load_module("os")
|
|
82
|
+
#
|
|
83
|
+
# Every step of that escape needs attribute access, so the evaluator below
|
|
84
|
+
# whitelists AST node types and rejects `ast.Attribute` outright. Anything not
|
|
85
|
+
# explicitly allowed raises ValueError rather than being evaluated.
|
|
86
|
+
|
|
87
|
+
_BIN_OPS: Dict[type, Callable[[Any, Any], Any]] = {
|
|
88
|
+
ast.Add: operator.add,
|
|
89
|
+
ast.Sub: operator.sub,
|
|
90
|
+
ast.Mult: operator.mul,
|
|
91
|
+
ast.Div: operator.truediv,
|
|
92
|
+
ast.FloorDiv: operator.floordiv,
|
|
93
|
+
ast.Mod: operator.mod,
|
|
94
|
+
ast.Pow: operator.pow,
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
_UNARY_OPS: Dict[type, Callable[[Any], Any]] = {
|
|
98
|
+
ast.UAdd: operator.pos,
|
|
99
|
+
ast.USub: operator.neg,
|
|
100
|
+
ast.Not: operator.not_,
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
_COMPARE_OPS: Dict[type, Callable[[Any, Any], Any]] = {
|
|
104
|
+
ast.Eq: operator.eq,
|
|
105
|
+
ast.NotEq: operator.ne,
|
|
106
|
+
ast.Lt: operator.lt,
|
|
107
|
+
ast.LtE: operator.le,
|
|
108
|
+
ast.Gt: operator.gt,
|
|
109
|
+
ast.GtE: operator.ge,
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
# Guards against expressions that are cheap to type and expensive to evaluate,
|
|
113
|
+
# e.g. `9**9**9`, which would otherwise hang the app.
|
|
114
|
+
_MAX_POW_EXPONENT = 1024
|
|
115
|
+
_MAX_EXPRESSION_LENGTH = 2000
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _safe_eval(expression: str, namespace: Dict[str, Any]) -> Any:
|
|
119
|
+
"""Evaluate an arithmetic expression without exposing the Python runtime.
|
|
120
|
+
|
|
121
|
+
Supports numeric and boolean literals, the names in *namespace*, the binary
|
|
122
|
+
and unary arithmetic operators, comparisons, ``and``/``or``/``not``, tuples
|
|
123
|
+
and lists, subscripting, and calls to the whitelisted functions in
|
|
124
|
+
*namespace*. Attribute access, imports, lambdas, comprehensions,
|
|
125
|
+
assignments, and every other construct are rejected.
|
|
126
|
+
|
|
127
|
+
Parameters
|
|
128
|
+
----------
|
|
129
|
+
expression : str
|
|
130
|
+
Expression source.
|
|
131
|
+
namespace : dict
|
|
132
|
+
Names visible to the expression — the math helpers in ``_SAFE_MATH``
|
|
133
|
+
plus one entry per DataFrame column.
|
|
134
|
+
|
|
135
|
+
Returns
|
|
136
|
+
-------
|
|
137
|
+
Any
|
|
138
|
+
The value of the expression.
|
|
139
|
+
|
|
140
|
+
Raises
|
|
141
|
+
------
|
|
142
|
+
ValueError
|
|
143
|
+
If the expression uses a construct that is not whitelisted, references
|
|
144
|
+
an unknown name, or exceeds the size/complexity guards.
|
|
145
|
+
"""
|
|
146
|
+
if len(expression) > _MAX_EXPRESSION_LENGTH:
|
|
147
|
+
raise ValueError(
|
|
148
|
+
f"expression is too long ({len(expression)} characters; "
|
|
149
|
+
f"limit is {_MAX_EXPRESSION_LENGTH})"
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
try:
|
|
153
|
+
tree = ast.parse(expression, mode="eval")
|
|
154
|
+
except SyntaxError as exc:
|
|
155
|
+
raise ValueError(f"could not parse expression: {exc.msg}") from exc
|
|
156
|
+
|
|
157
|
+
def _describe(node: ast.AST) -> str:
|
|
158
|
+
if isinstance(node, ast.Attribute):
|
|
159
|
+
return "attribute access (e.g. `x.attr`) is not allowed in formulas"
|
|
160
|
+
if isinstance(
|
|
161
|
+
node, (ast.Lambda, ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp)
|
|
162
|
+
):
|
|
163
|
+
return f"`{type(node).__name__}` is not allowed in formulas"
|
|
164
|
+
if isinstance(node, (ast.Import, ast.ImportFrom)):
|
|
165
|
+
return "imports are not allowed in formulas"
|
|
166
|
+
return f"`{type(node).__name__}` is not supported in formulas"
|
|
167
|
+
|
|
168
|
+
def _eval(node: ast.AST) -> Any:
|
|
169
|
+
if isinstance(node, ast.Expression):
|
|
170
|
+
return _eval(node.body)
|
|
171
|
+
|
|
172
|
+
if isinstance(node, ast.Constant):
|
|
173
|
+
if isinstance(node.value, (int, float, complex, bool)) or node.value is None:
|
|
174
|
+
return node.value
|
|
175
|
+
if isinstance(node.value, str):
|
|
176
|
+
return node.value
|
|
177
|
+
raise ValueError(f"literal of type {type(node.value).__name__} is not allowed")
|
|
178
|
+
|
|
179
|
+
if isinstance(node, ast.Name):
|
|
180
|
+
if node.id not in namespace:
|
|
181
|
+
raise ValueError(
|
|
182
|
+
f"unknown name '{node.id}' — expected a column name or one of: "
|
|
183
|
+
+ ", ".join(sorted(_SAFE_MATH))
|
|
184
|
+
)
|
|
185
|
+
return namespace[node.id]
|
|
186
|
+
|
|
187
|
+
if isinstance(node, ast.BinOp):
|
|
188
|
+
bin_op = _BIN_OPS.get(type(node.op))
|
|
189
|
+
if bin_op is None:
|
|
190
|
+
raise ValueError(f"operator `{type(node.op).__name__}` is not allowed")
|
|
191
|
+
left, right = _eval(node.left), _eval(node.right)
|
|
192
|
+
if bin_op is operator.pow and isinstance(right, (int, float)):
|
|
193
|
+
if abs(right) > _MAX_POW_EXPONENT:
|
|
194
|
+
raise ValueError(f"exponent {right} exceeds the limit of {_MAX_POW_EXPONENT}")
|
|
195
|
+
return bin_op(left, right)
|
|
196
|
+
|
|
197
|
+
if isinstance(node, ast.UnaryOp):
|
|
198
|
+
unary_op = _UNARY_OPS.get(type(node.op))
|
|
199
|
+
if unary_op is None:
|
|
200
|
+
raise ValueError(f"operator `{type(node.op).__name__}` is not allowed")
|
|
201
|
+
return unary_op(_eval(node.operand))
|
|
202
|
+
|
|
203
|
+
if isinstance(node, ast.BoolOp):
|
|
204
|
+
values = [_eval(v) for v in node.values]
|
|
205
|
+
result = values[0]
|
|
206
|
+
for value in values[1:]:
|
|
207
|
+
result = (result & value) if isinstance(node.op, ast.And) else (result | value)
|
|
208
|
+
return result
|
|
209
|
+
|
|
210
|
+
if isinstance(node, ast.Compare):
|
|
211
|
+
result = None
|
|
212
|
+
left = _eval(node.left)
|
|
213
|
+
for op_node, comparator_node in zip(node.ops, node.comparators):
|
|
214
|
+
cmp_op = _COMPARE_OPS.get(type(op_node))
|
|
215
|
+
if cmp_op is None:
|
|
216
|
+
raise ValueError(f"comparison `{type(op_node).__name__}` is not allowed")
|
|
217
|
+
right = _eval(comparator_node)
|
|
218
|
+
outcome = cmp_op(left, right)
|
|
219
|
+
result = outcome if result is None else (result & outcome)
|
|
220
|
+
left = right
|
|
221
|
+
return result
|
|
222
|
+
|
|
223
|
+
if isinstance(node, ast.Call):
|
|
224
|
+
# Only a bare whitelisted name may be called — never the result of
|
|
225
|
+
# another expression, which is how attribute-free escapes are built.
|
|
226
|
+
if not isinstance(node.func, ast.Name):
|
|
227
|
+
raise ValueError("only direct calls to the built-in formula functions are allowed")
|
|
228
|
+
if node.keywords:
|
|
229
|
+
raise ValueError("keyword arguments are not supported in formulas")
|
|
230
|
+
# Resolve against _SAFE_MATH directly rather than the namespace, so a
|
|
231
|
+
# column can never become callable no matter what it is named.
|
|
232
|
+
func = _SAFE_MATH.get(node.func.id)
|
|
233
|
+
if not callable(func):
|
|
234
|
+
raise ValueError(
|
|
235
|
+
f"'{node.func.id}' is not a callable formula function — allowed: "
|
|
236
|
+
+ ", ".join(sorted(k for k, v in _SAFE_MATH.items() if callable(v)))
|
|
237
|
+
)
|
|
238
|
+
return func(*[_eval(arg) for arg in node.args])
|
|
239
|
+
|
|
240
|
+
if isinstance(node, (ast.Tuple, ast.List)):
|
|
241
|
+
return [_eval(elt) for elt in node.elts]
|
|
242
|
+
|
|
243
|
+
if isinstance(node, ast.Subscript):
|
|
244
|
+
return _eval(node.value)[_eval(node.slice)]
|
|
245
|
+
|
|
246
|
+
if isinstance(node, ast.Slice):
|
|
247
|
+
return slice(
|
|
248
|
+
_eval(node.lower) if node.lower else None,
|
|
249
|
+
_eval(node.upper) if node.upper else None,
|
|
250
|
+
_eval(node.step) if node.step else None,
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
if isinstance(node, ast.IfExp):
|
|
254
|
+
return _eval(node.body) if _eval(node.test) else _eval(node.orelse)
|
|
255
|
+
|
|
256
|
+
raise ValueError(_describe(node))
|
|
257
|
+
|
|
258
|
+
return _eval(tree)
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def add_formula_column(
|
|
262
|
+
df: pd.DataFrame,
|
|
263
|
+
new_col: str,
|
|
264
|
+
expression: str,
|
|
265
|
+
) -> pd.DataFrame:
|
|
266
|
+
"""Compute a new column from a Python/NumPy expression.
|
|
267
|
+
|
|
268
|
+
Column names in the DataFrame are injected into the evaluation namespace
|
|
269
|
+
as local variables so they can be referenced directly in the expression.
|
|
270
|
+
|
|
271
|
+
Parameters
|
|
272
|
+
----------
|
|
273
|
+
df : pd.DataFrame
|
|
274
|
+
Input DataFrame (not modified).
|
|
275
|
+
new_col : str
|
|
276
|
+
Name for the new column.
|
|
277
|
+
expression : str
|
|
278
|
+
An arithmetic expression using column names and the math functions
|
|
279
|
+
listed below. For example: ``"log(A) / B"`` or
|
|
280
|
+
``"(A - mean(A)) / std(A)"``.
|
|
281
|
+
|
|
282
|
+
Available functions: ``log``, ``log2``, ``log10``, ``exp``, ``sqrt``,
|
|
283
|
+
``abs``, ``sin``, ``cos``, ``tan``, ``mean``, ``std``, ``min``,
|
|
284
|
+
``max``, ``sum``, ``cumsum``, ``diff`` (all operate on arrays).
|
|
285
|
+
Constants: ``pi``, ``e``, ``nan``.
|
|
286
|
+
|
|
287
|
+
The expression is evaluated by :func:`_safe_eval`, which permits
|
|
288
|
+
arithmetic, comparisons, ``and``/``or``/``not``, conditional
|
|
289
|
+
expressions, subscripting, and calls to the functions above. Attribute
|
|
290
|
+
access, imports, lambdas, comprehensions, and assignments are rejected.
|
|
291
|
+
|
|
292
|
+
Returns
|
|
293
|
+
-------
|
|
294
|
+
pd.DataFrame
|
|
295
|
+
Copy of *df* with *new_col* appended.
|
|
296
|
+
|
|
297
|
+
Raises
|
|
298
|
+
------
|
|
299
|
+
ValueError
|
|
300
|
+
If *expression* or *new_col* is empty, if *expression* cannot be
|
|
301
|
+
parsed, if it references an unknown name, or if it uses a construct
|
|
302
|
+
that is not whitelisted.
|
|
303
|
+
Exception
|
|
304
|
+
Re-raised if evaluating a whitelisted expression itself raises — for
|
|
305
|
+
example ``ZeroDivisionError`` for ``"1 / 0"``.
|
|
306
|
+
"""
|
|
307
|
+
if not expression.strip():
|
|
308
|
+
raise ValueError("expression must not be empty")
|
|
309
|
+
if not new_col.strip():
|
|
310
|
+
raise ValueError("new_col must not be empty")
|
|
311
|
+
|
|
312
|
+
# Build evaluation namespace: math helpers + column arrays
|
|
313
|
+
namespace = dict(_SAFE_MATH)
|
|
314
|
+
for col in df.columns:
|
|
315
|
+
namespace[col] = df[col].to_numpy(dtype=float, na_value=np.nan)
|
|
316
|
+
|
|
317
|
+
result = _safe_eval(expression, namespace)
|
|
318
|
+
|
|
319
|
+
out = df.copy()
|
|
320
|
+
out[new_col] = result
|
|
321
|
+
return out
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def normalize_column(
|
|
325
|
+
df: pd.DataFrame,
|
|
326
|
+
column: str,
|
|
327
|
+
method: str = "min-max",
|
|
328
|
+
new_col: Optional[str] = None,
|
|
329
|
+
) -> pd.DataFrame:
|
|
330
|
+
"""Normalize a numeric column and store the result.
|
|
331
|
+
|
|
332
|
+
Parameters
|
|
333
|
+
----------
|
|
334
|
+
df : pd.DataFrame
|
|
335
|
+
Input DataFrame (not modified).
|
|
336
|
+
column : str
|
|
337
|
+
Name of the column to normalize.
|
|
338
|
+
method : str
|
|
339
|
+
Normalization strategy. One of:
|
|
340
|
+
|
|
341
|
+
- ``'min-max'`` — scale to [0, 1]
|
|
342
|
+
- ``'z-score'`` — subtract mean, divide by std
|
|
343
|
+
- ``'pct-max'`` — divide by maximum (percentage of max)
|
|
344
|
+
- ``'area'`` — divide by the trapezoidal area (unit-area signal)
|
|
345
|
+
new_col : str, optional
|
|
346
|
+
Name for the output column. Defaults to ``"<column>_norm"``.
|
|
347
|
+
|
|
348
|
+
Returns
|
|
349
|
+
-------
|
|
350
|
+
pd.DataFrame
|
|
351
|
+
Copy of *df* with the normalized column appended (or replaced if
|
|
352
|
+
*new_col* already exists).
|
|
353
|
+
|
|
354
|
+
Raises
|
|
355
|
+
------
|
|
356
|
+
KeyError
|
|
357
|
+
If *column* is not in *df*.
|
|
358
|
+
ValueError
|
|
359
|
+
If *method* is not one of the supported options, or the column
|
|
360
|
+
has no variance / area == 0 (would cause division by zero).
|
|
361
|
+
"""
|
|
362
|
+
if column not in df.columns:
|
|
363
|
+
raise KeyError(f"Column '{column}' not found in DataFrame")
|
|
364
|
+
|
|
365
|
+
methods = ("min-max", "z-score", "pct-max", "area")
|
|
366
|
+
if method not in methods:
|
|
367
|
+
raise ValueError(f"method must be one of {methods}; got '{method}'")
|
|
368
|
+
|
|
369
|
+
y = df[column].to_numpy(dtype=float)
|
|
370
|
+
|
|
371
|
+
if method == "min-max":
|
|
372
|
+
lo, hi = np.nanmin(y), np.nanmax(y)
|
|
373
|
+
if hi == lo:
|
|
374
|
+
raise ValueError(f"Column '{column}' is constant — cannot apply min-max normalization")
|
|
375
|
+
result = (y - lo) / (hi - lo)
|
|
376
|
+
|
|
377
|
+
elif method == "z-score":
|
|
378
|
+
mu = np.nanmean(y)
|
|
379
|
+
sigma = np.nanstd(y)
|
|
380
|
+
if sigma == 0:
|
|
381
|
+
raise ValueError(
|
|
382
|
+
f"Column '{column}' has zero variance — cannot apply z-score normalization"
|
|
383
|
+
)
|
|
384
|
+
result = (y - mu) / sigma
|
|
385
|
+
|
|
386
|
+
elif method == "pct-max":
|
|
387
|
+
mx = np.nanmax(np.abs(y))
|
|
388
|
+
if mx == 0:
|
|
389
|
+
raise ValueError(f"Column '{column}' is all-zero — cannot apply pct-max normalization")
|
|
390
|
+
result = y / mx
|
|
391
|
+
|
|
392
|
+
else: # area
|
|
393
|
+
area = np.trapezoid(np.abs(y))
|
|
394
|
+
if area == 0:
|
|
395
|
+
raise ValueError(f"Column '{column}' has zero area — cannot apply area normalization")
|
|
396
|
+
result = y / area
|
|
397
|
+
|
|
398
|
+
out = df.copy()
|
|
399
|
+
col_name = new_col if new_col else f"{column}_norm"
|
|
400
|
+
out[col_name] = result
|
|
401
|
+
return out
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
# ─── Shape / structure ────────────────────────────────────────────────────────
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def transpose_dataframe(df: pd.DataFrame) -> pd.DataFrame:
|
|
408
|
+
"""Transpose a DataFrame (rows ↔ columns).
|
|
409
|
+
|
|
410
|
+
Parameters
|
|
411
|
+
----------
|
|
412
|
+
df : pd.DataFrame
|
|
413
|
+
Input DataFrame.
|
|
414
|
+
|
|
415
|
+
Returns
|
|
416
|
+
-------
|
|
417
|
+
pd.DataFrame
|
|
418
|
+
Transposed copy with reset index and columns converted to strings.
|
|
419
|
+
"""
|
|
420
|
+
transposed = df.T.copy()
|
|
421
|
+
# Convert column index (originally row index) to strings for consistency
|
|
422
|
+
transposed.columns = [str(c) for c in transposed.columns]
|
|
423
|
+
transposed = transposed.reset_index()
|
|
424
|
+
transposed = transposed.rename(columns={"index": "column"})
|
|
425
|
+
return transposed
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
def pivot_dataframe(
|
|
429
|
+
df: pd.DataFrame,
|
|
430
|
+
index: str,
|
|
431
|
+
columns: str,
|
|
432
|
+
values: str,
|
|
433
|
+
) -> pd.DataFrame:
|
|
434
|
+
"""Pivot a long-format DataFrame to wide format.
|
|
435
|
+
|
|
436
|
+
Parameters
|
|
437
|
+
----------
|
|
438
|
+
df : pd.DataFrame
|
|
439
|
+
Input DataFrame in long format.
|
|
440
|
+
index : str
|
|
441
|
+
Column to use as the new row index.
|
|
442
|
+
columns : str
|
|
443
|
+
Column whose unique values become new column headers.
|
|
444
|
+
values : str
|
|
445
|
+
Column to use for cell values.
|
|
446
|
+
|
|
447
|
+
Returns
|
|
448
|
+
-------
|
|
449
|
+
pd.DataFrame
|
|
450
|
+
Wide-format DataFrame with reset index.
|
|
451
|
+
|
|
452
|
+
Raises
|
|
453
|
+
------
|
|
454
|
+
KeyError
|
|
455
|
+
If any of *index*, *columns*, or *values* are not in *df*.
|
|
456
|
+
"""
|
|
457
|
+
for col in (index, columns, values):
|
|
458
|
+
if col not in df.columns:
|
|
459
|
+
raise KeyError(f"Column '{col}' not found in DataFrame")
|
|
460
|
+
|
|
461
|
+
result = df.pivot(index=index, columns=columns, values=values)
|
|
462
|
+
result.columns.name = None
|
|
463
|
+
result = result.reset_index()
|
|
464
|
+
return result
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def melt_dataframe(
|
|
468
|
+
df: pd.DataFrame,
|
|
469
|
+
id_vars: List[str],
|
|
470
|
+
value_vars: Optional[List[str]] = None,
|
|
471
|
+
var_name: str = "variable",
|
|
472
|
+
value_name: str = "value",
|
|
473
|
+
) -> pd.DataFrame:
|
|
474
|
+
"""Melt a wide-format DataFrame to long format.
|
|
475
|
+
|
|
476
|
+
Parameters
|
|
477
|
+
----------
|
|
478
|
+
df : pd.DataFrame
|
|
479
|
+
Input DataFrame in wide format.
|
|
480
|
+
id_vars : list of str
|
|
481
|
+
Columns to keep as identifier variables.
|
|
482
|
+
value_vars : list of str, optional
|
|
483
|
+
Columns to melt. Defaults to all columns not in *id_vars*.
|
|
484
|
+
var_name : str
|
|
485
|
+
Name for the variable column in the output.
|
|
486
|
+
value_name : str
|
|
487
|
+
Name for the value column in the output.
|
|
488
|
+
|
|
489
|
+
Returns
|
|
490
|
+
-------
|
|
491
|
+
pd.DataFrame
|
|
492
|
+
Long-format DataFrame.
|
|
493
|
+
"""
|
|
494
|
+
return df.melt(
|
|
495
|
+
id_vars=id_vars,
|
|
496
|
+
value_vars=value_vars,
|
|
497
|
+
var_name=var_name,
|
|
498
|
+
value_name=value_name,
|
|
499
|
+
)
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
# ─── Filtering / sorting ──────────────────────────────────────────────────────
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def filter_rows(
|
|
506
|
+
df: pd.DataFrame,
|
|
507
|
+
condition: str,
|
|
508
|
+
keep: bool = True,
|
|
509
|
+
) -> pd.DataFrame:
|
|
510
|
+
"""Filter rows using a pandas ``query``-style condition string.
|
|
511
|
+
|
|
512
|
+
Parameters
|
|
513
|
+
----------
|
|
514
|
+
df : pd.DataFrame
|
|
515
|
+
Input DataFrame (not modified).
|
|
516
|
+
condition : str
|
|
517
|
+
A pandas ``DataFrame.query`` expression such as ``"A > 0"`` or
|
|
518
|
+
``"B == 'control'"`` or ``"A > 0 and C < 10"``.
|
|
519
|
+
keep : bool
|
|
520
|
+
If ``True`` (default), keep rows that match the condition.
|
|
521
|
+
If ``False``, drop rows that match (i.e. keep the complement).
|
|
522
|
+
|
|
523
|
+
Returns
|
|
524
|
+
-------
|
|
525
|
+
pd.DataFrame
|
|
526
|
+
Filtered copy with reset index.
|
|
527
|
+
|
|
528
|
+
Raises
|
|
529
|
+
------
|
|
530
|
+
ValueError
|
|
531
|
+
If *condition* is empty.
|
|
532
|
+
Exception
|
|
533
|
+
Re-raised if ``query`` fails (bad syntax, unknown column, etc.).
|
|
534
|
+
"""
|
|
535
|
+
if not condition.strip():
|
|
536
|
+
raise ValueError("condition must not be empty")
|
|
537
|
+
|
|
538
|
+
mask = df.eval(condition)
|
|
539
|
+
if not keep:
|
|
540
|
+
mask = ~mask
|
|
541
|
+
return df[mask].reset_index(drop=True)
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
def sort_dataframe(
|
|
545
|
+
df: pd.DataFrame,
|
|
546
|
+
by: List[str],
|
|
547
|
+
ascending: bool = True,
|
|
548
|
+
) -> pd.DataFrame:
|
|
549
|
+
"""Sort a DataFrame by one or more columns.
|
|
550
|
+
|
|
551
|
+
Parameters
|
|
552
|
+
----------
|
|
553
|
+
df : pd.DataFrame
|
|
554
|
+
Input DataFrame (not modified).
|
|
555
|
+
by : list of str
|
|
556
|
+
Column name(s) to sort by.
|
|
557
|
+
ascending : bool
|
|
558
|
+
Sort direction. Use ``True`` for ascending (default), ``False``
|
|
559
|
+
for descending.
|
|
560
|
+
|
|
561
|
+
Returns
|
|
562
|
+
-------
|
|
563
|
+
pd.DataFrame
|
|
564
|
+
Sorted copy with reset index.
|
|
565
|
+
|
|
566
|
+
Raises
|
|
567
|
+
------
|
|
568
|
+
KeyError
|
|
569
|
+
If any column in *by* is not in *df*.
|
|
570
|
+
ValueError
|
|
571
|
+
If *by* is empty.
|
|
572
|
+
"""
|
|
573
|
+
if not by:
|
|
574
|
+
raise ValueError("'by' must contain at least one column name")
|
|
575
|
+
for col in by:
|
|
576
|
+
if col not in df.columns:
|
|
577
|
+
raise KeyError(f"Column '{col}' not found in DataFrame")
|
|
578
|
+
|
|
579
|
+
return df.sort_values(by=by, ascending=ascending).reset_index(drop=True)
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
# ─── Merging ──────────────────────────────────────────────────────────────────
|
|
583
|
+
|
|
584
|
+
|
|
585
|
+
def merge_dataframes(
|
|
586
|
+
left: pd.DataFrame,
|
|
587
|
+
right: pd.DataFrame,
|
|
588
|
+
on: str,
|
|
589
|
+
how: str = "inner",
|
|
590
|
+
suffixes: tuple = ("_left", "_right"),
|
|
591
|
+
) -> pd.DataFrame:
|
|
592
|
+
"""Join two DataFrames on a shared key column.
|
|
593
|
+
|
|
594
|
+
Parameters
|
|
595
|
+
----------
|
|
596
|
+
left : pd.DataFrame
|
|
597
|
+
Left DataFrame.
|
|
598
|
+
right : pd.DataFrame
|
|
599
|
+
Right DataFrame.
|
|
600
|
+
on : str
|
|
601
|
+
Column name to join on (must exist in both DataFrames).
|
|
602
|
+
how : str
|
|
603
|
+
Type of join: ``'inner'``, ``'left'``, ``'right'``, or ``'outer'``.
|
|
604
|
+
suffixes : tuple of str
|
|
605
|
+
Suffixes to append to overlapping column names.
|
|
606
|
+
|
|
607
|
+
Returns
|
|
608
|
+
-------
|
|
609
|
+
pd.DataFrame
|
|
610
|
+
Merged DataFrame.
|
|
611
|
+
|
|
612
|
+
Raises
|
|
613
|
+
------
|
|
614
|
+
KeyError
|
|
615
|
+
If *on* is not present in both DataFrames.
|
|
616
|
+
ValueError
|
|
617
|
+
If *how* is not a valid join type.
|
|
618
|
+
"""
|
|
619
|
+
valid_how = ("inner", "left", "right", "outer")
|
|
620
|
+
if how not in valid_how:
|
|
621
|
+
raise ValueError(f"how must be one of {valid_how}; got '{how}'")
|
|
622
|
+
if on not in left.columns:
|
|
623
|
+
raise KeyError(f"Key column '{on}' not found in left DataFrame")
|
|
624
|
+
if on not in right.columns:
|
|
625
|
+
raise KeyError(f"Key column '{on}' not found in right DataFrame")
|
|
626
|
+
|
|
627
|
+
return pd.merge(left, right, on=on, how=how, suffixes=suffixes)
|
|
628
|
+
|
|
629
|
+
|
|
630
|
+
# ─── Missing values ───────────────────────────────────────────────────────────
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
def fill_nan(
|
|
634
|
+
df: pd.DataFrame,
|
|
635
|
+
columns: Optional[List[str]] = None,
|
|
636
|
+
method: str = "mean",
|
|
637
|
+
) -> pd.DataFrame:
|
|
638
|
+
"""Fill NaN values in selected columns.
|
|
639
|
+
|
|
640
|
+
Parameters
|
|
641
|
+
----------
|
|
642
|
+
df : pd.DataFrame
|
|
643
|
+
Input DataFrame (not modified).
|
|
644
|
+
columns : list of str, optional
|
|
645
|
+
Columns to fill. Defaults to all numeric columns.
|
|
646
|
+
method : str
|
|
647
|
+
Fill strategy. One of:
|
|
648
|
+
|
|
649
|
+
- ``'mean'`` — replace with column mean
|
|
650
|
+
- ``'median'`` — replace with column median
|
|
651
|
+
- ``'zero'`` — replace with 0
|
|
652
|
+
- ``'interpolate'`` — linear interpolation along the column
|
|
653
|
+
- ``'forward'`` — forward-fill (propagate last valid value)
|
|
654
|
+
- ``'backward'`` — backward-fill
|
|
655
|
+
|
|
656
|
+
Returns
|
|
657
|
+
-------
|
|
658
|
+
pd.DataFrame
|
|
659
|
+
Copy of *df* with NaN values filled.
|
|
660
|
+
|
|
661
|
+
Raises
|
|
662
|
+
------
|
|
663
|
+
ValueError
|
|
664
|
+
If *method* is not recognized.
|
|
665
|
+
KeyError
|
|
666
|
+
If any column in *columns* is not in *df*.
|
|
667
|
+
"""
|
|
668
|
+
methods = ("mean", "median", "zero", "interpolate", "forward", "backward")
|
|
669
|
+
if method not in methods:
|
|
670
|
+
raise ValueError(f"method must be one of {methods}; got '{method}'")
|
|
671
|
+
|
|
672
|
+
out = df.copy()
|
|
673
|
+
cols = columns if columns is not None else list(df.select_dtypes(include="number").columns)
|
|
674
|
+
|
|
675
|
+
for col in cols:
|
|
676
|
+
if col not in df.columns:
|
|
677
|
+
raise KeyError(f"Column '{col}' not found in DataFrame")
|
|
678
|
+
if method == "mean":
|
|
679
|
+
out[col] = out[col].fillna(out[col].mean())
|
|
680
|
+
elif method == "median":
|
|
681
|
+
out[col] = out[col].fillna(out[col].median())
|
|
682
|
+
elif method == "zero":
|
|
683
|
+
out[col] = out[col].fillna(0)
|
|
684
|
+
elif method == "interpolate":
|
|
685
|
+
out[col] = out[col].interpolate(method="linear", limit_direction="both")
|
|
686
|
+
elif method == "forward":
|
|
687
|
+
out[col] = out[col].ffill()
|
|
688
|
+
elif method == "backward":
|
|
689
|
+
out[col] = out[col].bfill()
|
|
690
|
+
|
|
691
|
+
return out
|
|
692
|
+
|
|
693
|
+
|
|
694
|
+
def drop_nan(
|
|
695
|
+
df: pd.DataFrame,
|
|
696
|
+
columns: Optional[List[str]] = None,
|
|
697
|
+
) -> pd.DataFrame:
|
|
698
|
+
"""Drop rows that contain NaN in any (or selected) columns.
|
|
699
|
+
|
|
700
|
+
Parameters
|
|
701
|
+
----------
|
|
702
|
+
df : pd.DataFrame
|
|
703
|
+
Input DataFrame (not modified).
|
|
704
|
+
columns : list of str, optional
|
|
705
|
+
Only consider NaN in these columns. If ``None``, any NaN in any
|
|
706
|
+
column will trigger row removal.
|
|
707
|
+
|
|
708
|
+
Returns
|
|
709
|
+
-------
|
|
710
|
+
pd.DataFrame
|
|
711
|
+
Filtered copy with reset index.
|
|
712
|
+
"""
|
|
713
|
+
return df.dropna(subset=columns).reset_index(drop=True)
|
|
714
|
+
|
|
715
|
+
|
|
716
|
+
# ─── Time-series transforms ───────────────────────────────────────────────────
|
|
717
|
+
|
|
718
|
+
|
|
719
|
+
def resample_dataframe(
|
|
720
|
+
df: pd.DataFrame,
|
|
721
|
+
x_col: str,
|
|
722
|
+
y_cols: List[str],
|
|
723
|
+
n_points: int,
|
|
724
|
+
method: str = "cubic",
|
|
725
|
+
x_min: Optional[float] = None,
|
|
726
|
+
x_max: Optional[float] = None,
|
|
727
|
+
) -> pd.DataFrame:
|
|
728
|
+
"""Resample one or more y-columns to a new, uniform x-grid.
|
|
729
|
+
|
|
730
|
+
Parameters
|
|
731
|
+
----------
|
|
732
|
+
df : pd.DataFrame
|
|
733
|
+
Input DataFrame (not modified).
|
|
734
|
+
x_col : str
|
|
735
|
+
Column to use as the independent variable (must be numeric, monotonic).
|
|
736
|
+
y_cols : list of str
|
|
737
|
+
Columns to resample.
|
|
738
|
+
n_points : int
|
|
739
|
+
Number of points in the new x-grid.
|
|
740
|
+
method : str
|
|
741
|
+
Interpolation method: ``'cubic'``, ``'linear'``, or ``'nearest'``.
|
|
742
|
+
x_min : float, optional
|
|
743
|
+
Lower bound of the new x-grid. Defaults to ``min(x_col)``.
|
|
744
|
+
x_max : float, optional
|
|
745
|
+
Upper bound of the new x-grid. Defaults to ``max(x_col)``.
|
|
746
|
+
|
|
747
|
+
Returns
|
|
748
|
+
-------
|
|
749
|
+
pd.DataFrame
|
|
750
|
+
New DataFrame with *x_col* set to the uniform grid and each y-column
|
|
751
|
+
resampled.
|
|
752
|
+
|
|
753
|
+
Raises
|
|
754
|
+
------
|
|
755
|
+
KeyError
|
|
756
|
+
If *x_col* or any column in *y_cols* is not in *df*.
|
|
757
|
+
ValueError
|
|
758
|
+
If *method* is not supported, *n_points* < 2, or *x_col* is not
|
|
759
|
+
monotonic.
|
|
760
|
+
"""
|
|
761
|
+
if x_col not in df.columns:
|
|
762
|
+
raise KeyError(f"Column '{x_col}' not found in DataFrame")
|
|
763
|
+
for col in y_cols:
|
|
764
|
+
if col not in df.columns:
|
|
765
|
+
raise KeyError(f"Column '{col}' not found in DataFrame")
|
|
766
|
+
|
|
767
|
+
valid_methods = ("cubic", "linear", "nearest")
|
|
768
|
+
if method not in valid_methods:
|
|
769
|
+
raise ValueError(f"method must be one of {valid_methods}; got '{method}'")
|
|
770
|
+
|
|
771
|
+
if n_points < 2:
|
|
772
|
+
raise ValueError("n_points must be >= 2")
|
|
773
|
+
|
|
774
|
+
x = df[x_col].to_numpy(dtype=float)
|
|
775
|
+
if not (np.all(np.diff(x) > 0) or np.all(np.diff(x) < 0)):
|
|
776
|
+
raise ValueError(f"Column '{x_col}' must be strictly monotonic for resampling")
|
|
777
|
+
|
|
778
|
+
lo = x_min if x_min is not None else float(np.min(x))
|
|
779
|
+
hi = x_max if x_max is not None else float(np.max(x))
|
|
780
|
+
x_new = np.linspace(lo, hi, n_points)
|
|
781
|
+
|
|
782
|
+
kind = "cubic" if method == "cubic" else method
|
|
783
|
+
out = pd.DataFrame({x_col: x_new})
|
|
784
|
+
for col in y_cols:
|
|
785
|
+
y = df[col].to_numpy(dtype=float)
|
|
786
|
+
f = interp1d(x, y, kind=kind, bounds_error=False, fill_value="extrapolate")
|
|
787
|
+
out[col] = f(x_new)
|
|
788
|
+
|
|
789
|
+
return out
|
|
790
|
+
|
|
791
|
+
|
|
792
|
+
def rolling_transform(
|
|
793
|
+
df: pd.DataFrame,
|
|
794
|
+
columns: List[str],
|
|
795
|
+
operation: str = "rolling_mean",
|
|
796
|
+
window: int = 5,
|
|
797
|
+
) -> pd.DataFrame:
|
|
798
|
+
"""Apply rolling or cumulative transforms to selected columns.
|
|
799
|
+
|
|
800
|
+
Parameters
|
|
801
|
+
----------
|
|
802
|
+
df : pd.DataFrame
|
|
803
|
+
Input DataFrame (not modified).
|
|
804
|
+
columns : list of str
|
|
805
|
+
Columns to transform.
|
|
806
|
+
operation : str
|
|
807
|
+
Transform to apply. One of:
|
|
808
|
+
|
|
809
|
+
- ``'rolling_mean'`` — rolling (sliding-window) mean
|
|
810
|
+
- ``'rolling_sum'`` — rolling sum
|
|
811
|
+
- ``'cumsum'`` — cumulative sum
|
|
812
|
+
- ``'cumprod'`` — cumulative product
|
|
813
|
+
window : int
|
|
814
|
+
Window size (only used for rolling operations). Must be >= 1.
|
|
815
|
+
|
|
816
|
+
Returns
|
|
817
|
+
-------
|
|
818
|
+
pd.DataFrame
|
|
819
|
+
Copy of *df* with the transformed columns appended as
|
|
820
|
+
``"<col>_<operation>"``.
|
|
821
|
+
|
|
822
|
+
Raises
|
|
823
|
+
------
|
|
824
|
+
ValueError
|
|
825
|
+
If *operation* is not recognized or *window* < 1.
|
|
826
|
+
KeyError
|
|
827
|
+
If any column in *columns* is not in *df*.
|
|
828
|
+
"""
|
|
829
|
+
valid_ops = ("rolling_mean", "rolling_sum", "cumsum", "cumprod")
|
|
830
|
+
if operation not in valid_ops:
|
|
831
|
+
raise ValueError(f"operation must be one of {valid_ops}; got '{operation}'")
|
|
832
|
+
if window < 1:
|
|
833
|
+
raise ValueError("window must be >= 1")
|
|
834
|
+
|
|
835
|
+
out = df.copy()
|
|
836
|
+
for col in columns:
|
|
837
|
+
if col not in df.columns:
|
|
838
|
+
raise KeyError(f"Column '{col}' not found in DataFrame")
|
|
839
|
+
new_col = f"{col}_{operation}"
|
|
840
|
+
if operation == "rolling_mean":
|
|
841
|
+
out[new_col] = df[col].rolling(window=window, center=True, min_periods=1).mean()
|
|
842
|
+
elif operation == "rolling_sum":
|
|
843
|
+
out[new_col] = df[col].rolling(window=window, center=True, min_periods=1).sum()
|
|
844
|
+
elif operation == "cumsum":
|
|
845
|
+
out[new_col] = df[col].cumsum()
|
|
846
|
+
elif operation == "cumprod":
|
|
847
|
+
out[new_col] = df[col].cumprod()
|
|
848
|
+
|
|
849
|
+
return out
|