eosframes 1.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eosframes/__init__.py +121 -0
- eosframes/cli.py +764 -0
- eosframes/exceptions.py +20 -0
- eosframes/hub.py +152 -0
- eosframes/logger.py +127 -0
- eosframes/naming.py +610 -0
- eosframes/ops.py +711 -0
- eosframes/read.py +213 -0
- eosframes/scale.py +1713 -0
- eosframes/stack.py +204 -0
- eosframes/utils.py +23 -0
- eosframes/write.py +201 -0
- eosframes-1.1.0.dist-info/METADATA +112 -0
- eosframes-1.1.0.dist-info/RECORD +17 -0
- eosframes-1.1.0.dist-info/WHEEL +4 -0
- eosframes-1.1.0.dist-info/entry_points.txt +3 -0
- eosframes-1.1.0.dist-info/licenses/LICENSE +21 -0
eosframes/ops.py
ADDED
|
@@ -0,0 +1,711 @@
|
|
|
1
|
+
"""File-level operations: split, convert, stack, unstack, append, dedupe.
|
|
2
|
+
|
|
3
|
+
Every public function in this module follows the same shape:
|
|
4
|
+
|
|
5
|
+
1. Validate the destination path against the naming convention
|
|
6
|
+
(:func:`eosframes.naming.is_valid_name` and friends).
|
|
7
|
+
2. Refuse to overwrite an existing output.
|
|
8
|
+
3. Read the input(s) with :func:`eosframes.read.read_csv` /
|
|
9
|
+
:func:`eosframes.read.read_h5`, which attach ``df.model_id`` /
|
|
10
|
+
``df.version``.
|
|
11
|
+
4. Cross-validate model IDs (and versions where relevant) between input
|
|
12
|
+
and output before performing the operation.
|
|
13
|
+
5. Re-set ``df.model_id`` / ``df.version`` after any pandas operation
|
|
14
|
+
that may drop loose attributes (``concat`` / ``drop_duplicates``),
|
|
15
|
+
then write via :func:`eosframes.write.write_csv` /
|
|
16
|
+
:func:`eosframes.write.write_h5`.
|
|
17
|
+
|
|
18
|
+
The CLI in :mod:`eosframes.cli` is a thin Click adapter on top of these
|
|
19
|
+
functions; business logic lives here.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
import os
|
|
23
|
+
import re
|
|
24
|
+
from typing import Dict, List, Tuple
|
|
25
|
+
|
|
26
|
+
import numpy as np
|
|
27
|
+
import pandas as pd
|
|
28
|
+
|
|
29
|
+
from . import hub
|
|
30
|
+
from .exceptions import EosframesError
|
|
31
|
+
from .logger import get_logger
|
|
32
|
+
from .naming import (
|
|
33
|
+
is_model_id_valid,
|
|
34
|
+
is_valid_name,
|
|
35
|
+
is_valid_stack_explicit_name,
|
|
36
|
+
is_valid_stack_mix_name,
|
|
37
|
+
make_stack_explicit_name,
|
|
38
|
+
make_stack_mix_name,
|
|
39
|
+
parse_name,
|
|
40
|
+
parse_stack_explicit_name,
|
|
41
|
+
parse_stack_mix_name,
|
|
42
|
+
)
|
|
43
|
+
from .read import read_csv, read_h5
|
|
44
|
+
from .stack import hstack
|
|
45
|
+
from .utils import chunker
|
|
46
|
+
from .write import write_csv, write_h5
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _read_file(path: str) -> pd.DataFrame:
|
|
50
|
+
"""Read a CSV or H5 file into a DataFrame with ``model_id`` set.
|
|
51
|
+
|
|
52
|
+
Internal dispatcher that delegates to :func:`eosframes.read.read_csv`
|
|
53
|
+
or :func:`eosframes.read.read_h5` based on the file extension.
|
|
54
|
+
"""
|
|
55
|
+
ext = os.path.splitext(path)[1].lower()
|
|
56
|
+
if ext == ".csv":
|
|
57
|
+
return read_csv(path)
|
|
58
|
+
if ext == ".h5":
|
|
59
|
+
return read_h5(path)
|
|
60
|
+
raise EosframesError(
|
|
61
|
+
f"Unsupported format '{ext}' for '{path}'. Expected .csv or .h5"
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _require_no_overwrite(path: str, *, kind: str = "file") -> None:
|
|
66
|
+
"""Refuse to clobber an existing output.
|
|
67
|
+
|
|
68
|
+
The two-message split (``"Remove it first."`` vs ``"Remove it or choose
|
|
69
|
+
a different name."``) is preserved by the *kind* argument so error
|
|
70
|
+
text users may already script against doesn't shift.
|
|
71
|
+
"""
|
|
72
|
+
if not os.path.exists(path):
|
|
73
|
+
return
|
|
74
|
+
if kind == "folder":
|
|
75
|
+
raise EosframesError(
|
|
76
|
+
f"Output folder '{path}' already exists. "
|
|
77
|
+
"Remove it or choose a different name."
|
|
78
|
+
)
|
|
79
|
+
raise EosframesError(
|
|
80
|
+
f"Output file '{path}' already exists. Remove it first."
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _compute_summary_stats(df: pd.DataFrame) -> List[Dict]:
|
|
85
|
+
"""Per-feature summary statistics for an Ersilia output frame.
|
|
86
|
+
|
|
87
|
+
Returns one dict per non-meta column, with keys
|
|
88
|
+
``column / dtype / missing / min / mean / max``. Numeric ``min`` /
|
|
89
|
+
``mean`` / ``max`` are ``None`` for non-numeric columns or columns
|
|
90
|
+
that are fully missing — pretty-printers handle the rendering.
|
|
91
|
+
|
|
92
|
+
Lifted out of :func:`eosframes.cli.summary` so the CLI command shrinks
|
|
93
|
+
to argument parsing + Rich rendering and the math can be tested
|
|
94
|
+
directly.
|
|
95
|
+
"""
|
|
96
|
+
feature_cols = [c for c in df.columns if c not in {"key", "input"}]
|
|
97
|
+
stats_rows: List[Dict] = []
|
|
98
|
+
for col in feature_cols:
|
|
99
|
+
series = df[col]
|
|
100
|
+
row: Dict = {
|
|
101
|
+
"column": col,
|
|
102
|
+
"dtype": str(series.dtype),
|
|
103
|
+
"missing": int(series.isna().sum()),
|
|
104
|
+
"min": None,
|
|
105
|
+
"mean": None,
|
|
106
|
+
"max": None,
|
|
107
|
+
}
|
|
108
|
+
if pd.api.types.is_numeric_dtype(series):
|
|
109
|
+
clean = series.dropna()
|
|
110
|
+
if len(clean):
|
|
111
|
+
row["min"] = float(clean.min())
|
|
112
|
+
row["mean"] = float(clean.mean())
|
|
113
|
+
row["max"] = float(clean.max())
|
|
114
|
+
stats_rows.append(row)
|
|
115
|
+
return stats_rows
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _require_valid_output_name(path: str) -> Dict:
|
|
119
|
+
"""Validate a data-file output path against the naming convention.
|
|
120
|
+
|
|
121
|
+
Returns the :func:`parse_name` dict on success; raises with the
|
|
122
|
+
standard ``"Expected: <model_id>_<version>.<ext>"`` message otherwise.
|
|
123
|
+
Used by every op whose output is a single data file (convert,
|
|
124
|
+
append, dedupe).
|
|
125
|
+
"""
|
|
126
|
+
if not is_valid_name(path):
|
|
127
|
+
raise EosframesError(
|
|
128
|
+
f"Output '{path}' does not follow the naming convention. "
|
|
129
|
+
"Expected: [prefix_]<model_id>_<version>.<ext> "
|
|
130
|
+
"with ext in {csv, h5}."
|
|
131
|
+
)
|
|
132
|
+
return parse_name(path)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def split_csv(input_path: str, output_folder: str, chunksize: int = 10000) -> int:
|
|
136
|
+
"""Split a CSV file into numbered chunk files inside a folder.
|
|
137
|
+
|
|
138
|
+
This is the only operation in the module that accepts inputs without
|
|
139
|
+
a model ID — its purpose is to pre-process inputs *before* a model
|
|
140
|
+
run, when the model is not yet known. Each chunk preserves the
|
|
141
|
+
original header. The chunk index is zero-padded to a width that
|
|
142
|
+
accommodates the largest index (``chunk_000.csv`` / ``chunk_007.csv``
|
|
143
|
+
for up to 1 000 chunks; ``chunk_000000.csv`` for >1 000).
|
|
144
|
+
|
|
145
|
+
Parameters
|
|
146
|
+
----------
|
|
147
|
+
input_path : str
|
|
148
|
+
Path to the input CSV file. No naming convention required.
|
|
149
|
+
output_folder : str
|
|
150
|
+
Path to the folder to create. Must not already exist.
|
|
151
|
+
chunksize : int, default ``10000``
|
|
152
|
+
Number of rows per chunk. Smaller values produce more files;
|
|
153
|
+
larger values mean each chunk takes longer to run downstream.
|
|
154
|
+
|
|
155
|
+
Returns
|
|
156
|
+
-------
|
|
157
|
+
int
|
|
158
|
+
Number of chunk files written.
|
|
159
|
+
|
|
160
|
+
Raises
|
|
161
|
+
------
|
|
162
|
+
EosframesError
|
|
163
|
+
If the output folder already exists.
|
|
164
|
+
"""
|
|
165
|
+
logger = get_logger()
|
|
166
|
+
_require_no_overwrite(output_folder, kind="folder")
|
|
167
|
+
df = pd.read_csv(input_path)
|
|
168
|
+
total_rows = len(df)
|
|
169
|
+
num_chunks = (total_rows + chunksize - 1) // chunksize
|
|
170
|
+
zfill = len(str(max(num_chunks - 1, 0)))
|
|
171
|
+
os.makedirs(output_folder)
|
|
172
|
+
logger.info(
|
|
173
|
+
"Splitting %d rows into %d chunks (chunksize=%d) → %s",
|
|
174
|
+
total_rows,
|
|
175
|
+
num_chunks,
|
|
176
|
+
chunksize,
|
|
177
|
+
output_folder,
|
|
178
|
+
)
|
|
179
|
+
for i, chunk in enumerate(chunker(df, chunksize)):
|
|
180
|
+
fname = f"chunk_{str(i).zfill(zfill)}.csv"
|
|
181
|
+
chunk.to_csv(os.path.join(output_folder, fname), index=False)
|
|
182
|
+
logger.info("Split complete: %d chunks written to %s", num_chunks, output_folder)
|
|
183
|
+
return num_chunks
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def convert_file(input_path: str, output_path: str) -> None:
|
|
187
|
+
"""Convert a file between formats, or assemble a folder of chunks.
|
|
188
|
+
|
|
189
|
+
Supported conversions:
|
|
190
|
+
|
|
191
|
+
* Folder of chunk CSVs → ``.csv`` (concatenate)
|
|
192
|
+
* Folder of chunk CSVs → ``.h5`` (concatenate, write HDF5)
|
|
193
|
+
* ``.csv`` → ``.h5``
|
|
194
|
+
* ``.h5`` → ``.csv``
|
|
195
|
+
|
|
196
|
+
The output is always a single file (the inverse — splitting a
|
|
197
|
+
single file into chunks — is :func:`split_csv`). When the input is
|
|
198
|
+
a folder, every CSV inside it is read with raw ``pandas.read_csv``
|
|
199
|
+
(no naming-convention check on the chunk files); the model ID is
|
|
200
|
+
taken from *output_path*.
|
|
201
|
+
|
|
202
|
+
Parameters
|
|
203
|
+
----------
|
|
204
|
+
input_path : str
|
|
205
|
+
A CSV file, an H5 file, or a folder of chunk CSVs.
|
|
206
|
+
output_path : str
|
|
207
|
+
Output file path. Must follow the Ersilia naming convention
|
|
208
|
+
(``[prefix_]<model_id>_<version>.csv`` or ``.h5``).
|
|
209
|
+
|
|
210
|
+
Raises
|
|
211
|
+
------
|
|
212
|
+
EosframesError
|
|
213
|
+
On naming convention violations of *output_path*, existing
|
|
214
|
+
output, an empty input folder, or unsupported file extensions.
|
|
215
|
+
"""
|
|
216
|
+
logger = get_logger()
|
|
217
|
+
parsed = _require_valid_output_name(output_path)
|
|
218
|
+
_require_no_overwrite(output_path)
|
|
219
|
+
|
|
220
|
+
model_id = parsed["model_id"]
|
|
221
|
+
out_ext = parsed["extension"]
|
|
222
|
+
|
|
223
|
+
if os.path.isdir(input_path):
|
|
224
|
+
csv_files = sorted(f for f in os.listdir(input_path) if f.endswith(".csv"))
|
|
225
|
+
if not csv_files:
|
|
226
|
+
raise EosframesError(f"No CSV files found in '{input_path}'")
|
|
227
|
+
logger.info("Reading %d chunk files from %s", len(csv_files), input_path)
|
|
228
|
+
frames = [pd.read_csv(os.path.join(input_path, f)) for f in csv_files]
|
|
229
|
+
df = pd.concat(frames, axis=0).reset_index(drop=True)
|
|
230
|
+
else:
|
|
231
|
+
in_ext = os.path.splitext(input_path)[1].lower()
|
|
232
|
+
if in_ext == ".csv":
|
|
233
|
+
if is_valid_name(input_path):
|
|
234
|
+
df = read_csv(input_path)
|
|
235
|
+
else:
|
|
236
|
+
logger.info("Reading %s", input_path)
|
|
237
|
+
df = pd.read_csv(input_path)
|
|
238
|
+
elif in_ext == ".h5":
|
|
239
|
+
df = read_h5(input_path)
|
|
240
|
+
else:
|
|
241
|
+
raise EosframesError(
|
|
242
|
+
f"Unsupported input format '{in_ext}'. Expected .csv or .h5"
|
|
243
|
+
)
|
|
244
|
+
|
|
245
|
+
logger.info("Converting %s → %s", input_path, output_path)
|
|
246
|
+
df.model_id = model_id
|
|
247
|
+
|
|
248
|
+
if out_ext == "csv":
|
|
249
|
+
write_csv(df, output_path)
|
|
250
|
+
else:
|
|
251
|
+
write_h5(df, output_path, dtype=np.float32)
|
|
252
|
+
|
|
253
|
+
logger.info("Done: %s", output_path)
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def stack_files(input_paths: List[str], output_path: str) -> None:
|
|
257
|
+
"""Horizontally stack outputs from multiple Ersilia models into one CSV.
|
|
258
|
+
|
|
259
|
+
The *output filename* selects the column-naming mode:
|
|
260
|
+
|
|
261
|
+
* **Mode A (eosmix)** — ``[prefix_]eosmix.csv``. Feature columns are
|
|
262
|
+
suffixed with ``_<model_id>_<version>`` so column names carry the
|
|
263
|
+
provenance. The output filename does not embed the model list.
|
|
264
|
+
* **Mode B (explicit)** — ``[prefix_]<m1>_<v1>_..._<mN>_<vN>.csv``.
|
|
265
|
+
Feature columns stay bare. The output filename must list every
|
|
266
|
+
stacked ``(model_id, version)`` in the same order as
|
|
267
|
+
*input_paths*.
|
|
268
|
+
|
|
269
|
+
All input files must follow the Ersilia naming convention and
|
|
270
|
+
contain the same molecules in the same row order. Duplicate
|
|
271
|
+
``(model_id, version)`` pairs across inputs are rejected — they
|
|
272
|
+
would collide in Mode A and produce ambiguous filenames in Mode B.
|
|
273
|
+
|
|
274
|
+
Parameters
|
|
275
|
+
----------
|
|
276
|
+
input_paths : list of str
|
|
277
|
+
Two or more CSV or H5 files, each following the naming
|
|
278
|
+
convention.
|
|
279
|
+
output_path : str
|
|
280
|
+
Output CSV path. Must follow either Mode A or Mode B.
|
|
281
|
+
|
|
282
|
+
Raises
|
|
283
|
+
------
|
|
284
|
+
EosframesError
|
|
285
|
+
On naming violations, duplicate ``(model_id, version)`` pairs,
|
|
286
|
+
Mode B model-order mismatch between filename and input order,
|
|
287
|
+
pre-existing output, or input row mismatch.
|
|
288
|
+
"""
|
|
289
|
+
logger = get_logger()
|
|
290
|
+
if len(input_paths) < 2:
|
|
291
|
+
raise EosframesError("At least two input files are required for stacking.")
|
|
292
|
+
_require_no_overwrite(output_path)
|
|
293
|
+
|
|
294
|
+
dfs = []
|
|
295
|
+
input_pairs: List[Tuple[str, str]] = [] # (model_id, version) per input
|
|
296
|
+
for path in input_paths:
|
|
297
|
+
parsed = parse_name(path)
|
|
298
|
+
if parsed is None or parsed["name_type"] not in {"csv", "h5"}:
|
|
299
|
+
raise EosframesError(
|
|
300
|
+
f"'{path}' does not follow the naming convention. "
|
|
301
|
+
"Expected: [prefix_]<model_id>_<version>.<ext> "
|
|
302
|
+
"with ext in {csv, h5}."
|
|
303
|
+
)
|
|
304
|
+
logger.info("Reading %s", path)
|
|
305
|
+
df = _read_file(path)
|
|
306
|
+
input_pairs.append((parsed["model_id"], parsed["version"]))
|
|
307
|
+
dfs.append(df)
|
|
308
|
+
|
|
309
|
+
# Resolve mode from the output filename.
|
|
310
|
+
mix_suggestion = make_stack_mix_name()
|
|
311
|
+
explicit_suggestion = make_stack_explicit_name(input_pairs)
|
|
312
|
+
|
|
313
|
+
if is_valid_stack_mix_name(output_path):
|
|
314
|
+
mode = "eosmix"
|
|
315
|
+
elif is_valid_stack_explicit_name(output_path):
|
|
316
|
+
out_parsed = parse_stack_explicit_name(output_path) or {"models": []}
|
|
317
|
+
out_pairs = out_parsed["models"]
|
|
318
|
+
if out_pairs != input_pairs:
|
|
319
|
+
raise EosframesError(
|
|
320
|
+
f"Model order mismatch in output filename '{os.path.basename(output_path)}'.\n"
|
|
321
|
+
f" From --input: {input_pairs}\n"
|
|
322
|
+
f" From --output: {out_pairs}\n"
|
|
323
|
+
"The output filename must list each (model_id, version) in the "
|
|
324
|
+
"same order as the inputs.\n"
|
|
325
|
+
f"Try: {explicit_suggestion}"
|
|
326
|
+
)
|
|
327
|
+
mode = "explicit"
|
|
328
|
+
else:
|
|
329
|
+
raise EosframesError(
|
|
330
|
+
f"'{os.path.basename(output_path)}' does not follow a stack "
|
|
331
|
+
"naming convention.\n\n"
|
|
332
|
+
"Choose exactly one of:\n"
|
|
333
|
+
" Mode A (eosmix): [prefix]_eosmix.csv\n"
|
|
334
|
+
" Feature columns are suffixed with _<model_id>_<version>.\n"
|
|
335
|
+
f" Try: {mix_suggestion}\n\n"
|
|
336
|
+
" Mode B (explicit): [prefix]_<m1>_<v1>_..._<mN>_<vN>.csv\n"
|
|
337
|
+
" Each stacked (model_id, version) appears in the filename "
|
|
338
|
+
"in -i order. Columns stay bare.\n"
|
|
339
|
+
f" Try: {explicit_suggestion}"
|
|
340
|
+
)
|
|
341
|
+
|
|
342
|
+
result = hstack(dfs, mode=mode)
|
|
343
|
+
|
|
344
|
+
meta_cols = [c for c in ("key", "input") if c in result.columns]
|
|
345
|
+
logger.info(
|
|
346
|
+
"Stacked %d files × %d rows → %d feature columns (mode=%s)",
|
|
347
|
+
len(dfs),
|
|
348
|
+
len(result),
|
|
349
|
+
len(result.columns) - len(meta_cols),
|
|
350
|
+
mode,
|
|
351
|
+
)
|
|
352
|
+
result.to_csv(output_path, index=False)
|
|
353
|
+
logger.info("Done: %s", output_path)
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def append_files(input_paths: List[str], output_path: str) -> None:
|
|
357
|
+
"""Vertically concatenate files from the same Ersilia model.
|
|
358
|
+
|
|
359
|
+
All input files must share the same model ID (encoded in their
|
|
360
|
+
filenames) and have identical column layouts. Rows are appended in
|
|
361
|
+
the order given. Duplicate keys, if any, are *not* removed —
|
|
362
|
+
follow with :func:`dedupe_file` if needed.
|
|
363
|
+
|
|
364
|
+
Parameters
|
|
365
|
+
----------
|
|
366
|
+
input_paths : list of str
|
|
367
|
+
Two or more CSV or H5 files. Each must follow the naming
|
|
368
|
+
convention.
|
|
369
|
+
output_path : str
|
|
370
|
+
Output file path. Must follow the naming convention; its
|
|
371
|
+
encoded model ID determines the expected model ID for all
|
|
372
|
+
inputs.
|
|
373
|
+
|
|
374
|
+
Raises
|
|
375
|
+
------
|
|
376
|
+
EosframesError
|
|
377
|
+
On invalid output naming, pre-existing output, model-ID
|
|
378
|
+
mismatch between an input and the output, or column mismatch
|
|
379
|
+
across inputs.
|
|
380
|
+
"""
|
|
381
|
+
logger = get_logger()
|
|
382
|
+
if len(input_paths) < 2:
|
|
383
|
+
raise EosframesError("At least two input files are required for appending.")
|
|
384
|
+
out_parsed = _require_valid_output_name(output_path)
|
|
385
|
+
_require_no_overwrite(output_path)
|
|
386
|
+
|
|
387
|
+
expected_model_id = out_parsed["model_id"]
|
|
388
|
+
out_ext = out_parsed["extension"]
|
|
389
|
+
|
|
390
|
+
dfs = []
|
|
391
|
+
reference_columns = None
|
|
392
|
+
for path in input_paths:
|
|
393
|
+
logger.info("Reading %s", path)
|
|
394
|
+
df = _read_file(path)
|
|
395
|
+
model_id = getattr(df, "model_id", None)
|
|
396
|
+
if model_id != expected_model_id:
|
|
397
|
+
raise EosframesError(
|
|
398
|
+
f"Model ID mismatch: '{path}' has model '{model_id}' "
|
|
399
|
+
f"but output expects '{expected_model_id}'."
|
|
400
|
+
)
|
|
401
|
+
cols = list(df.columns)
|
|
402
|
+
if reference_columns is None:
|
|
403
|
+
reference_columns = cols
|
|
404
|
+
elif cols != reference_columns:
|
|
405
|
+
raise EosframesError(
|
|
406
|
+
f"Column mismatch: '{path}' has columns {cols} "
|
|
407
|
+
f"but expected {reference_columns}."
|
|
408
|
+
)
|
|
409
|
+
dfs.append(df)
|
|
410
|
+
|
|
411
|
+
result = pd.concat(dfs, axis=0).reset_index(drop=True)
|
|
412
|
+
result.model_id = expected_model_id
|
|
413
|
+
logger.info("Appended %d files → %d rows total", len(dfs), len(result))
|
|
414
|
+
|
|
415
|
+
if out_ext == "csv":
|
|
416
|
+
write_csv(result, output_path)
|
|
417
|
+
else:
|
|
418
|
+
write_h5(result, output_path, dtype=np.float32)
|
|
419
|
+
|
|
420
|
+
logger.info("Done: %s", output_path)
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def dedupe_file(input_path: str, output_path: str) -> Tuple[int, int]:
|
|
424
|
+
"""Remove duplicate rows by ``key``, keeping the first occurrence.
|
|
425
|
+
|
|
426
|
+
Parameters
|
|
427
|
+
----------
|
|
428
|
+
input_path : str
|
|
429
|
+
Input CSV or H5 file. Must follow the naming convention and
|
|
430
|
+
contain a ``key`` column.
|
|
431
|
+
output_path : str
|
|
432
|
+
Output file path. Must follow the naming convention; its
|
|
433
|
+
encoded model ID must match the input.
|
|
434
|
+
|
|
435
|
+
Returns
|
|
436
|
+
-------
|
|
437
|
+
rows_before : int
|
|
438
|
+
Row count of the input file.
|
|
439
|
+
rows_after : int
|
|
440
|
+
Row count after deduplication. ``rows_before - rows_after`` is
|
|
441
|
+
the number of duplicate rows that were dropped.
|
|
442
|
+
|
|
443
|
+
Raises
|
|
444
|
+
------
|
|
445
|
+
EosframesError
|
|
446
|
+
On naming convention violations, pre-existing output,
|
|
447
|
+
model-ID mismatch, or a missing ``key`` column.
|
|
448
|
+
"""
|
|
449
|
+
logger = get_logger()
|
|
450
|
+
out_parsed = _require_valid_output_name(output_path)
|
|
451
|
+
_require_no_overwrite(output_path)
|
|
452
|
+
|
|
453
|
+
expected_model_id = out_parsed["model_id"]
|
|
454
|
+
out_ext = out_parsed["extension"]
|
|
455
|
+
|
|
456
|
+
logger.info("Reading %s", input_path)
|
|
457
|
+
df = _read_file(input_path)
|
|
458
|
+
|
|
459
|
+
model_id = getattr(df, "model_id", None)
|
|
460
|
+
if model_id != expected_model_id:
|
|
461
|
+
raise EosframesError(
|
|
462
|
+
f"Model ID mismatch: '{input_path}' has model '{model_id}' "
|
|
463
|
+
f"but output expects '{expected_model_id}'."
|
|
464
|
+
)
|
|
465
|
+
if "key" not in df.columns:
|
|
466
|
+
raise EosframesError(f"'{input_path}' does not contain a 'key' column.")
|
|
467
|
+
|
|
468
|
+
before = len(df)
|
|
469
|
+
df = df.drop_duplicates(subset="key", keep="first").reset_index(drop=True)
|
|
470
|
+
after = len(df)
|
|
471
|
+
logger.info("Removed %d duplicate(s), %d rows remaining", before - after, after)
|
|
472
|
+
|
|
473
|
+
df.model_id = expected_model_id
|
|
474
|
+
|
|
475
|
+
if out_ext == "csv":
|
|
476
|
+
write_csv(df, output_path)
|
|
477
|
+
else:
|
|
478
|
+
write_h5(df, output_path, dtype=np.float32)
|
|
479
|
+
|
|
480
|
+
logger.info("Done: %s", output_path)
|
|
481
|
+
return before, after
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
# Matches an eosmix column suffix at the END of a name:
|
|
485
|
+
# "<original>_<model_id>_<version>" where model_id = eos<d><3 alnum>, version = v<digits>
|
|
486
|
+
_EOSMIX_COL_RE = re.compile(
|
|
487
|
+
r"^(?P<original>.+)_(?P<model_id>eos\d[A-Za-z0-9]{3})_(?P<version>v\d+)$"
|
|
488
|
+
)
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def _classify_stack_columns_mode_a(
|
|
492
|
+
feature_cols: List[str],
|
|
493
|
+
) -> List[Tuple[str, str, str, str]]:
|
|
494
|
+
"""Parse Mode A feature columns into (original, model_id, version, suffixed).
|
|
495
|
+
|
|
496
|
+
Raises ``EosframesError`` listing any columns whose name doesn't match
|
|
497
|
+
the eosmix suffix pattern.
|
|
498
|
+
"""
|
|
499
|
+
parsed = []
|
|
500
|
+
bad: List[str] = []
|
|
501
|
+
for col in feature_cols:
|
|
502
|
+
m = _EOSMIX_COL_RE.match(col)
|
|
503
|
+
if not m or not is_model_id_valid(m.group("model_id")):
|
|
504
|
+
bad.append(col)
|
|
505
|
+
continue
|
|
506
|
+
parsed.append(
|
|
507
|
+
(m.group("original"), m.group("model_id"), m.group("version"), col)
|
|
508
|
+
)
|
|
509
|
+
if bad:
|
|
510
|
+
raise EosframesError(
|
|
511
|
+
"Some feature columns do not follow the Mode A suffix pattern "
|
|
512
|
+
"'<original>_<model_id>_<version>':\n "
|
|
513
|
+
+ ", ".join(bad[:10])
|
|
514
|
+
+ (" ..." if len(bad) > 10 else "")
|
|
515
|
+
)
|
|
516
|
+
return parsed
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
def _classify_stack_columns_mode_b(
|
|
520
|
+
feature_cols: List[str], models: List[Tuple[str, str]]
|
|
521
|
+
) -> List[Tuple[str, str, str]]:
|
|
522
|
+
"""Assign each bare feature column to a (model_id, version) via run_columns.csv.
|
|
523
|
+
|
|
524
|
+
Fetches run_columns.csv for each model, checks that every feature column
|
|
525
|
+
maps to exactly one model, and returns ``[(col, model_id, version), ...]``.
|
|
526
|
+
|
|
527
|
+
Raises ``EosframesError`` on ambiguous (a column listed for 2+ models),
|
|
528
|
+
unmatched (a column not listed for any model), or missing (a model's
|
|
529
|
+
run_columns lists columns not present in the stack) cases.
|
|
530
|
+
"""
|
|
531
|
+
# Fetch run_columns.csv for each model → dict of model -> set of column names.
|
|
532
|
+
model_cols: List[Tuple[Tuple[str, str], set]] = []
|
|
533
|
+
for model_id, version in models:
|
|
534
|
+
df_cols = hub.fetch_columns(model_id, version)
|
|
535
|
+
if "name" not in df_cols.columns:
|
|
536
|
+
raise EosframesError(
|
|
537
|
+
f"run_columns.csv for {model_id} {version} is missing a 'name' column."
|
|
538
|
+
)
|
|
539
|
+
model_cols.append(((model_id, version), set(df_cols["name"].astype(str))))
|
|
540
|
+
|
|
541
|
+
feat_set = set(feature_cols)
|
|
542
|
+
assignments: List[Tuple[str, str, str]] = []
|
|
543
|
+
|
|
544
|
+
# 1. Check ambiguity (a column listed for 2+ models) and unmatched (a
|
|
545
|
+
# column not listed for any model).
|
|
546
|
+
column_owners: dict = {}
|
|
547
|
+
for (model_id, version), cols in model_cols:
|
|
548
|
+
for c in cols:
|
|
549
|
+
column_owners.setdefault(c, []).append((model_id, version))
|
|
550
|
+
ambiguous = {
|
|
551
|
+
c: owners
|
|
552
|
+
for c, owners in column_owners.items()
|
|
553
|
+
if len(owners) > 1 and c in feat_set
|
|
554
|
+
}
|
|
555
|
+
if ambiguous:
|
|
556
|
+
msg_lines = [f" {c!r}: {owners}" for c, owners in list(ambiguous.items())[:5]]
|
|
557
|
+
raise EosframesError(
|
|
558
|
+
"Ambiguous columns — the following appear in run_columns.csv of multiple models:\n"
|
|
559
|
+
+ "\n".join(msg_lines)
|
|
560
|
+
)
|
|
561
|
+
unmatched = [c for c in feature_cols if c not in column_owners]
|
|
562
|
+
if unmatched:
|
|
563
|
+
raise EosframesError(
|
|
564
|
+
"Unmatched feature columns — none of the stacked models' "
|
|
565
|
+
"run_columns.csv lists these:\n "
|
|
566
|
+
+ ", ".join(unmatched[:10])
|
|
567
|
+
+ (" ..." if len(unmatched) > 10 else "")
|
|
568
|
+
)
|
|
569
|
+
|
|
570
|
+
# 2. Check every model's expected columns are present in the stack.
|
|
571
|
+
missing_report = []
|
|
572
|
+
for (model_id, version), cols in model_cols:
|
|
573
|
+
missing = [c for c in cols if c not in feat_set]
|
|
574
|
+
if missing:
|
|
575
|
+
missing_report.append(
|
|
576
|
+
f" {model_id} {version} is missing: {', '.join(missing[:10])}"
|
|
577
|
+
+ (" ..." if len(missing) > 10 else "")
|
|
578
|
+
)
|
|
579
|
+
if missing_report:
|
|
580
|
+
raise EosframesError(
|
|
581
|
+
"Stack file is missing feature columns required by the models' run_columns.csv:\n"
|
|
582
|
+
+ "\n".join(missing_report)
|
|
583
|
+
)
|
|
584
|
+
|
|
585
|
+
# 3. Build the assignment in the input order, respecting the model list order.
|
|
586
|
+
for (model_id, version), cols in model_cols:
|
|
587
|
+
for c in feature_cols:
|
|
588
|
+
if c in cols:
|
|
589
|
+
assignments.append((c, model_id, version))
|
|
590
|
+
|
|
591
|
+
return assignments
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
def unstack_file(input_path: str, output_folder: str) -> List[str]:
|
|
595
|
+
"""Split a horizontally stacked CSV back into per-model files.
|
|
596
|
+
|
|
597
|
+
The mode is resolved from the input filename:
|
|
598
|
+
|
|
599
|
+
* Mode A (``[prefix]_eosmix.csv``) — column names carry the model
|
|
600
|
+
provenance. Columns are grouped by the ``_<model_id>_<version>``
|
|
601
|
+
suffix; the suffix is stripped when writing each per-model file.
|
|
602
|
+
* Mode B (``[prefix]_<m1>_<v1>_..._<mN>_<vN>.csv``) — column names are
|
|
603
|
+
bare. Each model's ``run_columns.csv`` is fetched from GitHub
|
|
604
|
+
(via :func:`eosframes.fetch_columns`) and columns are distributed
|
|
605
|
+
by name.
|
|
606
|
+
|
|
607
|
+
The output folder must not already exist and is created fresh. Each
|
|
608
|
+
per-model file is written as ``<prefix>_<model_id>_<version>.csv`` with
|
|
609
|
+
``prefix`` inherited from the stacked filename (dropped when the input
|
|
610
|
+
is unprefixed).
|
|
611
|
+
|
|
612
|
+
Parameters
|
|
613
|
+
----------
|
|
614
|
+
input_path : str
|
|
615
|
+
Path to a stacked CSV (Mode A or Mode B).
|
|
616
|
+
output_folder : str
|
|
617
|
+
Destination folder; must not exist.
|
|
618
|
+
|
|
619
|
+
Returns
|
|
620
|
+
-------
|
|
621
|
+
list of str
|
|
622
|
+
Absolute paths of the per-model files that were written, in
|
|
623
|
+
the order of the stacked models.
|
|
624
|
+
|
|
625
|
+
Raises
|
|
626
|
+
------
|
|
627
|
+
EosframesError
|
|
628
|
+
On invalid filename, missing ``key`` / ``input`` columns, ambiguous
|
|
629
|
+
or unmatched columns (Mode B), or pre-existing output folder.
|
|
630
|
+
"""
|
|
631
|
+
logger = get_logger()
|
|
632
|
+
_require_no_overwrite(output_folder, kind="folder")
|
|
633
|
+
|
|
634
|
+
mix = parse_stack_mix_name(input_path)
|
|
635
|
+
explicit = parse_stack_explicit_name(input_path)
|
|
636
|
+
if mix is not None:
|
|
637
|
+
mode = "eosmix"
|
|
638
|
+
prefix = mix["prefix"]
|
|
639
|
+
elif explicit is not None:
|
|
640
|
+
mode = "explicit"
|
|
641
|
+
prefix = explicit["prefix"]
|
|
642
|
+
else:
|
|
643
|
+
raise EosframesError(
|
|
644
|
+
f"'{os.path.basename(input_path)}' does not follow a stack naming "
|
|
645
|
+
"convention.\n\n"
|
|
646
|
+
"Expected one of:\n"
|
|
647
|
+
" Mode A: [prefix]_eosmix.csv\n"
|
|
648
|
+
" Mode B: [prefix]_<m1>_<v1>_..._<mN>_<vN>.csv"
|
|
649
|
+
)
|
|
650
|
+
|
|
651
|
+
logger.info("Reading stacked CSV: %s (mode=%s)", input_path, mode)
|
|
652
|
+
df = pd.read_csv(input_path)
|
|
653
|
+
for col in ("key", "input"):
|
|
654
|
+
if col not in df.columns:
|
|
655
|
+
raise EosframesError(
|
|
656
|
+
f"'{input_path}' is missing the required '{col}' column."
|
|
657
|
+
)
|
|
658
|
+
feature_cols = [c for c in df.columns if c not in {"key", "input"}]
|
|
659
|
+
|
|
660
|
+
# Assemble per-(model, version) column lists. Each assignment entry is a
|
|
661
|
+
# tuple of stacked_col_name, model_id, version, output_col_name.
|
|
662
|
+
assignments: List[Tuple[str, str, str, str]]
|
|
663
|
+
if mode == "eosmix":
|
|
664
|
+
parsed = _classify_stack_columns_mode_a(feature_cols)
|
|
665
|
+
assignments = [
|
|
666
|
+
(stacked, mid, ver, original) for original, mid, ver, stacked in parsed
|
|
667
|
+
]
|
|
668
|
+
else:
|
|
669
|
+
pairs = explicit["models"]
|
|
670
|
+
mode_b = _classify_stack_columns_mode_b(feature_cols, pairs)
|
|
671
|
+
# Column names stay as-is in Mode B.
|
|
672
|
+
assignments = [(col, mid, ver, col) for col, mid, ver in mode_b]
|
|
673
|
+
|
|
674
|
+
# Group assignments by (model_id, version), preserving model order.
|
|
675
|
+
per_model: Dict[Tuple[str, str], List[Tuple[str, str]]] = {}
|
|
676
|
+
order: List[Tuple[str, str]] = []
|
|
677
|
+
for stacked, mid, ver, output_name in assignments:
|
|
678
|
+
key = (mid, ver)
|
|
679
|
+
if key not in per_model:
|
|
680
|
+
per_model[key] = []
|
|
681
|
+
order.append(key)
|
|
682
|
+
per_model[key].append((stacked, output_name))
|
|
683
|
+
|
|
684
|
+
# Create the destination folder and write each per-model CSV.
|
|
685
|
+
os.makedirs(output_folder)
|
|
686
|
+
written: List[str] = []
|
|
687
|
+
for model_id, version in order:
|
|
688
|
+
cols = per_model[(model_id, version)]
|
|
689
|
+
stacked_names = [s for s, _ in cols]
|
|
690
|
+
output_names = [o for _, o in cols]
|
|
691
|
+
sub = df[["key", "input", *stacked_names]].copy()
|
|
692
|
+
# Rename the suffixed columns back to their original names.
|
|
693
|
+
sub.rename(columns=dict(zip(stacked_names, output_names)), inplace=True)
|
|
694
|
+
sub.model_id = model_id
|
|
695
|
+
sub.version = version
|
|
696
|
+
out_basename = (
|
|
697
|
+
f"{prefix}_{model_id}_{version}.csv"
|
|
698
|
+
if prefix
|
|
699
|
+
else f"{model_id}_{version}.csv"
|
|
700
|
+
)
|
|
701
|
+
out_path = os.path.abspath(os.path.join(output_folder, out_basename))
|
|
702
|
+
write_csv(sub, out_path)
|
|
703
|
+
written.append(out_path)
|
|
704
|
+
|
|
705
|
+
logger.info(
|
|
706
|
+
"Unstacked %s → %d per-model files in %s",
|
|
707
|
+
input_path,
|
|
708
|
+
len(written),
|
|
709
|
+
output_folder,
|
|
710
|
+
)
|
|
711
|
+
return written
|