jarvisplot 1.3.2__tar.gz → 1.3.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/JarvisPLOT.egg-info/PKG-INFO +1 -1
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/JarvisPLOT.egg-info/SOURCES.txt +1 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/PKG-INFO +1 -1
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/config_runtime.py +4 -3
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/load_data.py +69 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/preprocessor_runtime.py +9 -5
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cache_store.py +7 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/core.py +23 -6
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/core_runtime.py +81 -1
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/data_loader.py +35 -5
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/data_loader_hdf5.py +113 -2
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/data_loader_runtime.py +271 -50
- jarvisplot-1.3.3/jarvisplot/data_loader_summary.py +164 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/pyproject.toml +1 -1
- jarvisplot-1.3.3/tests/test_data_loader_contracts.py +445 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/tests/test_figure_contracts.py +48 -0
- jarvisplot-1.3.3/tests/test_parse_data_metadata.py +153 -0
- jarvisplot-1.3.2/jarvisplot/data_loader_summary.py +0 -83
- jarvisplot-1.3.2/tests/test_data_loader_contracts.py +0 -96
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/JarvisPLOT.egg-info/dependency_links.txt +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/JarvisPLOT.egg-info/entry_points.txt +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/JarvisPLOT.egg-info/requires.txt +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/JarvisPLOT.egg-info/top_level.txt +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/README.md +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/__init__.py +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/adapters.py +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/adapters_rect.py +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/adapters_ternary.py +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/cards/std_axes_adapter_config.json +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/colorbar_runtime.py +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/data_pipelines.py +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/figure.py +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/helper.py +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/layer_runtime.py +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/layout_runtime.py +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/method_registry.py +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/preprocessor.py +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/profile_runtime.py +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/style_runtime.py +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/__init__.py +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/a4paper/1x1/ternary.json +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/a4paper/2x1/rect.json +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/a4paper/2x1/rect5x1.json +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/a4paper/2x1/rect_cmap.json +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/a4paper/2x1/ternary.json +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/a4paper/2x1/ternary_cmap.json +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/a4paper/4x1/rect.json +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/a4paper/4x1/rect_cmap.json +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/a4paper/4x1/ternary.json +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/a4paper/4x1/ternary_cmap.json +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/args.json +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/colors/colormaps.json +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/default/output.json +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/gambit/1x1/ternary.json +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/gambit/2x1/rect_cmap.json +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/gambit/2x1/ternary.json +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/gambit/2x1/ternary_cmap.json +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/icons/JarvisHEP.png +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/icons/gambit.png +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/icons/gambit_small.png +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/style_preference.json +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cli.py +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/client.py +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/config.py +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/core_assets.py +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/inner_func.py +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/memtrace.py +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/utils/__init__.py +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/utils/cmaps.py +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/utils/expression.py +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/utils/interpolator.py +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/utils/pathing.py +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/setup.cfg +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/tests/test_docs_status_labels.py +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/tests/test_pathing_contracts.py +0 -0
- {jarvisplot-1.3.2 → jarvisplot-1.3.3}/tests/test_template_json.py +0 -0
|
@@ -39,11 +39,12 @@ def apply_figure_config(fig, info: Mapping) -> bool:
|
|
|
39
39
|
return False
|
|
40
40
|
|
|
41
41
|
if "style" in info:
|
|
42
|
-
|
|
42
|
+
style_tokens = info["style"]
|
|
43
43
|
else:
|
|
44
|
-
|
|
44
|
+
style_tokens = ["a4paper_2x1"]
|
|
45
|
+
fig.style = style_tokens
|
|
45
46
|
fig.logger.debug("Figure style loaded")
|
|
46
|
-
if
|
|
47
|
+
if style_tokens and "gambit" in str(style_tokens[0]).lower():
|
|
47
48
|
fig.mode = "gambit"
|
|
48
49
|
|
|
49
50
|
if "frame" in info:
|
|
@@ -6,6 +6,28 @@ from ..utils.expression import eval_dataframe_expression
|
|
|
6
6
|
from .profile_runtime import eval_series, grid_profiling, profiling, _preprofiling
|
|
7
7
|
|
|
8
8
|
|
|
9
|
+
def _normalize_column_list(spec):
|
|
10
|
+
if spec is None:
|
|
11
|
+
return []
|
|
12
|
+
if isinstance(spec, dict):
|
|
13
|
+
for key in ("columns", "keep", "drop", "select", "retain", "value"):
|
|
14
|
+
if key in spec:
|
|
15
|
+
return _normalize_column_list(spec.get(key))
|
|
16
|
+
return []
|
|
17
|
+
if isinstance(spec, str):
|
|
18
|
+
text = spec.strip()
|
|
19
|
+
return [text] if text else []
|
|
20
|
+
if isinstance(spec, (list, tuple, set)):
|
|
21
|
+
out = []
|
|
22
|
+
for item in spec:
|
|
23
|
+
sval = str(item).strip()
|
|
24
|
+
if sval:
|
|
25
|
+
out.append(sval)
|
|
26
|
+
return out
|
|
27
|
+
sval = str(spec).strip()
|
|
28
|
+
return [sval] if sval else []
|
|
29
|
+
|
|
30
|
+
|
|
9
31
|
def filter(df, condition, logger):
|
|
10
32
|
try:
|
|
11
33
|
if isinstance(condition, bool):
|
|
@@ -55,6 +77,53 @@ def addcolumn(df, adds, logger):
|
|
|
55
77
|
)
|
|
56
78
|
)
|
|
57
79
|
return df
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def keep_columns(df, spec, logger):
|
|
83
|
+
try:
|
|
84
|
+
cols = _normalize_column_list(spec)
|
|
85
|
+
if not cols:
|
|
86
|
+
return df
|
|
87
|
+
if isinstance(df, pd.DataFrame):
|
|
88
|
+
keep = [c for c in cols if c in df.columns]
|
|
89
|
+
missing = [c for c in cols if c not in df.columns]
|
|
90
|
+
if missing and logger:
|
|
91
|
+
logger.warning(f"keep_columns missing columns ignored -> {missing}")
|
|
92
|
+
if not keep:
|
|
93
|
+
return df.iloc[0:0].copy()
|
|
94
|
+
return df.loc[:, keep].copy(deep=False)
|
|
95
|
+
if hasattr(df, "select"):
|
|
96
|
+
try:
|
|
97
|
+
return df.select(cols)
|
|
98
|
+
except Exception:
|
|
99
|
+
return df
|
|
100
|
+
return df
|
|
101
|
+
except Exception as e:
|
|
102
|
+
if logger:
|
|
103
|
+
logger.warning(f"keep_columns failed for spec={spec}: {e}")
|
|
104
|
+
return df
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def drop_columns(df, spec, logger):
|
|
108
|
+
try:
|
|
109
|
+
cols = _normalize_column_list(spec)
|
|
110
|
+
if not cols:
|
|
111
|
+
return df
|
|
112
|
+
if isinstance(df, pd.DataFrame):
|
|
113
|
+
existing = [c for c in cols if c in df.columns]
|
|
114
|
+
if not existing:
|
|
115
|
+
return df
|
|
116
|
+
return df.drop(columns=existing, errors="ignore")
|
|
117
|
+
if hasattr(df, "drop"):
|
|
118
|
+
try:
|
|
119
|
+
return df.drop(cols)
|
|
120
|
+
except Exception:
|
|
121
|
+
return df
|
|
122
|
+
return df
|
|
123
|
+
except Exception as e:
|
|
124
|
+
if logger:
|
|
125
|
+
logger.warning(f"drop_columns failed for spec={spec}: {e}")
|
|
126
|
+
return df
|
|
58
127
|
|
|
59
128
|
def sortby(df, expr, logger):
|
|
60
129
|
try:
|
|
@@ -13,7 +13,7 @@ except Exception:
|
|
|
13
13
|
pl = None
|
|
14
14
|
|
|
15
15
|
from ..memtrace import memtrace_checkpoint, memtrace_object_inventory
|
|
16
|
-
from .load_data import _preprofiling, addcolumn, filter as filter_df, grid_profiling, profiling, sortby
|
|
16
|
+
from .load_data import _preprofiling, addcolumn, drop_columns, filter as filter_df, grid_profiling, keep_columns, profiling, sortby
|
|
17
17
|
|
|
18
18
|
def resolve_source_data(preprocessor, source: Any, combine: str = "concat"):
|
|
19
19
|
if isinstance(source, str):
|
|
@@ -130,7 +130,7 @@ def apply_transforms_impl(
|
|
|
130
130
|
method = str(profile_cfg.get("method", "bridson")).lower()
|
|
131
131
|
if "bin" in profile_cfg:
|
|
132
132
|
binv = profile_cfg.get("bin")
|
|
133
|
-
preprocessor.
|
|
133
|
+
preprocessor._info(
|
|
134
134
|
"Runtime profile START:\n\t source \t-> {}\n\t step \t\t-> profile, \n\t method \t-> {}\n\t bin \t\t-> {}\n\t rows_before \t-> {}".format(
|
|
135
135
|
source_label or "<unknown>",
|
|
136
136
|
method,
|
|
@@ -183,7 +183,7 @@ def apply_transforms_impl(
|
|
|
183
183
|
profile_cfg = {"method": "grid"}
|
|
184
184
|
before_rows = preprocessor._safe_nrows(df)
|
|
185
185
|
binv = profile_cfg.get("bin", "default")
|
|
186
|
-
preprocessor.
|
|
186
|
+
preprocessor._info(
|
|
187
187
|
"Runtime profile START: source \t-> {}\n\t step \t-> 'grid_profile,\n\t method \t-> 'grid',\n\t bin \t-> {},\n\t rows_before \t-> {}".format(
|
|
188
188
|
source_label or "<unknown>",
|
|
189
189
|
binv,
|
|
@@ -227,6 +227,10 @@ def apply_transforms_impl(
|
|
|
227
227
|
df = sortby(df, trans["sortby"], preprocessor.logger)
|
|
228
228
|
elif "add_column" in trans:
|
|
229
229
|
df = addcolumn(df, trans["add_column"], preprocessor.logger)
|
|
230
|
+
elif "keep_columns" in trans:
|
|
231
|
+
df = keep_columns(df, trans.get("keep_columns"), preprocessor.logger)
|
|
232
|
+
elif "drop_columns" in trans:
|
|
233
|
+
df = drop_columns(df, trans.get("drop_columns"), preprocessor.logger)
|
|
230
234
|
|
|
231
235
|
if prev_df is not df:
|
|
232
236
|
collect_prev = preprocessor._should_collect_dataframe(prev_df)
|
|
@@ -384,7 +388,7 @@ def run_pipeline(preprocessor,
|
|
|
384
388
|
src_label = preprocessor._runtime_source_label(source)
|
|
385
389
|
if isinstance(source, str) and source in preprocessor._preprofile_alias_meta:
|
|
386
390
|
meta = preprocessor._preprofile_alias_meta.get(source, {})
|
|
387
|
-
preprocessor.
|
|
391
|
+
preprocessor._info(
|
|
388
392
|
"Runtime profile input:\n\t source \t-> {},\n uses preprofile alias:\n\t key \t\t-> {},\n\t origin \t-> {},\n\t cache_file \t-> {},\n\t rows_in \t-> {}.".format(
|
|
389
393
|
src_label,
|
|
390
394
|
meta.get("pre_key", "<unknown>")[:16] if isinstance(meta.get("pre_key"), str) else "<unknown>",
|
|
@@ -457,7 +461,7 @@ def run_pipeline(preprocessor,
|
|
|
457
461
|
cache_file = str((preprocessor.cache.data_dir / f"{key}.pkl").resolve())
|
|
458
462
|
except Exception:
|
|
459
463
|
pass
|
|
460
|
-
preprocessor.
|
|
464
|
+
preprocessor._info(
|
|
461
465
|
"Runtime profile cache STORE:\n\t source \t-> {},\n\t key \t\t-> {},\n\t cache_file \t-> {},\n\t rows \t\t-> {}.".format(
|
|
462
466
|
preprocessor._runtime_source_label(source),
|
|
463
467
|
key[:16],
|
|
@@ -62,6 +62,13 @@ class ProjectCache:
|
|
|
62
62
|
except Exception:
|
|
63
63
|
pass
|
|
64
64
|
|
|
65
|
+
def _info(self, msg: str) -> None:
|
|
66
|
+
if self.logger:
|
|
67
|
+
try:
|
|
68
|
+
self.logger.info(msg)
|
|
69
|
+
except Exception:
|
|
70
|
+
pass
|
|
71
|
+
|
|
65
72
|
@staticmethod
|
|
66
73
|
def _load_json(path: Path, default: Dict[str, Any]) -> Dict[str, Any]:
|
|
67
74
|
try:
|
|
@@ -23,10 +23,17 @@ from .core_runtime import (
|
|
|
23
23
|
plan_dataset_required_columns as runtime_plan_dataset_required_columns,
|
|
24
24
|
prepare_project_layout as runtime_prepare_project_layout,
|
|
25
25
|
prepare_usage_plan as runtime_prepare_usage_plan,
|
|
26
|
+
parse_hdf5_metadata_and_renew_yaml as runtime_parse_hdf5_metadata_and_renew_yaml,
|
|
26
27
|
rename_hdf5_and_renew_yaml as runtime_rename_hdf5_and_renew_yaml,
|
|
27
28
|
)
|
|
28
29
|
|
|
29
30
|
|
|
31
|
+
def _format_console_record(record):
|
|
32
|
+
module = record["extra"].get("module", "No module")
|
|
33
|
+
message = str(record["message"]).replace("{", "{{").replace("}", "}}").replace("<", "\\<")
|
|
34
|
+
return f"\n\n<cyan>{module}</cyan> \n\t-> <green>{record['time']:MM-DD HH:mm:ss.SSS}</green> - [<level>{record['level']}</level>] >>> \n<level>{message}</level> "
|
|
35
|
+
|
|
36
|
+
|
|
30
37
|
class JarvisPLOT():
|
|
31
38
|
def __init__(self) -> None:
|
|
32
39
|
self.variables = {}
|
|
@@ -53,7 +60,6 @@ class JarvisPLOT():
|
|
|
53
60
|
self.load_cmaps()
|
|
54
61
|
|
|
55
62
|
self.load_yaml()
|
|
56
|
-
self.prepare_project_layout()
|
|
57
63
|
|
|
58
64
|
# sys.exit()
|
|
59
65
|
if self.args.parse_data:
|
|
@@ -64,9 +70,10 @@ class JarvisPLOT():
|
|
|
64
70
|
elif self.args.out is not None and self.args.inplace:
|
|
65
71
|
self.logger.error("Conflicting arguments: --out and --inplace. Please choose only one.")
|
|
66
72
|
sys.exit(2)
|
|
67
|
-
self.
|
|
68
|
-
|
|
73
|
+
self.parse_hdf5_metadata_and_renew_yaml()
|
|
74
|
+
return
|
|
69
75
|
else:
|
|
76
|
+
self.prepare_project_layout()
|
|
70
77
|
self.load_dataset(eager=False)
|
|
71
78
|
self.plan_dataset_required_columns()
|
|
72
79
|
if self.shared is None:
|
|
@@ -316,7 +323,15 @@ class JarvisPLOT():
|
|
|
316
323
|
except Exception:
|
|
317
324
|
pass
|
|
318
325
|
return
|
|
319
|
-
|
|
326
|
+
resolved = os.path.abspath(yaml_path)
|
|
327
|
+
try:
|
|
328
|
+
self.parser_yaml(resolved)
|
|
329
|
+
except FileNotFoundError:
|
|
330
|
+
self.logger.error(f"YAML file not found: {resolved}")
|
|
331
|
+
sys.exit(2)
|
|
332
|
+
except OSError as e:
|
|
333
|
+
self.logger.error(f"Failed to open YAML file '{resolved}': {e}")
|
|
334
|
+
sys.exit(2)
|
|
320
335
|
|
|
321
336
|
def init_logger(self) -> None:
|
|
322
337
|
from datetime import datetime
|
|
@@ -335,8 +350,7 @@ class JarvisPLOT():
|
|
|
335
350
|
return record["extra"].get("to_console", False)
|
|
336
351
|
|
|
337
352
|
def custom_format(record):
|
|
338
|
-
|
|
339
|
-
return f"\n\n<cyan>{module}</cyan> \n\t-> <green>{record['time']:MM-DD HH:mm:ss.SSS}</green> - [<level>{record['level']}</level>] >>> \n<level>{record['message']}</level> "
|
|
353
|
+
return _format_console_record(record)
|
|
340
354
|
|
|
341
355
|
logger.add(
|
|
342
356
|
sys.stdout,
|
|
@@ -368,3 +382,6 @@ class JarvisPLOT():
|
|
|
368
382
|
|
|
369
383
|
def rename_hdf5_and_renew_yaml(self):
|
|
370
384
|
return runtime_rename_hdf5_and_renew_yaml(self)
|
|
385
|
+
|
|
386
|
+
def parse_hdf5_metadata_and_renew_yaml(self):
|
|
387
|
+
return runtime_parse_hdf5_metadata_and_renew_yaml(self)
|
|
@@ -7,6 +7,7 @@ import yaml
|
|
|
7
7
|
|
|
8
8
|
from .cache_store import ProjectCache
|
|
9
9
|
from .data_loader import JP_ROW_IDX
|
|
10
|
+
from .data_loader_hdf5 import scan_hdf5_leaf_metadata
|
|
10
11
|
from .utils.pathing import resolve_project_path
|
|
11
12
|
|
|
12
13
|
|
|
@@ -130,7 +131,7 @@ def plan_dataset_required_columns(core) -> None:
|
|
|
130
131
|
dts.set_required_columns(required if required else None, retained=retained if retained else None)
|
|
131
132
|
if core.logger:
|
|
132
133
|
sample = ", ".join(sorted(list(retained))[:12]) if retained else "<none>"
|
|
133
|
-
core.logger.
|
|
134
|
+
core.logger.info(
|
|
134
135
|
"Dataset required columns planned:\n\t dataset \t-> {}\n\t required \t-> {}\n\t retained \t-> {}\n\t sample \t-> {}".format(
|
|
135
136
|
name,
|
|
136
137
|
len(required),
|
|
@@ -237,3 +238,82 @@ def rename_hdf5_and_renew_yaml(core):
|
|
|
237
238
|
allow_unicode=True,
|
|
238
239
|
width=100000,
|
|
239
240
|
)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def parse_hdf5_metadata_and_renew_yaml(core):
|
|
244
|
+
def _as_quoted_str(value: Any) -> _QuotedString:
|
|
245
|
+
return _QuotedString(str(value))
|
|
246
|
+
|
|
247
|
+
def _normalize_whitelist_as_quoted(raw):
|
|
248
|
+
if isinstance(raw, list):
|
|
249
|
+
return [_as_quoted_str(v) for v in raw if v is not None and str(v).strip()]
|
|
250
|
+
if isinstance(raw, str):
|
|
251
|
+
sval = raw.strip()
|
|
252
|
+
if sval:
|
|
253
|
+
return _as_quoted_str(sval)
|
|
254
|
+
return []
|
|
255
|
+
|
|
256
|
+
for dcfg in core.yaml.config.get("DataSet", []):
|
|
257
|
+
if isinstance(dcfg, dict):
|
|
258
|
+
dcfg.pop("is_gambit", None)
|
|
259
|
+
dcfg.pop("columnmap", None)
|
|
260
|
+
|
|
261
|
+
for dcfg in core.yaml.config.get("DataSet", []):
|
|
262
|
+
if not isinstance(dcfg, dict):
|
|
263
|
+
continue
|
|
264
|
+
if str(dcfg.get("type", "")).strip().lower() != "hdf5":
|
|
265
|
+
continue
|
|
266
|
+
|
|
267
|
+
name = str(dcfg.get("name", "")).strip()
|
|
268
|
+
project_cfg = core.yaml.config.get("project", {})
|
|
269
|
+
if not isinstance(project_cfg, dict):
|
|
270
|
+
project_cfg = {}
|
|
271
|
+
workdir = project_cfg.get("workdir", core.yaml.dir)
|
|
272
|
+
path = resolve_project_path(str(dcfg.get("path", "")).strip(), base_dir=workdir or core.yaml.dir)
|
|
273
|
+
group = str(dcfg.get("dataset", "")).strip() or None
|
|
274
|
+
old_columns = dcfg.get("columns", {})
|
|
275
|
+
if not isinstance(old_columns, dict):
|
|
276
|
+
old_columns = {}
|
|
277
|
+
|
|
278
|
+
metadata = scan_hdf5_leaf_metadata(str(path), group=group)
|
|
279
|
+
usable = [item for item in metadata if not str(item.get("path", "")).endswith("_isvalid")]
|
|
280
|
+
if not usable:
|
|
281
|
+
raise RuntimeError(
|
|
282
|
+
"No usable leaf datasets found for HDF5 parse-data: "
|
|
283
|
+
f"dataset='{name}', path='{path}', group='{group or '<root>'}'."
|
|
284
|
+
)
|
|
285
|
+
|
|
286
|
+
vmap_list = []
|
|
287
|
+
for ii, item in enumerate(usable):
|
|
288
|
+
source = str(item["path"])
|
|
289
|
+
target = f"Var{ii}@{name}"
|
|
290
|
+
vmap_list.append(
|
|
291
|
+
{
|
|
292
|
+
"source": _as_quoted_str(source),
|
|
293
|
+
"target": target,
|
|
294
|
+
}
|
|
295
|
+
)
|
|
296
|
+
|
|
297
|
+
columns_payload = {}
|
|
298
|
+
for k, v in old_columns.items():
|
|
299
|
+
if k in {"rename", "load_whitelist"}:
|
|
300
|
+
continue
|
|
301
|
+
columns_payload[k] = v
|
|
302
|
+
columns_payload["rename"] = vmap_list
|
|
303
|
+
|
|
304
|
+
if "load_whitelist" in old_columns:
|
|
305
|
+
columns_payload["load_whitelist"] = _normalize_whitelist_as_quoted(old_columns.get("load_whitelist"))
|
|
306
|
+
|
|
307
|
+
core.yaml.update_dataset(name, {"columns": columns_payload})
|
|
308
|
+
|
|
309
|
+
with open(core.args.out, "w", encoding="utf-8") as f1:
|
|
310
|
+
yaml.dump(
|
|
311
|
+
core.yaml.config,
|
|
312
|
+
f1,
|
|
313
|
+
Dumper=_QuotedDumper,
|
|
314
|
+
sort_keys=False,
|
|
315
|
+
default_flow_style=False,
|
|
316
|
+
indent=2,
|
|
317
|
+
allow_unicode=True,
|
|
318
|
+
width=100000,
|
|
319
|
+
)
|
|
@@ -64,6 +64,7 @@ class DataSet():
|
|
|
64
64
|
def __init__(self):
|
|
65
65
|
self._file: Optional[str] = None
|
|
66
66
|
self.path: Optional[str] = None
|
|
67
|
+
self.rootpath: Optional[str] = None
|
|
67
68
|
self._type: Optional[str] = None
|
|
68
69
|
self.base: Optional[str] = None
|
|
69
70
|
self.keys: Optional[List[str]] = None
|
|
@@ -86,6 +87,7 @@ class DataSet():
|
|
|
86
87
|
|
|
87
88
|
def setinfo(self, dtinfo, rootpath, eager: bool = False, cache=None):
|
|
88
89
|
self.cache = cache
|
|
90
|
+
self.rootpath = str(rootpath) if rootpath is not None else None
|
|
89
91
|
raw_path = str(dtinfo['path'])
|
|
90
92
|
resolved = resolve_project_path(raw_path, base_dir=rootpath or ".")
|
|
91
93
|
self.file = str(resolved)
|
|
@@ -142,6 +144,18 @@ class DataSet():
|
|
|
142
144
|
except Exception as e:
|
|
143
145
|
if self.logger:
|
|
144
146
|
self.logger.warning(f"Dataset '{self.name}' lazy metadata failed: {e}")
|
|
147
|
+
elif self.type == "parquet":
|
|
148
|
+
if pl is not None:
|
|
149
|
+
try:
|
|
150
|
+
scan = pl.scan_parquet(self.path)
|
|
151
|
+
self.keys = list(scan.collect_schema().names())
|
|
152
|
+
if self.logger:
|
|
153
|
+
self.logger.debug(
|
|
154
|
+
f"Dataset '{self.name}' registered in lazy mode (parquet columns={len(self.keys)})."
|
|
155
|
+
)
|
|
156
|
+
except Exception as e:
|
|
157
|
+
if self.logger:
|
|
158
|
+
self.logger.warning(f"Dataset '{self.name}' lazy metadata failed: {e}")
|
|
145
159
|
else:
|
|
146
160
|
if self.logger:
|
|
147
161
|
self.logger.debug(f"Dataset '{self.name}' registered in lazy mode.")
|
|
@@ -175,6 +189,8 @@ class DataSet():
|
|
|
175
189
|
return self.data
|
|
176
190
|
if self.type == "csv":
|
|
177
191
|
self.load_csv()
|
|
192
|
+
elif self.type == "parquet":
|
|
193
|
+
self.load_parquet()
|
|
178
194
|
elif self.type == "hdf5":
|
|
179
195
|
self.load_hdf5()
|
|
180
196
|
else:
|
|
@@ -240,6 +256,8 @@ class DataSet():
|
|
|
240
256
|
def _summary_name(self) -> str:
|
|
241
257
|
if self.type == "hdf5":
|
|
242
258
|
return f" HDF5 loaded!\n\t name -> {self.name}\n\t group -> {self.group}\n\t path -> {self.path}"
|
|
259
|
+
if self.type == "parquet":
|
|
260
|
+
return f" Parquet loaded!\n\t name -> {self.name}\n\t path -> {self.path}"
|
|
243
261
|
return f" CSV loaded!\n\t name -> {self.name}\n\t path -> {self.path}"
|
|
244
262
|
|
|
245
263
|
def _debug_enabled(self) -> bool:
|
|
@@ -279,12 +297,18 @@ class DataSet():
|
|
|
279
297
|
if self.data is not None:
|
|
280
298
|
try:
|
|
281
299
|
if isinstance(self._materialized_manifest, dict):
|
|
282
|
-
msg = self._materialized_summary(
|
|
300
|
+
msg = self._materialized_summary(
|
|
301
|
+
self._materialized_manifest,
|
|
302
|
+
stats=self._materialized_numeric_bounds(),
|
|
303
|
+
)
|
|
283
304
|
else:
|
|
284
305
|
msg = summary.dataframe_summary(self.data, name=self._summary_name())
|
|
285
306
|
except Exception:
|
|
286
307
|
if isinstance(self._materialized_manifest, dict):
|
|
287
|
-
msg = self._materialized_summary(
|
|
308
|
+
msg = self._materialized_summary(
|
|
309
|
+
self._materialized_manifest,
|
|
310
|
+
stats=self._materialized_numeric_bounds(),
|
|
311
|
+
)
|
|
288
312
|
else:
|
|
289
313
|
msg = f"Data backend: {type(self.data).__name__}"
|
|
290
314
|
if self.cache is not None:
|
|
@@ -369,8 +393,11 @@ class DataSet():
|
|
|
369
393
|
def _materialized_source_fingerprint(self):
|
|
370
394
|
return hdf5.materialized_source_fingerprint(self)
|
|
371
395
|
|
|
372
|
-
def _materialized_summary(self, manifest: Dict[str, Any]) -> str:
|
|
373
|
-
return hdf5.materialized_summary(self, manifest)
|
|
396
|
+
def _materialized_summary(self, manifest: Dict[str, Any], stats: Optional[Dict[str, Dict[str, Any]]] = None) -> str:
|
|
397
|
+
return hdf5.materialized_summary(self, manifest, stats=stats)
|
|
398
|
+
|
|
399
|
+
def _materialized_numeric_bounds(self):
|
|
400
|
+
return runtime.materialized_numeric_bounds(self)
|
|
374
401
|
|
|
375
402
|
@staticmethod
|
|
376
403
|
def _sql_bool_ops(expr: Any) -> str:
|
|
@@ -430,6 +457,9 @@ class DataSet():
|
|
|
430
457
|
def load_hdf5(self):
|
|
431
458
|
return runtime.load_hdf5(self)
|
|
432
459
|
|
|
460
|
+
def load_parquet(self):
|
|
461
|
+
return runtime.load_parquet(self)
|
|
462
|
+
|
|
433
463
|
def apply_is_valid_policy(self, kkeys):
|
|
434
464
|
if self.data is None:
|
|
435
465
|
self.load(force=False)
|
|
@@ -473,7 +503,7 @@ class DataSet():
|
|
|
473
503
|
sps = self.data.shape
|
|
474
504
|
mask = self.data[isvalids].all(axis=1)
|
|
475
505
|
self.data = self.data[mask]
|
|
476
|
-
self.logger.
|
|
506
|
+
self.logger.info(
|
|
477
507
|
"DataSet Shape: \n\t Before filtering -> {}\n\t After filtering -> {}".format(
|
|
478
508
|
sps, self.data.shape
|
|
479
509
|
)
|
|
@@ -3,7 +3,9 @@ from __future__ import annotations
|
|
|
3
3
|
import os
|
|
4
4
|
from typing import Any, Dict, List, Optional
|
|
5
5
|
|
|
6
|
-
|
|
6
|
+
import h5py
|
|
7
|
+
|
|
8
|
+
from .data_loader_summary import dataframe_summary_human_bytes, format_table
|
|
7
9
|
|
|
8
10
|
try:
|
|
9
11
|
import polars as pl
|
|
@@ -118,6 +120,50 @@ def build_hdf5_whitelist(dataset) -> Optional[set[str]]:
|
|
|
118
120
|
return with_isvalid
|
|
119
121
|
|
|
120
122
|
|
|
123
|
+
def scan_hdf5_leaf_metadata(path: str, group: Optional[str] = None) -> List[Dict[str, Any]]:
|
|
124
|
+
"""Return lightweight metadata for leaf datasets without reading dataset bodies."""
|
|
125
|
+
|
|
126
|
+
def _walk(node, prefix: str = "") -> None:
|
|
127
|
+
for name, child in node.items():
|
|
128
|
+
child_path = f"{prefix}/{name}" if prefix else name
|
|
129
|
+
if isinstance(child, h5py.Dataset):
|
|
130
|
+
metadata.append(
|
|
131
|
+
{
|
|
132
|
+
"path": child_path,
|
|
133
|
+
"shape": tuple(getattr(child, "shape", ()) or ()),
|
|
134
|
+
"dtype": str(getattr(child, "dtype", "NA")),
|
|
135
|
+
}
|
|
136
|
+
)
|
|
137
|
+
elif isinstance(child, h5py.Group):
|
|
138
|
+
_walk(child, child_path)
|
|
139
|
+
|
|
140
|
+
group_name = str(group).strip() if group is not None else ""
|
|
141
|
+
metadata: List[Dict[str, Any]] = []
|
|
142
|
+
|
|
143
|
+
try:
|
|
144
|
+
with h5py.File(path, "r") as h5f:
|
|
145
|
+
if group_name:
|
|
146
|
+
if group_name not in h5f:
|
|
147
|
+
raise RuntimeError(f"HDF5 group '{group_name}' not found in '{path}'.")
|
|
148
|
+
root = h5f[group_name]
|
|
149
|
+
if not isinstance(root, h5py.Group):
|
|
150
|
+
raise RuntimeError(f"HDF5 group '{group_name}' in '{path}' is not a group.")
|
|
151
|
+
_walk(root, group_name)
|
|
152
|
+
else:
|
|
153
|
+
_walk(h5f, "")
|
|
154
|
+
except FileNotFoundError as e:
|
|
155
|
+
raise RuntimeError(f"HDF5 file not found: {path}") from e
|
|
156
|
+
except OSError as e:
|
|
157
|
+
raise RuntimeError(f"Unable to open HDF5 file '{path}': {e}") from e
|
|
158
|
+
|
|
159
|
+
if not metadata:
|
|
160
|
+
if group_name:
|
|
161
|
+
raise RuntimeError(f"HDF5 group '{group_name}' in '{path}' contains no leaf datasets.")
|
|
162
|
+
raise RuntimeError(f"No leaf datasets found in HDF5 file '{path}'.")
|
|
163
|
+
|
|
164
|
+
return metadata
|
|
165
|
+
|
|
166
|
+
|
|
121
167
|
def shape_token(df) -> str:
|
|
122
168
|
import pandas as pd
|
|
123
169
|
|
|
@@ -169,7 +215,7 @@ def materialized_cache_key(dataset):
|
|
|
169
215
|
return dataset.cache.cache_key({"kind": "hdf5-polars-materialized", "source": source_fp})
|
|
170
216
|
|
|
171
217
|
|
|
172
|
-
def materialized_summary(dataset, manifest: Dict[str, Any]) -> str:
|
|
218
|
+
def materialized_summary(dataset, manifest: Dict[str, Any], stats: Optional[Dict[str, Dict[str, Any]]] = None) -> str:
|
|
173
219
|
rows = manifest.get("rows", "NA")
|
|
174
220
|
cols = manifest.get("cols", len(manifest.get("columns", [])))
|
|
175
221
|
parts = manifest.get("parts", 0)
|
|
@@ -187,6 +233,71 @@ def materialized_summary(dataset, manifest: Dict[str, Any]) -> str:
|
|
|
187
233
|
if manifest.get("columns"):
|
|
188
234
|
lines.append("=== Materialized Columns ===")
|
|
189
235
|
lines.append(", ".join(str(c) for c in manifest.get("columns", [])))
|
|
236
|
+
if isinstance(stats, dict) and stats:
|
|
237
|
+
lines.append("=== Materialized Stats ===")
|
|
238
|
+
stat_rows = []
|
|
239
|
+
df = getattr(dataset, "data", None)
|
|
240
|
+
schema_names = None
|
|
241
|
+
schema_dtypes = {}
|
|
242
|
+
if pl is not None and isinstance(df, pl.LazyFrame):
|
|
243
|
+
try:
|
|
244
|
+
schema = df.collect_schema()
|
|
245
|
+
if hasattr(schema, "names"):
|
|
246
|
+
schema_names = set(str(x) for x in schema.names())
|
|
247
|
+
elif isinstance(schema, dict):
|
|
248
|
+
schema_names = set(str(x) for x in schema.keys())
|
|
249
|
+
schema_dtypes = {str(k): str(v) for k, v in schema.items()}
|
|
250
|
+
elif hasattr(schema, "items"):
|
|
251
|
+
schema_names = set(str(x) for x in schema.keys())
|
|
252
|
+
schema_dtypes = {str(k): str(v) for k, v in schema.items()}
|
|
253
|
+
except Exception:
|
|
254
|
+
schema_names = set()
|
|
255
|
+
elif pl is not None and isinstance(df, pl.DataFrame):
|
|
256
|
+
try:
|
|
257
|
+
schema_names = set(str(x) for x in df.columns)
|
|
258
|
+
except Exception:
|
|
259
|
+
schema_names = set()
|
|
260
|
+
elif hasattr(df, "columns"):
|
|
261
|
+
try:
|
|
262
|
+
schema_names = set(str(x) for x in getattr(df, "columns", []))
|
|
263
|
+
except Exception:
|
|
264
|
+
schema_names = set()
|
|
265
|
+
for col in manifest.get("columns", []):
|
|
266
|
+
if col not in stats:
|
|
267
|
+
continue
|
|
268
|
+
bounds = stats.get(col, {})
|
|
269
|
+
dtype = "NA"
|
|
270
|
+
nonnull_text = "NA"
|
|
271
|
+
if pl is not None and isinstance(df, pl.LazyFrame):
|
|
272
|
+
dtype = schema_dtypes.get(col, "NA")
|
|
273
|
+
elif schema_names is not None and col in schema_names and hasattr(df, "__getitem__"):
|
|
274
|
+
series = df[col]
|
|
275
|
+
dtype = str(series.dtype)
|
|
276
|
+
try:
|
|
277
|
+
nonnull_pct = (float(series.notna().sum()) / float(len(series)) * 100.0) if len(series) else 0.0
|
|
278
|
+
except Exception:
|
|
279
|
+
nonnull_pct = 0.0
|
|
280
|
+
nonnull_text = f"{nonnull_pct:.1f}%"
|
|
281
|
+
else:
|
|
282
|
+
dtype = str(type(bounds.get("min", None)).__name__)
|
|
283
|
+
stat_rows.append(
|
|
284
|
+
(
|
|
285
|
+
str(col),
|
|
286
|
+
dtype,
|
|
287
|
+
nonnull_text,
|
|
288
|
+
bounds.get("min", None),
|
|
289
|
+
bounds.get("max", None),
|
|
290
|
+
)
|
|
291
|
+
)
|
|
292
|
+
|
|
293
|
+
if stat_rows:
|
|
294
|
+
lines.append(
|
|
295
|
+
format_table(
|
|
296
|
+
["name", "dtype", "nonnull%", "[min]", "[max]"],
|
|
297
|
+
stat_rows,
|
|
298
|
+
aligns=["left", "left", "right", "right", "right"],
|
|
299
|
+
)
|
|
300
|
+
)
|
|
190
301
|
return "\n".join(lines)
|
|
191
302
|
def sql_bool_ops(expr: Any) -> str:
|
|
192
303
|
return str(expr).replace("&&", " AND ").replace("||", " OR ")
|