jarvisplot 1.3.2__tar.gz → 1.3.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/JarvisPLOT.egg-info/PKG-INFO +1 -1
  2. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/JarvisPLOT.egg-info/SOURCES.txt +1 -0
  3. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/PKG-INFO +1 -1
  4. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/config_runtime.py +4 -3
  5. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/load_data.py +69 -0
  6. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/preprocessor_runtime.py +9 -5
  7. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cache_store.py +7 -0
  8. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/core.py +23 -6
  9. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/core_runtime.py +81 -1
  10. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/data_loader.py +35 -5
  11. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/data_loader_hdf5.py +113 -2
  12. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/data_loader_runtime.py +271 -50
  13. jarvisplot-1.3.3/jarvisplot/data_loader_summary.py +164 -0
  14. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/pyproject.toml +1 -1
  15. jarvisplot-1.3.3/tests/test_data_loader_contracts.py +445 -0
  16. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/tests/test_figure_contracts.py +48 -0
  17. jarvisplot-1.3.3/tests/test_parse_data_metadata.py +153 -0
  18. jarvisplot-1.3.2/jarvisplot/data_loader_summary.py +0 -83
  19. jarvisplot-1.3.2/tests/test_data_loader_contracts.py +0 -96
  20. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/JarvisPLOT.egg-info/dependency_links.txt +0 -0
  21. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/JarvisPLOT.egg-info/entry_points.txt +0 -0
  22. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/JarvisPLOT.egg-info/requires.txt +0 -0
  23. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/JarvisPLOT.egg-info/top_level.txt +0 -0
  24. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/README.md +0 -0
  25. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/__init__.py +0 -0
  26. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/adapters.py +0 -0
  27. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/adapters_rect.py +0 -0
  28. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/adapters_ternary.py +0 -0
  29. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/cards/std_axes_adapter_config.json +0 -0
  30. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/colorbar_runtime.py +0 -0
  31. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/data_pipelines.py +0 -0
  32. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/figure.py +0 -0
  33. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/helper.py +0 -0
  34. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/layer_runtime.py +0 -0
  35. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/layout_runtime.py +0 -0
  36. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/method_registry.py +0 -0
  37. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/preprocessor.py +0 -0
  38. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/profile_runtime.py +0 -0
  39. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/Figure/style_runtime.py +0 -0
  40. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/__init__.py +0 -0
  41. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/a4paper/1x1/ternary.json +0 -0
  42. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/a4paper/2x1/rect.json +0 -0
  43. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/a4paper/2x1/rect5x1.json +0 -0
  44. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/a4paper/2x1/rect_cmap.json +0 -0
  45. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/a4paper/2x1/ternary.json +0 -0
  46. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/a4paper/2x1/ternary_cmap.json +0 -0
  47. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/a4paper/4x1/rect.json +0 -0
  48. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/a4paper/4x1/rect_cmap.json +0 -0
  49. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/a4paper/4x1/ternary.json +0 -0
  50. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/a4paper/4x1/ternary_cmap.json +0 -0
  51. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/args.json +0 -0
  52. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/colors/colormaps.json +0 -0
  53. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/default/output.json +0 -0
  54. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/gambit/1x1/ternary.json +0 -0
  55. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/gambit/2x1/rect_cmap.json +0 -0
  56. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/gambit/2x1/ternary.json +0 -0
  57. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/gambit/2x1/ternary_cmap.json +0 -0
  58. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/icons/JarvisHEP.png +0 -0
  59. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/icons/gambit.png +0 -0
  60. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/icons/gambit_small.png +0 -0
  61. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cards/style_preference.json +0 -0
  62. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/cli.py +0 -0
  63. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/client.py +0 -0
  64. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/config.py +0 -0
  65. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/core_assets.py +0 -0
  66. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/inner_func.py +0 -0
  67. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/memtrace.py +0 -0
  68. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/utils/__init__.py +0 -0
  69. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/utils/cmaps.py +0 -0
  70. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/utils/expression.py +0 -0
  71. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/utils/interpolator.py +0 -0
  72. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/jarvisplot/utils/pathing.py +0 -0
  73. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/setup.cfg +0 -0
  74. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/tests/test_docs_status_labels.py +0 -0
  75. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/tests/test_pathing_contracts.py +0 -0
  76. {jarvisplot-1.3.2 → jarvisplot-1.3.3}/tests/test_template_json.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: JarvisPLOT
3
- Version: 1.3.2
3
+ Version: 1.3.3
4
4
  Summary: Jarvis-PLOT: YAML-driven plotting engine
5
5
  Requires-Python: >=3.10
6
6
  Description-Content-Type: text/markdown
@@ -67,5 +67,6 @@ jarvisplot/utils/pathing.py
67
67
  tests/test_data_loader_contracts.py
68
68
  tests/test_docs_status_labels.py
69
69
  tests/test_figure_contracts.py
70
+ tests/test_parse_data_metadata.py
70
71
  tests/test_pathing_contracts.py
71
72
  tests/test_template_json.py
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: JarvisPLOT
3
- Version: 1.3.2
3
+ Version: 1.3.3
4
4
  Summary: Jarvis-PLOT: YAML-driven plotting engine
5
5
  Requires-Python: >=3.10
6
6
  Description-Content-Type: text/markdown
@@ -39,11 +39,12 @@ def apply_figure_config(fig, info: Mapping) -> bool:
39
39
  return False
40
40
 
41
41
  if "style" in info:
42
- fig.style = info["style"]
42
+ style_tokens = info["style"]
43
43
  else:
44
- fig.style = ["a4paper_2x1"]
44
+ style_tokens = ["a4paper_2x1"]
45
+ fig.style = style_tokens
45
46
  fig.logger.debug("Figure style loaded")
46
- if fig.style and "gambit" in str(fig.style[0]).lower():
47
+ if style_tokens and "gambit" in str(style_tokens[0]).lower():
47
48
  fig.mode = "gambit"
48
49
 
49
50
  if "frame" in info:
@@ -6,6 +6,28 @@ from ..utils.expression import eval_dataframe_expression
6
6
  from .profile_runtime import eval_series, grid_profiling, profiling, _preprofiling
7
7
 
8
8
 
9
+ def _normalize_column_list(spec):
10
+ if spec is None:
11
+ return []
12
+ if isinstance(spec, dict):
13
+ for key in ("columns", "keep", "drop", "select", "retain", "value"):
14
+ if key in spec:
15
+ return _normalize_column_list(spec.get(key))
16
+ return []
17
+ if isinstance(spec, str):
18
+ text = spec.strip()
19
+ return [text] if text else []
20
+ if isinstance(spec, (list, tuple, set)):
21
+ out = []
22
+ for item in spec:
23
+ sval = str(item).strip()
24
+ if sval:
25
+ out.append(sval)
26
+ return out
27
+ sval = str(spec).strip()
28
+ return [sval] if sval else []
29
+
30
+
9
31
  def filter(df, condition, logger):
10
32
  try:
11
33
  if isinstance(condition, bool):
@@ -55,6 +77,53 @@ def addcolumn(df, adds, logger):
55
77
  )
56
78
  )
57
79
  return df
80
+
81
+
82
+ def keep_columns(df, spec, logger):
83
+ try:
84
+ cols = _normalize_column_list(spec)
85
+ if not cols:
86
+ return df
87
+ if isinstance(df, pd.DataFrame):
88
+ keep = [c for c in cols if c in df.columns]
89
+ missing = [c for c in cols if c not in df.columns]
90
+ if missing and logger:
91
+ logger.warning(f"keep_columns missing columns ignored -> {missing}")
92
+ if not keep:
93
+ return df.iloc[0:0].copy()
94
+ return df.loc[:, keep].copy(deep=False)
95
+ if hasattr(df, "select"):
96
+ try:
97
+ return df.select(cols)
98
+ except Exception:
99
+ return df
100
+ return df
101
+ except Exception as e:
102
+ if logger:
103
+ logger.warning(f"keep_columns failed for spec={spec}: {e}")
104
+ return df
105
+
106
+
107
+ def drop_columns(df, spec, logger):
108
+ try:
109
+ cols = _normalize_column_list(spec)
110
+ if not cols:
111
+ return df
112
+ if isinstance(df, pd.DataFrame):
113
+ existing = [c for c in cols if c in df.columns]
114
+ if not existing:
115
+ return df
116
+ return df.drop(columns=existing, errors="ignore")
117
+ if hasattr(df, "drop"):
118
+ try:
119
+ return df.drop(cols)
120
+ except Exception:
121
+ return df
122
+ return df
123
+ except Exception as e:
124
+ if logger:
125
+ logger.warning(f"drop_columns failed for spec={spec}: {e}")
126
+ return df
58
127
 
59
128
  def sortby(df, expr, logger):
60
129
  try:
@@ -13,7 +13,7 @@ except Exception:
13
13
  pl = None
14
14
 
15
15
  from ..memtrace import memtrace_checkpoint, memtrace_object_inventory
16
- from .load_data import _preprofiling, addcolumn, filter as filter_df, grid_profiling, profiling, sortby
16
+ from .load_data import _preprofiling, addcolumn, drop_columns, filter as filter_df, grid_profiling, keep_columns, profiling, sortby
17
17
 
18
18
  def resolve_source_data(preprocessor, source: Any, combine: str = "concat"):
19
19
  if isinstance(source, str):
@@ -130,7 +130,7 @@ def apply_transforms_impl(
130
130
  method = str(profile_cfg.get("method", "bridson")).lower()
131
131
  if "bin" in profile_cfg:
132
132
  binv = profile_cfg.get("bin")
133
- preprocessor._warn(
133
+ preprocessor._info(
134
134
  "Runtime profile START:\n\t source \t-> {}\n\t step \t\t-> profile, \n\t method \t-> {}\n\t bin \t\t-> {}\n\t rows_before \t-> {}".format(
135
135
  source_label or "<unknown>",
136
136
  method,
@@ -183,7 +183,7 @@ def apply_transforms_impl(
183
183
  profile_cfg = {"method": "grid"}
184
184
  before_rows = preprocessor._safe_nrows(df)
185
185
  binv = profile_cfg.get("bin", "default")
186
- preprocessor._warn(
186
+ preprocessor._info(
187
187
  "Runtime profile START: source \t-> {}\n\t step \t-> 'grid_profile,\n\t method \t-> 'grid',\n\t bin \t-> {},\n\t rows_before \t-> {}".format(
188
188
  source_label or "<unknown>",
189
189
  binv,
@@ -227,6 +227,10 @@ def apply_transforms_impl(
227
227
  df = sortby(df, trans["sortby"], preprocessor.logger)
228
228
  elif "add_column" in trans:
229
229
  df = addcolumn(df, trans["add_column"], preprocessor.logger)
230
+ elif "keep_columns" in trans:
231
+ df = keep_columns(df, trans.get("keep_columns"), preprocessor.logger)
232
+ elif "drop_columns" in trans:
233
+ df = drop_columns(df, trans.get("drop_columns"), preprocessor.logger)
230
234
 
231
235
  if prev_df is not df:
232
236
  collect_prev = preprocessor._should_collect_dataframe(prev_df)
@@ -384,7 +388,7 @@ def run_pipeline(preprocessor,
384
388
  src_label = preprocessor._runtime_source_label(source)
385
389
  if isinstance(source, str) and source in preprocessor._preprofile_alias_meta:
386
390
  meta = preprocessor._preprofile_alias_meta.get(source, {})
387
- preprocessor._warn(
391
+ preprocessor._info(
388
392
  "Runtime profile input:\n\t source \t-> {},\n uses preprofile alias:\n\t key \t\t-> {},\n\t origin \t-> {},\n\t cache_file \t-> {},\n\t rows_in \t-> {}.".format(
389
393
  src_label,
390
394
  meta.get("pre_key", "<unknown>")[:16] if isinstance(meta.get("pre_key"), str) else "<unknown>",
@@ -457,7 +461,7 @@ def run_pipeline(preprocessor,
457
461
  cache_file = str((preprocessor.cache.data_dir / f"{key}.pkl").resolve())
458
462
  except Exception:
459
463
  pass
460
- preprocessor._warn(
464
+ preprocessor._info(
461
465
  "Runtime profile cache STORE:\n\t source \t-> {},\n\t key \t\t-> {},\n\t cache_file \t-> {},\n\t rows \t\t-> {}.".format(
462
466
  preprocessor._runtime_source_label(source),
463
467
  key[:16],
@@ -62,6 +62,13 @@ class ProjectCache:
62
62
  except Exception:
63
63
  pass
64
64
 
65
+ def _info(self, msg: str) -> None:
66
+ if self.logger:
67
+ try:
68
+ self.logger.info(msg)
69
+ except Exception:
70
+ pass
71
+
65
72
  @staticmethod
66
73
  def _load_json(path: Path, default: Dict[str, Any]) -> Dict[str, Any]:
67
74
  try:
@@ -23,10 +23,17 @@ from .core_runtime import (
23
23
  plan_dataset_required_columns as runtime_plan_dataset_required_columns,
24
24
  prepare_project_layout as runtime_prepare_project_layout,
25
25
  prepare_usage_plan as runtime_prepare_usage_plan,
26
+ parse_hdf5_metadata_and_renew_yaml as runtime_parse_hdf5_metadata_and_renew_yaml,
26
27
  rename_hdf5_and_renew_yaml as runtime_rename_hdf5_and_renew_yaml,
27
28
  )
28
29
 
29
30
 
31
+ def _format_console_record(record):
32
+ module = record["extra"].get("module", "No module")
33
+ message = str(record["message"]).replace("{", "{{").replace("}", "}}").replace("<", "\\<")
34
+ return f"\n\n<cyan>{module}</cyan> \n\t-> <green>{record['time']:MM-DD HH:mm:ss.SSS}</green> - [<level>{record['level']}</level>] >>> \n<level>{message}</level> "
35
+
36
+
30
37
  class JarvisPLOT():
31
38
  def __init__(self) -> None:
32
39
  self.variables = {}
@@ -53,7 +60,6 @@ class JarvisPLOT():
53
60
  self.load_cmaps()
54
61
 
55
62
  self.load_yaml()
56
- self.prepare_project_layout()
57
63
 
58
64
  # sys.exit()
59
65
  if self.args.parse_data:
@@ -64,9 +70,10 @@ class JarvisPLOT():
64
70
  elif self.args.out is not None and self.args.inplace:
65
71
  self.logger.error("Conflicting arguments: --out and --inplace. Please choose only one.")
66
72
  sys.exit(2)
67
- self.load_dataset(eager=True)
68
- self.rename_hdf5_and_renew_yaml()
73
+ self.parse_hdf5_metadata_and_renew_yaml()
74
+ return
69
75
  else:
76
+ self.prepare_project_layout()
70
77
  self.load_dataset(eager=False)
71
78
  self.plan_dataset_required_columns()
72
79
  if self.shared is None:
@@ -316,7 +323,15 @@ class JarvisPLOT():
316
323
  except Exception:
317
324
  pass
318
325
  return
319
- self.parser_yaml(os.path.abspath(yaml_path))
326
+ resolved = os.path.abspath(yaml_path)
327
+ try:
328
+ self.parser_yaml(resolved)
329
+ except FileNotFoundError:
330
+ self.logger.error(f"YAML file not found: {resolved}")
331
+ sys.exit(2)
332
+ except OSError as e:
333
+ self.logger.error(f"Failed to open YAML file '{resolved}': {e}")
334
+ sys.exit(2)
320
335
 
321
336
  def init_logger(self) -> None:
322
337
  from datetime import datetime
@@ -335,8 +350,7 @@ class JarvisPLOT():
335
350
  return record["extra"].get("to_console", False)
336
351
 
337
352
  def custom_format(record):
338
- module = record["extra"].get("module", "No module")
339
- return f"\n\n<cyan>{module}</cyan> \n\t-> <green>{record['time']:MM-DD HH:mm:ss.SSS}</green> - [<level>{record['level']}</level>] >>> \n<level>{record['message']}</level> "
353
+ return _format_console_record(record)
340
354
 
341
355
  logger.add(
342
356
  sys.stdout,
@@ -368,3 +382,6 @@ class JarvisPLOT():
368
382
 
369
383
  def rename_hdf5_and_renew_yaml(self):
370
384
  return runtime_rename_hdf5_and_renew_yaml(self)
385
+
386
+ def parse_hdf5_metadata_and_renew_yaml(self):
387
+ return runtime_parse_hdf5_metadata_and_renew_yaml(self)
@@ -7,6 +7,7 @@ import yaml
7
7
 
8
8
  from .cache_store import ProjectCache
9
9
  from .data_loader import JP_ROW_IDX
10
+ from .data_loader_hdf5 import scan_hdf5_leaf_metadata
10
11
  from .utils.pathing import resolve_project_path
11
12
 
12
13
 
@@ -130,7 +131,7 @@ def plan_dataset_required_columns(core) -> None:
130
131
  dts.set_required_columns(required if required else None, retained=retained if retained else None)
131
132
  if core.logger:
132
133
  sample = ", ".join(sorted(list(retained))[:12]) if retained else "<none>"
133
- core.logger.warning(
134
+ core.logger.info(
134
135
  "Dataset required columns planned:\n\t dataset \t-> {}\n\t required \t-> {}\n\t retained \t-> {}\n\t sample \t-> {}".format(
135
136
  name,
136
137
  len(required),
@@ -237,3 +238,82 @@ def rename_hdf5_and_renew_yaml(core):
237
238
  allow_unicode=True,
238
239
  width=100000,
239
240
  )
241
+
242
+
243
+ def parse_hdf5_metadata_and_renew_yaml(core):
244
+ def _as_quoted_str(value: Any) -> _QuotedString:
245
+ return _QuotedString(str(value))
246
+
247
+ def _normalize_whitelist_as_quoted(raw):
248
+ if isinstance(raw, list):
249
+ return [_as_quoted_str(v) for v in raw if v is not None and str(v).strip()]
250
+ if isinstance(raw, str):
251
+ sval = raw.strip()
252
+ if sval:
253
+ return _as_quoted_str(sval)
254
+ return []
255
+
256
+ for dcfg in core.yaml.config.get("DataSet", []):
257
+ if isinstance(dcfg, dict):
258
+ dcfg.pop("is_gambit", None)
259
+ dcfg.pop("columnmap", None)
260
+
261
+ for dcfg in core.yaml.config.get("DataSet", []):
262
+ if not isinstance(dcfg, dict):
263
+ continue
264
+ if str(dcfg.get("type", "")).strip().lower() != "hdf5":
265
+ continue
266
+
267
+ name = str(dcfg.get("name", "")).strip()
268
+ project_cfg = core.yaml.config.get("project", {})
269
+ if not isinstance(project_cfg, dict):
270
+ project_cfg = {}
271
+ workdir = project_cfg.get("workdir", core.yaml.dir)
272
+ path = resolve_project_path(str(dcfg.get("path", "")).strip(), base_dir=workdir or core.yaml.dir)
273
+ group = str(dcfg.get("dataset", "")).strip() or None
274
+ old_columns = dcfg.get("columns", {})
275
+ if not isinstance(old_columns, dict):
276
+ old_columns = {}
277
+
278
+ metadata = scan_hdf5_leaf_metadata(str(path), group=group)
279
+ usable = [item for item in metadata if not str(item.get("path", "")).endswith("_isvalid")]
280
+ if not usable:
281
+ raise RuntimeError(
282
+ "No usable leaf datasets found for HDF5 parse-data: "
283
+ f"dataset='{name}', path='{path}', group='{group or '<root>'}'."
284
+ )
285
+
286
+ vmap_list = []
287
+ for ii, item in enumerate(usable):
288
+ source = str(item["path"])
289
+ target = f"Var{ii}@{name}"
290
+ vmap_list.append(
291
+ {
292
+ "source": _as_quoted_str(source),
293
+ "target": target,
294
+ }
295
+ )
296
+
297
+ columns_payload = {}
298
+ for k, v in old_columns.items():
299
+ if k in {"rename", "load_whitelist"}:
300
+ continue
301
+ columns_payload[k] = v
302
+ columns_payload["rename"] = vmap_list
303
+
304
+ if "load_whitelist" in old_columns:
305
+ columns_payload["load_whitelist"] = _normalize_whitelist_as_quoted(old_columns.get("load_whitelist"))
306
+
307
+ core.yaml.update_dataset(name, {"columns": columns_payload})
308
+
309
+ with open(core.args.out, "w", encoding="utf-8") as f1:
310
+ yaml.dump(
311
+ core.yaml.config,
312
+ f1,
313
+ Dumper=_QuotedDumper,
314
+ sort_keys=False,
315
+ default_flow_style=False,
316
+ indent=2,
317
+ allow_unicode=True,
318
+ width=100000,
319
+ )
@@ -64,6 +64,7 @@ class DataSet():
64
64
  def __init__(self):
65
65
  self._file: Optional[str] = None
66
66
  self.path: Optional[str] = None
67
+ self.rootpath: Optional[str] = None
67
68
  self._type: Optional[str] = None
68
69
  self.base: Optional[str] = None
69
70
  self.keys: Optional[List[str]] = None
@@ -86,6 +87,7 @@ class DataSet():
86
87
 
87
88
  def setinfo(self, dtinfo, rootpath, eager: bool = False, cache=None):
88
89
  self.cache = cache
90
+ self.rootpath = str(rootpath) if rootpath is not None else None
89
91
  raw_path = str(dtinfo['path'])
90
92
  resolved = resolve_project_path(raw_path, base_dir=rootpath or ".")
91
93
  self.file = str(resolved)
@@ -142,6 +144,18 @@ class DataSet():
142
144
  except Exception as e:
143
145
  if self.logger:
144
146
  self.logger.warning(f"Dataset '{self.name}' lazy metadata failed: {e}")
147
+ elif self.type == "parquet":
148
+ if pl is not None:
149
+ try:
150
+ scan = pl.scan_parquet(self.path)
151
+ self.keys = list(scan.collect_schema().names())
152
+ if self.logger:
153
+ self.logger.debug(
154
+ f"Dataset '{self.name}' registered in lazy mode (parquet columns={len(self.keys)})."
155
+ )
156
+ except Exception as e:
157
+ if self.logger:
158
+ self.logger.warning(f"Dataset '{self.name}' lazy metadata failed: {e}")
145
159
  else:
146
160
  if self.logger:
147
161
  self.logger.debug(f"Dataset '{self.name}' registered in lazy mode.")
@@ -175,6 +189,8 @@ class DataSet():
175
189
  return self.data
176
190
  if self.type == "csv":
177
191
  self.load_csv()
192
+ elif self.type == "parquet":
193
+ self.load_parquet()
178
194
  elif self.type == "hdf5":
179
195
  self.load_hdf5()
180
196
  else:
@@ -240,6 +256,8 @@ class DataSet():
240
256
  def _summary_name(self) -> str:
241
257
  if self.type == "hdf5":
242
258
  return f" HDF5 loaded!\n\t name -> {self.name}\n\t group -> {self.group}\n\t path -> {self.path}"
259
+ if self.type == "parquet":
260
+ return f" Parquet loaded!\n\t name -> {self.name}\n\t path -> {self.path}"
243
261
  return f" CSV loaded!\n\t name -> {self.name}\n\t path -> {self.path}"
244
262
 
245
263
  def _debug_enabled(self) -> bool:
@@ -279,12 +297,18 @@ class DataSet():
279
297
  if self.data is not None:
280
298
  try:
281
299
  if isinstance(self._materialized_manifest, dict):
282
- msg = self._materialized_summary(self._materialized_manifest)
300
+ msg = self._materialized_summary(
301
+ self._materialized_manifest,
302
+ stats=self._materialized_numeric_bounds(),
303
+ )
283
304
  else:
284
305
  msg = summary.dataframe_summary(self.data, name=self._summary_name())
285
306
  except Exception:
286
307
  if isinstance(self._materialized_manifest, dict):
287
- msg = self._materialized_summary(self._materialized_manifest)
308
+ msg = self._materialized_summary(
309
+ self._materialized_manifest,
310
+ stats=self._materialized_numeric_bounds(),
311
+ )
288
312
  else:
289
313
  msg = f"Data backend: {type(self.data).__name__}"
290
314
  if self.cache is not None:
@@ -369,8 +393,11 @@ class DataSet():
369
393
  def _materialized_source_fingerprint(self):
370
394
  return hdf5.materialized_source_fingerprint(self)
371
395
 
372
- def _materialized_summary(self, manifest: Dict[str, Any]) -> str:
373
- return hdf5.materialized_summary(self, manifest)
396
+ def _materialized_summary(self, manifest: Dict[str, Any], stats: Optional[Dict[str, Dict[str, Any]]] = None) -> str:
397
+ return hdf5.materialized_summary(self, manifest, stats=stats)
398
+
399
+ def _materialized_numeric_bounds(self):
400
+ return runtime.materialized_numeric_bounds(self)
374
401
 
375
402
  @staticmethod
376
403
  def _sql_bool_ops(expr: Any) -> str:
@@ -430,6 +457,9 @@ class DataSet():
430
457
  def load_hdf5(self):
431
458
  return runtime.load_hdf5(self)
432
459
 
460
+ def load_parquet(self):
461
+ return runtime.load_parquet(self)
462
+
433
463
  def apply_is_valid_policy(self, kkeys):
434
464
  if self.data is None:
435
465
  self.load(force=False)
@@ -473,7 +503,7 @@ class DataSet():
473
503
  sps = self.data.shape
474
504
  mask = self.data[isvalids].all(axis=1)
475
505
  self.data = self.data[mask]
476
- self.logger.warning(
506
+ self.logger.info(
477
507
  "DataSet Shape: \n\t Before filtering -> {}\n\t After filtering -> {}".format(
478
508
  sps, self.data.shape
479
509
  )
@@ -3,7 +3,9 @@ from __future__ import annotations
3
3
  import os
4
4
  from typing import Any, Dict, List, Optional
5
5
 
6
- from .data_loader_summary import dataframe_summary_human_bytes
6
+ import h5py
7
+
8
+ from .data_loader_summary import dataframe_summary_human_bytes, format_table
7
9
 
8
10
  try:
9
11
  import polars as pl
@@ -118,6 +120,50 @@ def build_hdf5_whitelist(dataset) -> Optional[set[str]]:
118
120
  return with_isvalid
119
121
 
120
122
 
123
+ def scan_hdf5_leaf_metadata(path: str, group: Optional[str] = None) -> List[Dict[str, Any]]:
124
+ """Return lightweight metadata for leaf datasets without reading dataset bodies."""
125
+
126
+ def _walk(node, prefix: str = "") -> None:
127
+ for name, child in node.items():
128
+ child_path = f"{prefix}/{name}" if prefix else name
129
+ if isinstance(child, h5py.Dataset):
130
+ metadata.append(
131
+ {
132
+ "path": child_path,
133
+ "shape": tuple(getattr(child, "shape", ()) or ()),
134
+ "dtype": str(getattr(child, "dtype", "NA")),
135
+ }
136
+ )
137
+ elif isinstance(child, h5py.Group):
138
+ _walk(child, child_path)
139
+
140
+ group_name = str(group).strip() if group is not None else ""
141
+ metadata: List[Dict[str, Any]] = []
142
+
143
+ try:
144
+ with h5py.File(path, "r") as h5f:
145
+ if group_name:
146
+ if group_name not in h5f:
147
+ raise RuntimeError(f"HDF5 group '{group_name}' not found in '{path}'.")
148
+ root = h5f[group_name]
149
+ if not isinstance(root, h5py.Group):
150
+ raise RuntimeError(f"HDF5 group '{group_name}' in '{path}' is not a group.")
151
+ _walk(root, group_name)
152
+ else:
153
+ _walk(h5f, "")
154
+ except FileNotFoundError as e:
155
+ raise RuntimeError(f"HDF5 file not found: {path}") from e
156
+ except OSError as e:
157
+ raise RuntimeError(f"Unable to open HDF5 file '{path}': {e}") from e
158
+
159
+ if not metadata:
160
+ if group_name:
161
+ raise RuntimeError(f"HDF5 group '{group_name}' in '{path}' contains no leaf datasets.")
162
+ raise RuntimeError(f"No leaf datasets found in HDF5 file '{path}'.")
163
+
164
+ return metadata
165
+
166
+
121
167
  def shape_token(df) -> str:
122
168
  import pandas as pd
123
169
 
@@ -169,7 +215,7 @@ def materialized_cache_key(dataset):
169
215
  return dataset.cache.cache_key({"kind": "hdf5-polars-materialized", "source": source_fp})
170
216
 
171
217
 
172
- def materialized_summary(dataset, manifest: Dict[str, Any]) -> str:
218
+ def materialized_summary(dataset, manifest: Dict[str, Any], stats: Optional[Dict[str, Dict[str, Any]]] = None) -> str:
173
219
  rows = manifest.get("rows", "NA")
174
220
  cols = manifest.get("cols", len(manifest.get("columns", [])))
175
221
  parts = manifest.get("parts", 0)
@@ -187,6 +233,71 @@ def materialized_summary(dataset, manifest: Dict[str, Any]) -> str:
187
233
  if manifest.get("columns"):
188
234
  lines.append("=== Materialized Columns ===")
189
235
  lines.append(", ".join(str(c) for c in manifest.get("columns", [])))
236
+ if isinstance(stats, dict) and stats:
237
+ lines.append("=== Materialized Stats ===")
238
+ stat_rows = []
239
+ df = getattr(dataset, "data", None)
240
+ schema_names = None
241
+ schema_dtypes = {}
242
+ if pl is not None and isinstance(df, pl.LazyFrame):
243
+ try:
244
+ schema = df.collect_schema()
245
+ if hasattr(schema, "names"):
246
+ schema_names = set(str(x) for x in schema.names())
247
+ elif isinstance(schema, dict):
248
+ schema_names = set(str(x) for x in schema.keys())
249
+ schema_dtypes = {str(k): str(v) for k, v in schema.items()}
250
+ elif hasattr(schema, "items"):
251
+ schema_names = set(str(x) for x in schema.keys())
252
+ schema_dtypes = {str(k): str(v) for k, v in schema.items()}
253
+ except Exception:
254
+ schema_names = set()
255
+ elif pl is not None and isinstance(df, pl.DataFrame):
256
+ try:
257
+ schema_names = set(str(x) for x in df.columns)
258
+ except Exception:
259
+ schema_names = set()
260
+ elif hasattr(df, "columns"):
261
+ try:
262
+ schema_names = set(str(x) for x in getattr(df, "columns", []))
263
+ except Exception:
264
+ schema_names = set()
265
+ for col in manifest.get("columns", []):
266
+ if col not in stats:
267
+ continue
268
+ bounds = stats.get(col, {})
269
+ dtype = "NA"
270
+ nonnull_text = "NA"
271
+ if pl is not None and isinstance(df, pl.LazyFrame):
272
+ dtype = schema_dtypes.get(col, "NA")
273
+ elif schema_names is not None and col in schema_names and hasattr(df, "__getitem__"):
274
+ series = df[col]
275
+ dtype = str(series.dtype)
276
+ try:
277
+ nonnull_pct = (float(series.notna().sum()) / float(len(series)) * 100.0) if len(series) else 0.0
278
+ except Exception:
279
+ nonnull_pct = 0.0
280
+ nonnull_text = f"{nonnull_pct:.1f}%"
281
+ else:
282
+ dtype = str(type(bounds.get("min", None)).__name__)
283
+ stat_rows.append(
284
+ (
285
+ str(col),
286
+ dtype,
287
+ nonnull_text,
288
+ bounds.get("min", None),
289
+ bounds.get("max", None),
290
+ )
291
+ )
292
+
293
+ if stat_rows:
294
+ lines.append(
295
+ format_table(
296
+ ["name", "dtype", "nonnull%", "[min]", "[max]"],
297
+ stat_rows,
298
+ aligns=["left", "left", "right", "right", "right"],
299
+ )
300
+ )
190
301
  return "\n".join(lines)
191
302
  def sql_bool_ops(expr: Any) -> str:
192
303
  return str(expr).replace("&&", " AND ").replace("||", " OR ")