nd-prepare 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nd_prepare/__init__.py +11 -0
- nd_prepare/__main__.py +8 -0
- nd_prepare/builders.py +334 -0
- nd_prepare/cli.py +144 -0
- nd_prepare/common.py +32 -0
- nd_prepare/extract.py +78 -0
- nd_prepare/manifest.py +348 -0
- nd_prepare/pack.py +42 -0
- nd_prepare/prepare.py +182 -0
- nd_prepare/sources.json +867 -0
- nd_prepare/sources.py +129 -0
- nd_prepare/xsdir.py +346 -0
- nd_prepare-0.9.0.dist-info/METADATA +442 -0
- nd_prepare-0.9.0.dist-info/RECORD +18 -0
- nd_prepare-0.9.0.dist-info/WHEEL +5 -0
- nd_prepare-0.9.0.dist-info/entry_points.txt +2 -0
- nd_prepare-0.9.0.dist-info/licenses/COPYING +373 -0
- nd_prepare-0.9.0.dist-info/top_level.txt +1 -0
nd_prepare/__init__.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2026 ENEA
|
|
2
|
+
# SPDX-FileContributor: Alberto P
|
|
3
|
+
#
|
|
4
|
+
# SPDX-License-Identifier: MPL-2.0
|
|
5
|
+
|
|
6
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
7
|
+
|
|
8
|
+
try:
|
|
9
|
+
__version__ = version("nd-prepare")
|
|
10
|
+
except PackageNotFoundError:
|
|
11
|
+
__version__ = "unknown"
|
nd_prepare/__main__.py
ADDED
nd_prepare/builders.py
ADDED
|
@@ -0,0 +1,334 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2026 ENEA
|
|
2
|
+
# SPDX-FileContributor: Alberto P
|
|
3
|
+
#
|
|
4
|
+
# SPDX-License-Identifier: MPL-2.0
|
|
5
|
+
|
|
6
|
+
"""Build a library's payload directory (data files plus xsdir) from its sources.
|
|
7
|
+
|
|
8
|
+
Each mode builder extracts the archives into payload_dir and writes
|
|
9
|
+
payload_dir/xsdir. build_payload() then runs the mode-independent steps:
|
|
10
|
+
patches, tidying, the final xsdir pass (routes, blank records), nesting under
|
|
11
|
+
the library key and re-wrapping to 80 columns.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
import shutil
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from .common import log, log_wrapped
|
|
18
|
+
from .extract import extract_archive
|
|
19
|
+
from .sources import resolve_archive_source, source_label
|
|
20
|
+
from .xsdir import (
|
|
21
|
+
NAME_FIELD,
|
|
22
|
+
apply_xsdir_overrides,
|
|
23
|
+
ensure_newline,
|
|
24
|
+
extract_awr_table,
|
|
25
|
+
filter_existing_entries,
|
|
26
|
+
find_header_line,
|
|
27
|
+
is_blank_record,
|
|
28
|
+
is_xsdir_entry,
|
|
29
|
+
iter_xsdir_records,
|
|
30
|
+
normalize_xsdir_route,
|
|
31
|
+
prefix_xsdir_filename,
|
|
32
|
+
read_xsdir_text,
|
|
33
|
+
rename_xsdir_entries,
|
|
34
|
+
rename_xsdir_filenames,
|
|
35
|
+
set_xsdir_filename,
|
|
36
|
+
split_vendor_xsdir_header,
|
|
37
|
+
strip_xsdir_filename_prefixes,
|
|
38
|
+
wrap_xsdir_entries,
|
|
39
|
+
write_xsdir_text,
|
|
40
|
+
xsdir_record_fields,
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
# Where a builder parks the vendor's atomic-weight-ratio table until
|
|
44
|
+
# nest_payload_data moves it to docs/awr_<key>.
|
|
45
|
+
AWR_STAGING_NAME = "awr_table"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def stage_awr_table(header, payload_dir):
|
|
49
|
+
"""Keep the vendor's atomic-weight-ratio table as reference material: it
|
|
50
|
+
cannot stay in the xsdir, but its AWRs may differ from nd_manager's."""
|
|
51
|
+
awr_table = extract_awr_table(header)
|
|
52
|
+
if awr_table is not None:
|
|
53
|
+
write_xsdir_text(payload_dir / AWR_STAGING_NAME, awr_table)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _extracted(root, name, what, check=Path.is_file):
|
|
57
|
+
path = root / name
|
|
58
|
+
if not check(path):
|
|
59
|
+
raise RuntimeError(f"expected {what} not found after extraction: {name}")
|
|
60
|
+
return path
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def flatten_payload_dirs(payload_dir, dirnames):
|
|
64
|
+
"""Move everything under payload_dir/<dirname> up to payload_dir, for each
|
|
65
|
+
dirname, and remove the emptied directory chain (both "ace" and "neutron"
|
|
66
|
+
of "neutron/ace"). The caller strips the same prefixes from the xsdir."""
|
|
67
|
+
for dirname in dirnames:
|
|
68
|
+
src_dir = _extracted(payload_dir, dirname, "directory to flatten", Path.is_dir)
|
|
69
|
+
for item in sorted(src_dir.iterdir()):
|
|
70
|
+
target = payload_dir / item.name
|
|
71
|
+
if target.exists():
|
|
72
|
+
raise RuntimeError(f"cannot flatten {dirname}: {item.name} already exists in the payload root")
|
|
73
|
+
item.rename(target)
|
|
74
|
+
directory = src_dir
|
|
75
|
+
while directory != payload_dir:
|
|
76
|
+
try:
|
|
77
|
+
directory.rmdir()
|
|
78
|
+
except OSError as exc:
|
|
79
|
+
leftover = ", ".join(sorted(p.name for p in directory.iterdir()))
|
|
80
|
+
raise RuntimeError(
|
|
81
|
+
f"cannot flatten {dirname}: {directory.relative_to(payload_dir)} still holds {leftover}"
|
|
82
|
+
) from exc
|
|
83
|
+
directory = directory.parent
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _extract_archives(spec, cache_dir, payload_dir, strip_components=0):
|
|
87
|
+
seen = set()
|
|
88
|
+
for archive in spec["archives"]:
|
|
89
|
+
cached = resolve_archive_source(archive, cache_dir)
|
|
90
|
+
written = extract_archive(cached, archive["kind"], payload_dir, strip_components)
|
|
91
|
+
# Archives share one directory; they should be disjoint.
|
|
92
|
+
collisions = sorted(seen.intersection(written))
|
|
93
|
+
if collisions:
|
|
94
|
+
log(f"{cached.name}: overwrote {len(collisions)} file(s) extracted from an earlier archive:")
|
|
95
|
+
log_wrapped(collisions)
|
|
96
|
+
seen.update(written)
|
|
97
|
+
|
|
98
|
+
# Names that differ only in case are two files here but one on Windows and macOS.
|
|
99
|
+
by_lowercase = {}
|
|
100
|
+
for name in seen:
|
|
101
|
+
by_lowercase.setdefault(name.lower(), []).append(name)
|
|
102
|
+
clashes = sorted(" / ".join(sorted(names)) for names in by_lowercase.values() if len(names) > 1)
|
|
103
|
+
if clashes:
|
|
104
|
+
log(f"warning: {len(clashes)} file name(s) differ only in case; a case-insensitive file system keeps one:")
|
|
105
|
+
log_wrapped(clashes)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _read_vendor_xsdir(source, cache_dir, field):
|
|
109
|
+
"""(header, entries) of an xsdir source; raises if it has no "directory" line."""
|
|
110
|
+
raw_text = read_xsdir_text(resolve_archive_source(source, cache_dir))
|
|
111
|
+
header, entries = split_vendor_xsdir_header(raw_text)
|
|
112
|
+
if entries == raw_text:
|
|
113
|
+
raise RuntimeError(f"no 'directory' header line found in {field}: {source_label(source)}")
|
|
114
|
+
return header, entries
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def build_payload_ready(spec, cache_dir, payload_dir):
|
|
118
|
+
_extract_archives(spec, cache_dir, payload_dir)
|
|
119
|
+
|
|
120
|
+
# Flatten before reading the xsdir and strip its prefixes before filtering,
|
|
121
|
+
# so the filter looks where the files now are.
|
|
122
|
+
flatten_dirs = spec.get("flatten_dirs", [])
|
|
123
|
+
flatten_payload_dirs(payload_dir, flatten_dirs)
|
|
124
|
+
|
|
125
|
+
for old_name, new_name in spec.get("rename_members", {}).items():
|
|
126
|
+
old_path = _extracted(payload_dir, old_name, "member to rename", Path.exists)
|
|
127
|
+
new_path = payload_dir / new_name
|
|
128
|
+
if new_path.exists():
|
|
129
|
+
raise RuntimeError(f"cannot rename {old_name}: {new_name} already exists in the payload")
|
|
130
|
+
old_path.rename(new_path)
|
|
131
|
+
|
|
132
|
+
# Unlike drop_members, discarded files are removed before the xsdir is
|
|
133
|
+
# filtered, so their xsdir entries go too.
|
|
134
|
+
for discarded in spec.get("discard_members", []):
|
|
135
|
+
discarded_path = _extracted(payload_dir, discarded, "member to discard")
|
|
136
|
+
log(f"discarding {discarded}")
|
|
137
|
+
discarded_path.unlink()
|
|
138
|
+
|
|
139
|
+
xsdir_member = _extracted(payload_dir, spec["xsdir_member"], "xsdir member", Path.exists)
|
|
140
|
+
xsdir_path = payload_dir / "xsdir"
|
|
141
|
+
xsdir_member.rename(xsdir_path)
|
|
142
|
+
raw_text = read_xsdir_text(xsdir_path)
|
|
143
|
+
header, text = split_vendor_xsdir_header(raw_text)
|
|
144
|
+
# The "directory" line is optional here: some vendors ship entries only.
|
|
145
|
+
stray = find_header_line(raw_text) if text == raw_text else None
|
|
146
|
+
if stray is not None:
|
|
147
|
+
raise RuntimeError(f"{spec['xsdir_member']} has a header line ({stray!r}) but no 'directory' line")
|
|
148
|
+
stage_awr_table(header, payload_dir)
|
|
149
|
+
text = strip_xsdir_filename_prefixes(text, flatten_dirs)
|
|
150
|
+
text = filter_existing_entries(text, payload_dir)
|
|
151
|
+
if "rename_xsdir_tables" in spec:
|
|
152
|
+
text = rename_xsdir_entries(text, spec["rename_xsdir_tables"])
|
|
153
|
+
write_xsdir_text(xsdir_path, text)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def build_payload_fragments(spec, cache_dir, payload_dir):
|
|
157
|
+
_extract_archives(spec, cache_dir, payload_dir)
|
|
158
|
+
|
|
159
|
+
fragments = sorted(payload_dir.glob(spec["fragment_glob"]))
|
|
160
|
+
if not fragments:
|
|
161
|
+
raise RuntimeError(f"no xsdir fragments matched glob: {spec['fragment_glob']}")
|
|
162
|
+
|
|
163
|
+
lines = []
|
|
164
|
+
for fragment in fragments:
|
|
165
|
+
rel_dir = fragment.parent.relative_to(payload_dir).as_posix()
|
|
166
|
+
for record in iter_xsdir_records(read_xsdir_text(fragment)):
|
|
167
|
+
if is_blank_record(record):
|
|
168
|
+
continue
|
|
169
|
+
if not is_xsdir_entry(record):
|
|
170
|
+
raise RuntimeError(f"unexpected non-entry line in xsdir fragment {fragment}: {''.join(record).strip()}")
|
|
171
|
+
# The fragment holds a bare filename; make it relative to the archive
|
|
172
|
+
# root (a fragment at the root already is).
|
|
173
|
+
if rel_dir != ".":
|
|
174
|
+
record = prefix_xsdir_filename(record, rel_dir)
|
|
175
|
+
lines.extend(ensure_newline(record))
|
|
176
|
+
|
|
177
|
+
# Merged into xsdir, so the fragments are dead weight in the archive.
|
|
178
|
+
for fragment in fragments:
|
|
179
|
+
fragment.unlink()
|
|
180
|
+
|
|
181
|
+
flatten_dirs = spec.get("flatten_dirs", [])
|
|
182
|
+
flatten_payload_dirs(payload_dir, flatten_dirs)
|
|
183
|
+
# No filter_existing_entries: every entry comes from a fragment of a payload file.
|
|
184
|
+
text = strip_xsdir_filename_prefixes("".join(lines), flatten_dirs)
|
|
185
|
+
if "rename_xsdir_files" in spec:
|
|
186
|
+
text = rename_xsdir_filenames(text, spec["rename_xsdir_files"], payload_dir)
|
|
187
|
+
write_xsdir_text(payload_dir / "xsdir", text)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def build_payload_merge_master(spec, cache_dir, payload_dir):
|
|
191
|
+
_extract_archives(spec, cache_dir, payload_dir, 1 if spec.get("strip_top_level_dir") else 0)
|
|
192
|
+
|
|
193
|
+
header, text = _read_vendor_xsdir(spec["xsdir_source"], cache_dir, "xsdir_source")
|
|
194
|
+
stage_awr_table(header, payload_dir)
|
|
195
|
+
if "xsdir_overrides" in spec:
|
|
196
|
+
override_source = spec["xsdir_overrides"]
|
|
197
|
+
_, override_text = _read_vendor_xsdir(override_source, cache_dir, "xsdir_overrides")
|
|
198
|
+
text, replaced = apply_xsdir_overrides(text, override_text, override_source["entries"])
|
|
199
|
+
log(f"xsdir: replaced {len(replaced)} entries with the versions from {source_label(override_source)}:")
|
|
200
|
+
log_wrapped(replaced)
|
|
201
|
+
# The xsdir may also list sibling sublibraries built from another archive.
|
|
202
|
+
write_xsdir_text(payload_dir / "xsdir", filter_existing_entries(text, payload_dir))
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
BUILDERS = {
|
|
206
|
+
"ready": build_payload_ready,
|
|
207
|
+
"fragments": build_payload_fragments,
|
|
208
|
+
"merge-master": build_payload_merge_master,
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _patch_xsdir_entry(extract_dir, file_spec):
|
|
213
|
+
"""The xsdir record for a patch file, from its single-entry fragment, with
|
|
214
|
+
the fragment's placeholder filename replaced. Returns (zaid, record)."""
|
|
215
|
+
fragment_name = file_spec["xsdir_fragment"]
|
|
216
|
+
records = [
|
|
217
|
+
record for record in iter_xsdir_records(read_xsdir_text(extract_dir / fragment_name))
|
|
218
|
+
if not is_blank_record(record)
|
|
219
|
+
]
|
|
220
|
+
if len(records) != 1 or not is_xsdir_entry(records[0]):
|
|
221
|
+
raise RuntimeError(f"expected exactly one xsdir entry in patch fragment {fragment_name}")
|
|
222
|
+
(record,) = records
|
|
223
|
+
return xsdir_record_fields(record)[NAME_FIELD], ensure_newline(set_xsdir_filename(record, file_spec["dest"]))
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def apply_patches(spec, cache_dir, payload_dir):
|
|
227
|
+
"""Overlay supplementary files released after a library's main archive.
|
|
228
|
+
|
|
229
|
+
Each patch file replaces or adds a data file; with an xsdir_fragment, its
|
|
230
|
+
entry replaces the one with the same ZAID, or is appended if none matches.
|
|
231
|
+
"""
|
|
232
|
+
patches = spec.get("patches", [])
|
|
233
|
+
if not patches:
|
|
234
|
+
return
|
|
235
|
+
xsdir_path = payload_dir / "xsdir"
|
|
236
|
+
records = list(iter_xsdir_records(read_xsdir_text(xsdir_path)))
|
|
237
|
+
for index, patch in enumerate(patches):
|
|
238
|
+
cached = resolve_archive_source(patch, cache_dir)
|
|
239
|
+
extract_dir = payload_dir.parent / f"patch_{index}"
|
|
240
|
+
if extract_dir.exists():
|
|
241
|
+
shutil.rmtree(extract_dir)
|
|
242
|
+
extract_archive(cached, patch["kind"], extract_dir)
|
|
243
|
+
|
|
244
|
+
for file_spec in patch["files"]:
|
|
245
|
+
src = _extracted(extract_dir, file_spec["member"], "patch member")
|
|
246
|
+
dest = payload_dir / file_spec["dest"]
|
|
247
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
248
|
+
shutil.copyfile(src, dest)
|
|
249
|
+
|
|
250
|
+
if "xsdir_fragment" in file_spec:
|
|
251
|
+
zaid, new_record = _patch_xsdir_entry(extract_dir, file_spec)
|
|
252
|
+
for i, record in enumerate(records):
|
|
253
|
+
if is_xsdir_entry(record) and xsdir_record_fields(record)[NAME_FIELD] == zaid:
|
|
254
|
+
records[i] = new_record
|
|
255
|
+
break
|
|
256
|
+
else:
|
|
257
|
+
if records:
|
|
258
|
+
records[-1] = ensure_newline(records[-1])
|
|
259
|
+
records.append(new_record)
|
|
260
|
+
|
|
261
|
+
shutil.rmtree(extract_dir, ignore_errors=True)
|
|
262
|
+
write_xsdir_text(xsdir_path, "".join(line for record in records for line in record))
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def tidy_payload(spec, payload_dir):
|
|
266
|
+
"""Delete the drop_members, and move readme_member to a top-level README.md
|
|
267
|
+
(nd_manager install renames it to docs/README_<LibraryName>.md)."""
|
|
268
|
+
for dropped in spec.get("drop_members", []):
|
|
269
|
+
_extracted(payload_dir, dropped, "member to drop").unlink()
|
|
270
|
+
|
|
271
|
+
if "readme_member" in spec:
|
|
272
|
+
readme = _extracted(payload_dir, spec["readme_member"], "readme member")
|
|
273
|
+
if (payload_dir / "README.md").exists():
|
|
274
|
+
raise RuntimeError(f"cannot move {spec['readme_member']} to README.md: the payload already has one")
|
|
275
|
+
readme.rename(payload_dir / "README.md")
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def finalize_xsdir(payload_dir):
|
|
279
|
+
"""Reset every entry's access route to 0 and drop blank records, whichever
|
|
280
|
+
builder or patch produced them."""
|
|
281
|
+
xsdir_path = payload_dir / "xsdir"
|
|
282
|
+
text = normalize_xsdir_route(read_xsdir_text(xsdir_path))
|
|
283
|
+
kept = [
|
|
284
|
+
line
|
|
285
|
+
for record in iter_xsdir_records(text) if not is_blank_record(record)
|
|
286
|
+
for line in record
|
|
287
|
+
]
|
|
288
|
+
write_xsdir_text(xsdir_path, "".join(kept))
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def nest_payload_data(payload_dir, key):
|
|
292
|
+
"""Nest everything except xsdir and README.md under a <key>/ subdirectory,
|
|
293
|
+
so nd_manager install gives each library its own data directory. The
|
|
294
|
+
staged AWR table goes to docs/awr_<key> instead."""
|
|
295
|
+
kept_at_root = {key, "xsdir", "README.md"}
|
|
296
|
+
staged_awr = payload_dir / AWR_STAGING_NAME
|
|
297
|
+
if staged_awr.exists():
|
|
298
|
+
docs_dir = payload_dir / "docs"
|
|
299
|
+
# Fail rather than mix the AWR table into, or hide, a vendor docs/.
|
|
300
|
+
if docs_dir.exists():
|
|
301
|
+
raise RuntimeError(f"cannot write docs/awr_{key}: the payload root already has a docs entry")
|
|
302
|
+
docs_dir.mkdir()
|
|
303
|
+
staged_awr.rename(docs_dir / f"awr_{key}")
|
|
304
|
+
kept_at_root.add("docs")
|
|
305
|
+
|
|
306
|
+
nested = payload_dir / key
|
|
307
|
+
if nested.exists():
|
|
308
|
+
raise RuntimeError(f"cannot nest the payload under {key}/: the payload root already has a {key} entry")
|
|
309
|
+
nested.mkdir()
|
|
310
|
+
for item in payload_dir.iterdir():
|
|
311
|
+
if item.name not in kept_at_root:
|
|
312
|
+
item.rename(nested / item.name)
|
|
313
|
+
|
|
314
|
+
xsdir_path = payload_dir / "xsdir"
|
|
315
|
+
rewritten = []
|
|
316
|
+
for record in iter_xsdir_records(read_xsdir_text(xsdir_path)):
|
|
317
|
+
rewritten.extend(prefix_xsdir_filename(record, key) if is_xsdir_entry(record) else record)
|
|
318
|
+
write_xsdir_text(xsdir_path, "".join(rewritten))
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def wrap_payload_xsdir(payload_dir):
|
|
322
|
+
"""Re-wrap the xsdir entries to the MCNP line width. Runs after
|
|
323
|
+
nest_payload_data, whose <key>/ filename prefix lengthens the lines."""
|
|
324
|
+
xsdir_path = payload_dir / "xsdir"
|
|
325
|
+
write_xsdir_text(xsdir_path, wrap_xsdir_entries(read_xsdir_text(xsdir_path)))
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def build_payload(spec, cache_dir, payload_dir, key):
|
|
329
|
+
BUILDERS[spec["mode"]](spec, cache_dir, payload_dir)
|
|
330
|
+
apply_patches(spec, cache_dir, payload_dir)
|
|
331
|
+
tidy_payload(spec, payload_dir)
|
|
332
|
+
finalize_xsdir(payload_dir)
|
|
333
|
+
nest_payload_data(payload_dir, key)
|
|
334
|
+
wrap_payload_xsdir(payload_dir)
|
nd_prepare/cli.py
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2026 ENEA
|
|
2
|
+
# SPDX-FileContributor: Alberto P
|
|
3
|
+
#
|
|
4
|
+
# SPDX-License-Identifier: MPL-2.0
|
|
5
|
+
|
|
6
|
+
"""Command-line interface for nd-prepare: download and repackage external nuclear
|
|
7
|
+
data libraries into nd_manager local-repository format.
|
|
8
|
+
|
|
9
|
+
See sources.json for the per-library manifest this reads, and README.md for
|
|
10
|
+
usage notes and known caveats.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import argparse
|
|
14
|
+
import subprocess
|
|
15
|
+
import sys
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
from . import __version__, prepare
|
|
19
|
+
from .common import PACKAGE_DIR, log
|
|
20
|
+
from .manifest import (
|
|
21
|
+
format_manifest_listing,
|
|
22
|
+
load_manifest,
|
|
23
|
+
selected_entries,
|
|
24
|
+
validate_only,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def build_parser():
|
|
29
|
+
parser = argparse.ArgumentParser(prog="nd-prepare", description=__doc__)
|
|
30
|
+
parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
31
|
+
parser.add_argument(
|
|
32
|
+
"--manifest", default=str(PACKAGE_DIR / "sources.json"),
|
|
33
|
+
help="Path to the sources.json manifest to read (default: the copy bundled with the package).",
|
|
34
|
+
)
|
|
35
|
+
parser.add_argument(
|
|
36
|
+
"--workdir", default=None,
|
|
37
|
+
help="Scratch directory for extraction (default: ./work).",
|
|
38
|
+
)
|
|
39
|
+
parser.add_argument(
|
|
40
|
+
"--only", nargs="+", default=None,
|
|
41
|
+
help="Only process these library keys (e.g. FENDL31D). Suffix a key with :neutron, :tsl, "
|
|
42
|
+
":proton, or :dosimetry to process just one part.",
|
|
43
|
+
)
|
|
44
|
+
parser.add_argument(
|
|
45
|
+
"--list", action="store_true",
|
|
46
|
+
help="List the library keys and categories in the manifest (the values --only accepts), then exit.",
|
|
47
|
+
)
|
|
48
|
+
parser.add_argument(
|
|
49
|
+
"--download-only", action="store_true",
|
|
50
|
+
help="Only download the selected sources into cache/, then exit. A later run without "
|
|
51
|
+
"this flag builds purely from cache.",
|
|
52
|
+
)
|
|
53
|
+
parser.add_argument(
|
|
54
|
+
"--cleanup", action="store_true",
|
|
55
|
+
help="After each entry is packed and hashed, delete its extracted payload and cached "
|
|
56
|
+
"downloads (default: keep them so re-runs are cheap).",
|
|
57
|
+
)
|
|
58
|
+
parser.add_argument(
|
|
59
|
+
"--cleanup-work", action="store_true",
|
|
60
|
+
help="After each entry is packed and hashed, delete only its extracted payload under the "
|
|
61
|
+
"workdir, keeping the cached downloads (a subset of --cleanup).",
|
|
62
|
+
)
|
|
63
|
+
parser.add_argument(
|
|
64
|
+
"--compression", choices=["gz", "xz"], default="gz",
|
|
65
|
+
help="Archive compression: gz (default) is faster, xz smaller.",
|
|
66
|
+
)
|
|
67
|
+
parser.add_argument(
|
|
68
|
+
"--prune", action="store_true",
|
|
69
|
+
help="Remove the entries in output/libraries.json that the manifest no longer produces, "
|
|
70
|
+
"and delete their archives from output/, then exit.",
|
|
71
|
+
)
|
|
72
|
+
return parser
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def run_entries(args, manifest, settings, libraries_json, stale):
|
|
76
|
+
only = None
|
|
77
|
+
if args.only:
|
|
78
|
+
only = set(args.only)
|
|
79
|
+
validate_only(only, manifest)
|
|
80
|
+
|
|
81
|
+
handled = []
|
|
82
|
+
names_by_entry = {}
|
|
83
|
+
for entry_key, spec in selected_entries(manifest, only):
|
|
84
|
+
handled.append(entry_key)
|
|
85
|
+
if args.download_only:
|
|
86
|
+
prepare.download_entry(entry_key, spec, settings.cache_dir)
|
|
87
|
+
else:
|
|
88
|
+
names_by_entry[entry_key] = prepare.process_entry(entry_key, spec, settings, libraries_json)
|
|
89
|
+
prepare.save_output_libraries_json(settings.libraries_json_path, libraries_json)
|
|
90
|
+
|
|
91
|
+
if not handled:
|
|
92
|
+
log("nothing selected; no entries processed")
|
|
93
|
+
elif args.download_only:
|
|
94
|
+
log(f"download-only: sources for selected entries are cached under {settings.cache_dir}")
|
|
95
|
+
else:
|
|
96
|
+
log(f"libraries.json written to {settings.libraries_json_path}")
|
|
97
|
+
|
|
98
|
+
prepare.report_table_name_collisions(names_by_entry)
|
|
99
|
+
if stale and not args.download_only:
|
|
100
|
+
prepare.warn_stale_entries(stale)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def run():
|
|
104
|
+
args = build_parser().parse_args()
|
|
105
|
+
manifest = load_manifest(Path(args.manifest).resolve())
|
|
106
|
+
|
|
107
|
+
if args.list:
|
|
108
|
+
print(format_manifest_listing(manifest))
|
|
109
|
+
return
|
|
110
|
+
|
|
111
|
+
# Run artifacts go under the current directory, not next to the (installed)
|
|
112
|
+
# manifest; local "path" sources are also resolved against it.
|
|
113
|
+
settings = prepare.RunSettings(
|
|
114
|
+
cache_dir=Path.cwd() / manifest["cache_dir"],
|
|
115
|
+
output_dir=Path.cwd() / manifest["output_dir"],
|
|
116
|
+
workdir=Path(args.workdir).resolve() if args.workdir else Path.cwd() / "work",
|
|
117
|
+
compression=args.compression,
|
|
118
|
+
cleanup=args.cleanup,
|
|
119
|
+
cleanup_work=args.cleanup_work,
|
|
120
|
+
)
|
|
121
|
+
libraries_json = prepare.load_output_libraries_json(settings.libraries_json_path)
|
|
122
|
+
stale = prepare.stale_library_keys(libraries_json, manifest)
|
|
123
|
+
|
|
124
|
+
if args.prune:
|
|
125
|
+
if stale:
|
|
126
|
+
prepare.prune_stale_entries(libraries_json, stale, settings.output_dir)
|
|
127
|
+
prepare.save_output_libraries_json(settings.libraries_json_path, libraries_json)
|
|
128
|
+
else:
|
|
129
|
+
log("prune: libraries.json has no stale entries")
|
|
130
|
+
return
|
|
131
|
+
|
|
132
|
+
run_entries(args, manifest, settings, libraries_json, stale)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def main():
|
|
136
|
+
# Expected failures exit with a one-line message instead of a traceback.
|
|
137
|
+
try:
|
|
138
|
+
run()
|
|
139
|
+
except subprocess.SubprocessError as exc:
|
|
140
|
+
sys.exit(f"download failed: {exc}")
|
|
141
|
+
except (RuntimeError, OSError) as exc:
|
|
142
|
+
sys.exit(f"error: {exc}")
|
|
143
|
+
except KeyboardInterrupt:
|
|
144
|
+
sys.exit(130)
|
nd_prepare/common.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2026 ENEA
|
|
2
|
+
# SPDX-FileContributor: Alberto P
|
|
3
|
+
#
|
|
4
|
+
# SPDX-License-Identifier: MPL-2.0
|
|
5
|
+
|
|
6
|
+
"""Constants and logging helpers shared by the nd_prepare modules."""
|
|
7
|
+
|
|
8
|
+
import textwrap
|
|
9
|
+
from datetime import date, datetime, time, timezone
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
PACKAGE_DIR = Path(__file__).resolve().parent
|
|
13
|
+
CATEGORIES = ("neutron", "tsl", "proton", "dosimetry")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def date_to_epoch(text):
|
|
17
|
+
"""Seconds since the epoch of midnight UTC on a YYYY-MM-DD date; ValueError otherwise."""
|
|
18
|
+
return int(datetime.combine(date.fromisoformat(text), time(), tzinfo=timezone.utc).timestamp())
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def log(msg):
|
|
22
|
+
print(f"[prepare] {msg}", flush=True)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def log_wrapped(items, indent=" "):
|
|
26
|
+
"""Log items space-separated, wrapped to 100 columns."""
|
|
27
|
+
for line in textwrap.wrap(" ".join(items), width=100):
|
|
28
|
+
log(f"{indent}{line}")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def plural_entries(count):
|
|
32
|
+
return "entry" if count == 1 else "entries"
|
nd_prepare/extract.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2026 ENEA
|
|
2
|
+
# SPDX-FileContributor: Alberto P
|
|
3
|
+
#
|
|
4
|
+
# SPDX-License-Identifier: MPL-2.0
|
|
5
|
+
|
|
6
|
+
"""Safe extraction of zip and tar archives (regular files only)."""
|
|
7
|
+
|
|
8
|
+
import shutil
|
|
9
|
+
import tarfile
|
|
10
|
+
import zipfile
|
|
11
|
+
from pathlib import Path, PurePosixPath, PureWindowsPath
|
|
12
|
+
|
|
13
|
+
from .common import log
|
|
14
|
+
|
|
15
|
+
JUNK_DIR_NAMES = {"__MACOSX"}
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _member_name(raw_name, strip_components):
|
|
19
|
+
"""Normalized relative name of an archive member after dropping its leading
|
|
20
|
+
components, or None if nothing is left or it is junk. Raises on a
|
|
21
|
+
zip-slip / path-traversal name."""
|
|
22
|
+
# Checked as POSIX and Windows paths: a drive letter or backslash is harmless on one, an escape on the other.
|
|
23
|
+
posix = PurePosixPath(raw_name)
|
|
24
|
+
windows = PureWindowsPath(raw_name)
|
|
25
|
+
if posix.is_absolute() or windows.anchor or ".." in posix.parts or ".." in windows.parts:
|
|
26
|
+
raise RuntimeError(f"unsafe path in archive: {raw_name}")
|
|
27
|
+
parts = posix.parts
|
|
28
|
+
if any(part.startswith(".") or part in JUNK_DIR_NAMES for part in parts):
|
|
29
|
+
return None
|
|
30
|
+
parts = parts[strip_components:]
|
|
31
|
+
return Path(*parts).as_posix() if parts else None
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _write_member(src, dest_dir, name):
|
|
35
|
+
target = dest_dir / name
|
|
36
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
37
|
+
with src, open(target, "wb") as out:
|
|
38
|
+
shutil.copyfileobj(src, out)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def extract_zip(archive_path, dest_dir, strip_components=0):
|
|
42
|
+
written = []
|
|
43
|
+
with zipfile.ZipFile(archive_path) as zf:
|
|
44
|
+
for info in zf.infolist():
|
|
45
|
+
if info.is_dir():
|
|
46
|
+
continue
|
|
47
|
+
name = _member_name(info.filename, strip_components)
|
|
48
|
+
if name is not None:
|
|
49
|
+
_write_member(zf.open(info), dest_dir, name)
|
|
50
|
+
written.append(name)
|
|
51
|
+
return written
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def extract_tar(archive_path, dest_dir, strip_components=0):
|
|
55
|
+
written = []
|
|
56
|
+
# Stream mode ("r|*") decompresses the archive once, front to back; random
|
|
57
|
+
# access would decompress a gzip stream again on every seek back.
|
|
58
|
+
with tarfile.open(archive_path, mode="r|*") as tf:
|
|
59
|
+
for member in tf:
|
|
60
|
+
if not member.isfile():
|
|
61
|
+
if member.issym() or member.islnk():
|
|
62
|
+
log(f"skipping link member: {member.name}")
|
|
63
|
+
continue
|
|
64
|
+
name = _member_name(member.name, strip_components)
|
|
65
|
+
if name is not None:
|
|
66
|
+
_write_member(tf.extractfile(member), dest_dir, name)
|
|
67
|
+
written.append(name)
|
|
68
|
+
return written
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def extract_archive(archive_path, kind, dest_dir, strip_components=0):
|
|
72
|
+
"""Extract an archive's regular files into dest_dir; return the relative
|
|
73
|
+
names written."""
|
|
74
|
+
if kind == "zip":
|
|
75
|
+
return extract_zip(archive_path, dest_dir, strip_components)
|
|
76
|
+
if kind == "tar":
|
|
77
|
+
return extract_tar(archive_path, dest_dir, strip_components)
|
|
78
|
+
raise ValueError(f"unknown archive kind: {kind}")
|