galform-execution 0.2.2__tar.gz → 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {galform_execution-0.2.2 → galform_execution-0.2.4}/PKG-INFO +1 -1
- {galform_execution-0.2.2 → galform_execution-0.2.4}/galform_execution/__init__.py +1 -1
- galform_execution-0.2.4/galform_execution/config/redshift_lists/COLIBRE-L100m6.txt +138 -0
- galform_execution-0.2.4/galform_execution/config/redshift_lists/COLIBRE-L200m6.txt +142 -0
- {galform_execution-0.2.2 → galform_execution-0.2.4}/galform_execution/config/redshift_lists/Mill1.txt +1 -0
- galform_execution-0.2.4/galform_execution/config/simulations/colibre_family.json +66 -0
- {galform_execution-0.2.2 → galform_execution-0.2.4}/galform_execution/submit_galform_job.py +58 -6
- {galform_execution-0.2.2 → galform_execution-0.2.4}/galform_execution.egg-info/PKG-INFO +1 -1
- {galform_execution-0.2.2 → galform_execution-0.2.4}/galform_execution.egg-info/SOURCES.txt +1 -0
- {galform_execution-0.2.2 → galform_execution-0.2.4}/pyproject.toml +9 -1
- {galform_execution-0.2.2 → galform_execution-0.2.4}/tests/test_submit_galform_job.py +112 -0
- galform_execution-0.2.2/galform_execution/config/redshift_lists/COLIBRE-L100m6.txt +0 -3
- galform_execution-0.2.2/galform_execution/config/simulations/colibre_family.json +0 -43
- {galform_execution-0.2.2 → galform_execution-0.2.4}/LICENSE +0 -0
- {galform_execution-0.2.2 → galform_execution-0.2.4}/README.md +0 -0
- {galform_execution-0.2.2 → galform_execution-0.2.4}/galform_execution/__main__.py +0 -0
- {galform_execution-0.2.2 → galform_execution-0.2.4}/galform_execution/config/dust_params.json +0 -0
- {galform_execution-0.2.2 → galform_execution-0.2.4}/galform_execution/config/models.json +0 -0
- {galform_execution-0.2.2 → galform_execution-0.2.4}/galform_execution/config/partition_configs.json +0 -0
- {galform_execution-0.2.2 → galform_execution-0.2.4}/galform_execution/config/redshift_lists/FLAMINGO-L1000N1800.txt +0 -0
- {galform_execution-0.2.2 → galform_execution-0.2.4}/galform_execution/config/redshift_lists/L800.txt +0 -0
- {galform_execution-0.2.2 → galform_execution-0.2.4}/galform_execution/config/redshift_lists/Mill2.txt +0 -0
- {galform_execution-0.2.2 → galform_execution-0.2.4}/galform_execution/config/run_flags.json +0 -0
- {galform_execution-0.2.2 → galform_execution-0.2.4}/galform_execution/config/simulations/dove_family.json +0 -0
- {galform_execution-0.2.2 → galform_execution-0.2.4}/galform_execution/config/simulations/eagle_family.json +0 -0
- {galform_execution-0.2.2 → galform_execution-0.2.4}/galform_execution/config/simulations/flamingo_family.json +0 -0
- {galform_execution-0.2.2 → galform_execution-0.2.4}/galform_execution/config/simulations/millennium_family.json +0 -0
- {galform_execution-0.2.2 → galform_execution-0.2.4}/galform_execution/config/simulations/nifty_family.json +0 -0
- {galform_execution-0.2.2 → galform_execution-0.2.4}/galform_execution.egg-info/dependency_links.txt +0 -0
- {galform_execution-0.2.2 → galform_execution-0.2.4}/galform_execution.egg-info/entry_points.txt +0 -0
- {galform_execution-0.2.2 → galform_execution-0.2.4}/galform_execution.egg-info/top_level.txt +0 -0
- {galform_execution-0.2.2 → galform_execution-0.2.4}/setup.cfg +0 -0
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
# COLIBRE-L100m6 snapshot -> output redshift.
|
|
2
|
+
# Full 128-snapshot table, generated 2026-07-28 from the authoritative
|
|
3
|
+
# outputTimes/{snapshotNumber,redshift} datasets in
|
|
4
|
+
# /cosma8/data/dp004/jch/COLIBRE/HBT/L100_m6/DMO/trees/tree_127.0.hdf5.
|
|
5
|
+
# Previously only snapshots 18/92/102/127 were listed, which silently
|
|
6
|
+
# capped every COLIBRE run to z in {10, 1, 0.5, 0}: the tcsh script does
|
|
7
|
+
# `set z = awk '$1==iz {print $2}' $snapshot_file` and aborts on a miss,
|
|
8
|
+
# so an unlisted snapshot was simply not runnable. All four original
|
|
9
|
+
# entries were re-verified against the tree metadata before extending.
|
|
10
|
+
# iz z
|
|
11
|
+
0 30
|
|
12
|
+
1 25
|
|
13
|
+
2 22.5
|
|
14
|
+
3 20
|
|
15
|
+
4 19
|
|
16
|
+
5 18
|
|
17
|
+
6 17
|
|
18
|
+
7 16
|
|
19
|
+
8 15
|
|
20
|
+
9 14.5
|
|
21
|
+
10 14
|
|
22
|
+
11 13.5
|
|
23
|
+
12 13
|
|
24
|
+
13 12.5
|
|
25
|
+
14 12
|
|
26
|
+
15 11.5
|
|
27
|
+
16 11
|
|
28
|
+
17 10.5
|
|
29
|
+
18 10
|
|
30
|
+
19 9.75
|
|
31
|
+
20 9.5
|
|
32
|
+
21 9.25
|
|
33
|
+
22 9
|
|
34
|
+
23 8.75
|
|
35
|
+
24 8.5
|
|
36
|
+
25 8.25
|
|
37
|
+
26 8
|
|
38
|
+
27 7.75
|
|
39
|
+
28 7.5
|
|
40
|
+
29 7.375
|
|
41
|
+
30 7.25
|
|
42
|
+
31 7.125
|
|
43
|
+
32 7
|
|
44
|
+
33 6.875
|
|
45
|
+
34 6.75
|
|
46
|
+
35 6.625
|
|
47
|
+
36 6.5
|
|
48
|
+
37 6.375
|
|
49
|
+
38 6.25
|
|
50
|
+
39 6.125
|
|
51
|
+
40 6
|
|
52
|
+
41 5.875
|
|
53
|
+
42 5.75
|
|
54
|
+
43 5.625
|
|
55
|
+
44 5.5
|
|
56
|
+
45 5.375
|
|
57
|
+
46 5.25
|
|
58
|
+
47 5.125
|
|
59
|
+
48 5
|
|
60
|
+
49 4.875
|
|
61
|
+
50 4.75
|
|
62
|
+
51 4.625
|
|
63
|
+
52 4.5
|
|
64
|
+
53 4.375
|
|
65
|
+
54 4.25
|
|
66
|
+
55 4.125
|
|
67
|
+
56 4
|
|
68
|
+
57 3.875
|
|
69
|
+
58 3.75
|
|
70
|
+
59 3.625
|
|
71
|
+
60 3.5
|
|
72
|
+
61 3.375
|
|
73
|
+
62 3.25
|
|
74
|
+
63 3.125
|
|
75
|
+
64 3
|
|
76
|
+
65 2.875
|
|
77
|
+
66 2.75
|
|
78
|
+
67 2.625
|
|
79
|
+
68 2.5
|
|
80
|
+
69 2.4375
|
|
81
|
+
70 2.375
|
|
82
|
+
71 2.3125
|
|
83
|
+
72 2.25
|
|
84
|
+
73 2.1875
|
|
85
|
+
74 2.125
|
|
86
|
+
75 2.0625
|
|
87
|
+
76 2
|
|
88
|
+
77 1.9375
|
|
89
|
+
78 1.875
|
|
90
|
+
79 1.8125
|
|
91
|
+
80 1.75
|
|
92
|
+
81 1.6875
|
|
93
|
+
82 1.625
|
|
94
|
+
83 1.5625
|
|
95
|
+
84 1.5
|
|
96
|
+
85 1.4375
|
|
97
|
+
86 1.375
|
|
98
|
+
87 1.3125
|
|
99
|
+
88 1.25
|
|
100
|
+
89 1.1875
|
|
101
|
+
90 1.125
|
|
102
|
+
91 1.0625
|
|
103
|
+
92 1
|
|
104
|
+
93 0.95
|
|
105
|
+
94 0.9
|
|
106
|
+
95 0.85
|
|
107
|
+
96 0.8
|
|
108
|
+
97 0.75
|
|
109
|
+
98 0.7
|
|
110
|
+
99 0.65
|
|
111
|
+
100 0.6
|
|
112
|
+
101 0.55
|
|
113
|
+
102 0.5
|
|
114
|
+
103 0.475
|
|
115
|
+
104 0.45
|
|
116
|
+
105 0.425
|
|
117
|
+
106 0.4
|
|
118
|
+
107 0.375
|
|
119
|
+
108 0.35
|
|
120
|
+
109 0.325
|
|
121
|
+
110 0.3
|
|
122
|
+
111 0.275
|
|
123
|
+
112 0.25
|
|
124
|
+
113 0.225
|
|
125
|
+
114 0.2
|
|
126
|
+
115 0.18
|
|
127
|
+
116 0.16
|
|
128
|
+
117 0.14
|
|
129
|
+
118 0.12
|
|
130
|
+
119 0.1
|
|
131
|
+
120 0.08
|
|
132
|
+
121 0.06
|
|
133
|
+
122 0.05
|
|
134
|
+
123 0.04
|
|
135
|
+
124 0.03
|
|
136
|
+
125 0.02
|
|
137
|
+
126 0.01
|
|
138
|
+
127 0
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
# COLIBRE-L200m6 snapshot -> output redshift.
|
|
2
|
+
# Full 128-snapshot table. Re-verified 2026-08-20 (imf project, ROADMAP.md
|
|
3
|
+
# item 8.4) to be byte-identical to COLIBRE-L100m6.txt's iz/z mapping,
|
|
4
|
+
# confirmed directly against the authoritative outputTimes/{snapshotNumber,
|
|
5
|
+
# redshift} datasets in both
|
|
6
|
+
# /cosma8/data/dp004/jch/COLIBRE/HBT/L100_m6/DMO/trees/tree_127.0.hdf5 and
|
|
7
|
+
# /cosma8/data/dp004/jch/COLIBRE/HBT/L200_m6/DMO/trees/tree_127.0.hdf5
|
|
8
|
+
# (np.allclose on redshift, exact match on snapshotNumber, both length 128).
|
|
9
|
+
# Content is therefore an exact copy of COLIBRE-L100m6.txt, kept as its own
|
|
10
|
+
# file (rather than reusing that filename from this simulation's config
|
|
11
|
+
# entry) for naming clarity and so the two boxes' redshift lists can
|
|
12
|
+
# diverge safely in future if COLIBRE-L200m6's trees are ever rebuilt with
|
|
13
|
+
# different output times.
|
|
14
|
+
# iz z
|
|
15
|
+
0 30
|
|
16
|
+
1 25
|
|
17
|
+
2 22.5
|
|
18
|
+
3 20
|
|
19
|
+
4 19
|
|
20
|
+
5 18
|
|
21
|
+
6 17
|
|
22
|
+
7 16
|
|
23
|
+
8 15
|
|
24
|
+
9 14.5
|
|
25
|
+
10 14
|
|
26
|
+
11 13.5
|
|
27
|
+
12 13
|
|
28
|
+
13 12.5
|
|
29
|
+
14 12
|
|
30
|
+
15 11.5
|
|
31
|
+
16 11
|
|
32
|
+
17 10.5
|
|
33
|
+
18 10
|
|
34
|
+
19 9.75
|
|
35
|
+
20 9.5
|
|
36
|
+
21 9.25
|
|
37
|
+
22 9
|
|
38
|
+
23 8.75
|
|
39
|
+
24 8.5
|
|
40
|
+
25 8.25
|
|
41
|
+
26 8
|
|
42
|
+
27 7.75
|
|
43
|
+
28 7.5
|
|
44
|
+
29 7.375
|
|
45
|
+
30 7.25
|
|
46
|
+
31 7.125
|
|
47
|
+
32 7
|
|
48
|
+
33 6.875
|
|
49
|
+
34 6.75
|
|
50
|
+
35 6.625
|
|
51
|
+
36 6.5
|
|
52
|
+
37 6.375
|
|
53
|
+
38 6.25
|
|
54
|
+
39 6.125
|
|
55
|
+
40 6
|
|
56
|
+
41 5.875
|
|
57
|
+
42 5.75
|
|
58
|
+
43 5.625
|
|
59
|
+
44 5.5
|
|
60
|
+
45 5.375
|
|
61
|
+
46 5.25
|
|
62
|
+
47 5.125
|
|
63
|
+
48 5
|
|
64
|
+
49 4.875
|
|
65
|
+
50 4.75
|
|
66
|
+
51 4.625
|
|
67
|
+
52 4.5
|
|
68
|
+
53 4.375
|
|
69
|
+
54 4.25
|
|
70
|
+
55 4.125
|
|
71
|
+
56 4
|
|
72
|
+
57 3.875
|
|
73
|
+
58 3.75
|
|
74
|
+
59 3.625
|
|
75
|
+
60 3.5
|
|
76
|
+
61 3.375
|
|
77
|
+
62 3.25
|
|
78
|
+
63 3.125
|
|
79
|
+
64 3
|
|
80
|
+
65 2.875
|
|
81
|
+
66 2.75
|
|
82
|
+
67 2.625
|
|
83
|
+
68 2.5
|
|
84
|
+
69 2.4375
|
|
85
|
+
70 2.375
|
|
86
|
+
71 2.3125
|
|
87
|
+
72 2.25
|
|
88
|
+
73 2.1875
|
|
89
|
+
74 2.125
|
|
90
|
+
75 2.0625
|
|
91
|
+
76 2
|
|
92
|
+
77 1.9375
|
|
93
|
+
78 1.875
|
|
94
|
+
79 1.8125
|
|
95
|
+
80 1.75
|
|
96
|
+
81 1.6875
|
|
97
|
+
82 1.625
|
|
98
|
+
83 1.5625
|
|
99
|
+
84 1.5
|
|
100
|
+
85 1.4375
|
|
101
|
+
86 1.375
|
|
102
|
+
87 1.3125
|
|
103
|
+
88 1.25
|
|
104
|
+
89 1.1875
|
|
105
|
+
90 1.125
|
|
106
|
+
91 1.0625
|
|
107
|
+
92 1
|
|
108
|
+
93 0.95
|
|
109
|
+
94 0.9
|
|
110
|
+
95 0.85
|
|
111
|
+
96 0.8
|
|
112
|
+
97 0.75
|
|
113
|
+
98 0.7
|
|
114
|
+
99 0.65
|
|
115
|
+
100 0.6
|
|
116
|
+
101 0.55
|
|
117
|
+
102 0.5
|
|
118
|
+
103 0.475
|
|
119
|
+
104 0.45
|
|
120
|
+
105 0.425
|
|
121
|
+
106 0.4
|
|
122
|
+
107 0.375
|
|
123
|
+
108 0.35
|
|
124
|
+
109 0.325
|
|
125
|
+
110 0.3
|
|
126
|
+
111 0.275
|
|
127
|
+
112 0.25
|
|
128
|
+
113 0.225
|
|
129
|
+
114 0.2
|
|
130
|
+
115 0.18
|
|
131
|
+
116 0.16
|
|
132
|
+
117 0.14
|
|
133
|
+
118 0.12
|
|
134
|
+
119 0.1
|
|
135
|
+
120 0.08
|
|
136
|
+
121 0.06
|
|
137
|
+
122 0.05
|
|
138
|
+
123 0.04
|
|
139
|
+
124 0.03
|
|
140
|
+
125 0.02
|
|
141
|
+
126 0.01
|
|
142
|
+
127 0
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
{
|
|
2
|
+
"COLIBRE-L100m6": {
|
|
3
|
+
"aquarius_particle_file": "/cosma8/data/dp004/jch/COLIBRE/HBT/L100_m6/DMO/trees/particle_lists/particle_list_127",
|
|
4
|
+
"aquarius_tree_file": "/cosma8/data/dp004/jch/COLIBRE/HBT/L100_m6/DMO/trees/tree_127",
|
|
5
|
+
"h0": 0.681,
|
|
6
|
+
"iz0": 127,
|
|
7
|
+
"iz_list": [
|
|
8
|
+
127,
|
|
9
|
+
102,
|
|
10
|
+
92
|
|
11
|
+
],
|
|
12
|
+
"lambda0": 0.693922,
|
|
13
|
+
"lbox": 68.1,
|
|
14
|
+
"mpart": 1569616.5,
|
|
15
|
+
"nbody_trees_dir": "/cosma8/data/dp004/jch/COLIBRE/HBT/L100_m6/DMO/trees",
|
|
16
|
+
"nvol_range": "1-64",
|
|
17
|
+
"omega0": 0.306078,
|
|
18
|
+
"omegab": 0.0486,
|
|
19
|
+
"pk_file": "Power_Spec/pk_FLAMINGO_DMO_FIDUCIAL.dat",
|
|
20
|
+
"sigma8": 0.807,
|
|
21
|
+
"snapshot_file": "COLIBRE-L100m6.txt",
|
|
22
|
+
"volume": 4934.706890625
|
|
23
|
+
},
|
|
24
|
+
"COLIBRE-L200m6": {
|
|
25
|
+
"_note": "Re-added 2026-08-20 (imf project, ROADMAP.md/achievements.md item 8.4) after this entry was found missing entirely from colibre_family.json -- git history shows no commit ever added it, so the entry used by the real 2026-08-08 21-run L200m6 redshift-ladder campaign must have existed only as an uncommitted local edit that was later lost. Re-derived from ground truth, not guessed: aquarius_tree_file/particle_file paths confirmed to exist on disk with the same naming convention as COLIBRE-L100m6 (256 tree/particle-list files each, vs L100m6's 64); lbox=136.2 and volume=9869.413781249998 independently re-derived (200*h0 for lbox; lbox**3/nvol_range for volume) and cross-checked byte-for-byte against a real completed job's own substituted .input.temp file at /cosma/apps/durham/dc-hick2/galform/params/COLIBRE-L200m6_lc16_iz127_ivol8.input.temp, which literally has 'volume = 9869.413781249998'. mpart is still an inference, not a measurement (both L100m6 and L200m6 trees store simulation.particleMass=-1.0, verified directly from tree_127.0.hdf5's own attrs) -- set equal to L100m6's on the basis both share the 'm6' resolution-tier label, same as before this entry was lost; see achievements.md's 'L200m6 config investigation' section and ROADMAP.md item 1 for why this is still provisional pending COLIBRE-team confirmation. snapshot_file COLIBRE-L200m6.txt is a fresh copy of COLIBRE-L100m6.txt's 128-row table, re-verified 2026-08-20 byte-identical against both trees' own outputTimes/{snapshotNumber,redshift} HDF5 datasets (not reused from the stale, unverified 2026-07-28 claim -- independently re-checked this pass).",
|
|
26
|
+
"aquarius_particle_file": "/cosma8/data/dp004/jch/COLIBRE/HBT/L200_m6/DMO/trees/particle_lists/particle_list_127",
|
|
27
|
+
"aquarius_tree_file": "/cosma8/data/dp004/jch/COLIBRE/HBT/L200_m6/DMO/trees/tree_127",
|
|
28
|
+
"h0": 0.681,
|
|
29
|
+
"iz0": 127,
|
|
30
|
+
"iz_list": [
|
|
31
|
+
127,
|
|
32
|
+
102,
|
|
33
|
+
92
|
|
34
|
+
],
|
|
35
|
+
"lambda0": 0.693922,
|
|
36
|
+
"lbox": 136.2,
|
|
37
|
+
"mpart": 1569616.5,
|
|
38
|
+
"nbody_trees_dir": "/cosma8/data/dp004/jch/COLIBRE/HBT/L200_m6/DMO/trees",
|
|
39
|
+
"nvol_range": "1-256",
|
|
40
|
+
"omega0": 0.306078,
|
|
41
|
+
"omegab": 0.0486,
|
|
42
|
+
"pk_file": "Power_Spec/pk_FLAMINGO_DMO_FIDUCIAL.dat",
|
|
43
|
+
"sigma8": 0.807,
|
|
44
|
+
"snapshot_file": "COLIBRE-L200m6.txt",
|
|
45
|
+
"volume": 9869.413781249998
|
|
46
|
+
},
|
|
47
|
+
"COLIBRE-L400m7": {
|
|
48
|
+
"_note": "GALFORM-format aquarius trees not yet built from COLIBRE DMO. dhalo trees at /cosma7/data/dp004/jch/COLIBRE/L400_m7/DMO/dhalo_trees/ (not yet in GALFORM aquarius format). Update tree paths and iz0/iz_list/nvol_range before submitting.",
|
|
49
|
+
"aquarius_particle_file": "/cosma5/data/jch/Galform/Merger_Trees/COLIBRE/L400_m7/particle_lists/particle_list_TBD",
|
|
50
|
+
"aquarius_tree_file": "/cosma5/data/jch/Galform/Merger_Trees/COLIBRE/L400_m7/treedir_TBD/tree_TBD",
|
|
51
|
+
"h0": 0.681,
|
|
52
|
+
"iz0": null,
|
|
53
|
+
"iz_list": [],
|
|
54
|
+
"lambda0": 0.693922,
|
|
55
|
+
"lbox": 272.4,
|
|
56
|
+
"mpart": 12556932.0,
|
|
57
|
+
"nbody_trees_dir": "/cosma5/data/jch/Galform/Merger_Trees/COLIBRE/L400_m7",
|
|
58
|
+
"nvol_range": "1-64",
|
|
59
|
+
"omega0": 0.306078,
|
|
60
|
+
"omegab": 0.0486,
|
|
61
|
+
"pk_file": "Power_Spec/pk_COLIBRE_norm.dat",
|
|
62
|
+
"sigma8": 0.807,
|
|
63
|
+
"snapshot_file": "/cosma5/data/jch/Galform/Merger_Trees/COLIBRE/L400_m7/redshift_list.txt",
|
|
64
|
+
"volume": null
|
|
65
|
+
}
|
|
66
|
+
}
|
|
@@ -9,7 +9,7 @@ import sys
|
|
|
9
9
|
import time
|
|
10
10
|
from dataclasses import dataclass, field, fields
|
|
11
11
|
from pathlib import Path
|
|
12
|
-
from typing import Dict, List, Optional, Tuple
|
|
12
|
+
from typing import Dict, List, Optional, Sequence, Tuple
|
|
13
13
|
|
|
14
14
|
|
|
15
15
|
@dataclass
|
|
@@ -244,6 +244,20 @@ def _resolve_log_path(explicit: Optional[str], output_folder_name: str) -> Path:
|
|
|
244
244
|
return _default_cosma_user_root() / output_folder_name / "logs"
|
|
245
245
|
|
|
246
246
|
|
|
247
|
+
def _validate_ivols(ivols: Sequence[int], sim_config) -> List[int]:
|
|
248
|
+
"""Check an explicit ivol list: non-empty, unique, integer, in [0, k)."""
|
|
249
|
+
out = [int(i) for i in ivols]
|
|
250
|
+
if not out:
|
|
251
|
+
raise ValueError("ivols must not be empty")
|
|
252
|
+
if len(set(out)) != len(out):
|
|
253
|
+
raise ValueError("ivols must be unique")
|
|
254
|
+
k = _parse_nvol_range(sim_config.nvol_range)[1] if sim_config is not None else None
|
|
255
|
+
bad = [i for i in out if i < 0 or (k is not None and i >= k)]
|
|
256
|
+
if bad:
|
|
257
|
+
raise ValueError(f"ivols out of range [0, {k}): {bad[:5]}")
|
|
258
|
+
return out
|
|
259
|
+
|
|
260
|
+
|
|
247
261
|
def _parse_nvol_range(nvol_range: str) -> Tuple[int, int]:
|
|
248
262
|
"""Parse a legacy nvol range string (e.g. ``'12'`` or ``'1001-1024'``)."""
|
|
249
263
|
raw = str(nvol_range).strip()
|
|
@@ -303,7 +317,14 @@ class GalformSubmitter:
|
|
|
303
317
|
submit_retry_backoff: float = 2.0,
|
|
304
318
|
mail_user: Optional[str] = None,
|
|
305
319
|
mail_type: str = "END,FAIL",
|
|
320
|
+
ivols: Optional[Sequence[int]] = None,
|
|
306
321
|
):
|
|
322
|
+
"""Configure a submitter.
|
|
323
|
+
|
|
324
|
+
``ivols``: optional explicit list of 0-based subvolume indices (the ``ivol<N>``
|
|
325
|
+
output directories), for non-contiguous selections such as a random m-of-k draw.
|
|
326
|
+
Mutually exclusive with ``nvol``/``nvol_range``; task id t runs ``ivols[t-1]``.
|
|
327
|
+
"""
|
|
307
328
|
self.galform_dir = Path(galform_dir)
|
|
308
329
|
self.nbody_sim = nbody_sim
|
|
309
330
|
self.model = model
|
|
@@ -378,7 +399,7 @@ class GalformSubmitter:
|
|
|
378
399
|
if nvol is not None and nvol_range is not None:
|
|
379
400
|
raise ValueError("Specify only one of nvol and nvol_range")
|
|
380
401
|
resolved_nvol_range = nvol if nvol is not None else nvol_range
|
|
381
|
-
if iz_list is None or resolved_nvol_range is None:
|
|
402
|
+
if iz_list is None or (resolved_nvol_range is None and ivols is None):
|
|
382
403
|
raise ValueError(
|
|
383
404
|
f"Unknown simulation '{nbody_sim}'. "
|
|
384
405
|
"Provide iz_list and nvol explicitly."
|
|
@@ -396,8 +417,19 @@ class GalformSubmitter:
|
|
|
396
417
|
if self.iz is not None:
|
|
397
418
|
self.iz_list = [self.iz]
|
|
398
419
|
|
|
399
|
-
|
|
400
|
-
|
|
420
|
+
if ivols is not None:
|
|
421
|
+
if nvol is not None or nvol_range is not None:
|
|
422
|
+
raise ValueError("Specify only one of ivols and nvol/nvol_range")
|
|
423
|
+
self.ivols = _validate_ivols(ivols, self.sim_config)
|
|
424
|
+
self.nvol_range = (
|
|
425
|
+
f"{min(self.ivols) + 1}-{max(self.ivols) + 1}" # informational
|
|
426
|
+
)
|
|
427
|
+
self.nvol_start, self.nvol_end = 1, len(self.ivols)
|
|
428
|
+
self.nvol_count = len(self.ivols)
|
|
429
|
+
else:
|
|
430
|
+
self.ivols = None
|
|
431
|
+
self.nvol_start, self.nvol_end = _parse_nvol_range(self.nvol_range)
|
|
432
|
+
self.nvol_count = self.nvol_end - self.nvol_start + 1
|
|
401
433
|
|
|
402
434
|
# Validate
|
|
403
435
|
if not self.galform_dir.is_dir():
|
|
@@ -497,7 +529,18 @@ class GalformSubmitter:
|
|
|
497
529
|
lines = [
|
|
498
530
|
"# ---- model parameter file setup ----",
|
|
499
531
|
f"set base_inputs_file = {mc.base_inputs_file}",
|
|
500
|
-
|
|
532
|
+
# The job id MUST stay in this path. Without it, any two jobs
|
|
533
|
+
# sharing (Nbody_sim, model, iz, ivol) that run concurrently write
|
|
534
|
+
# the same file: each begins with `cp $base_inputs_file ...`, which
|
|
535
|
+
# wipes the other's substitutions, so GALFORM can read a parameter
|
|
536
|
+
# set belonging to a different job. This is not hypothetical -- it
|
|
537
|
+
# silently mixed x_imf branches across most of the 2026 redshift-
|
|
538
|
+
# ladder and counter-ladder campaigns, whose three branches per
|
|
539
|
+
# rung collide on exactly this key. Affected runs are identifiable
|
|
540
|
+
# after the fact only from each galaxies.hdf5's /Parameters group.
|
|
541
|
+
"set galform_inputs_file = "
|
|
542
|
+
"./params/${Nbody_sim}_${model}_iz${iz}_ivol${ivol}"
|
|
543
|
+
"_job${SLURM_JOB_ID}.input.temp",
|
|
501
544
|
"\\mkdir -p ./params",
|
|
502
545
|
"cp $base_inputs_file $galform_inputs_file",
|
|
503
546
|
]
|
|
@@ -765,6 +808,15 @@ rm -f $galform_inputs_file
|
|
|
765
808
|
exit
|
|
766
809
|
"""
|
|
767
810
|
|
|
811
|
+
def _ivol_assignment(self) -> str:
|
|
812
|
+
"""tcsh lines mapping the 1-based task id to a 0-based ivol."""
|
|
813
|
+
if self.ivols is None:
|
|
814
|
+
return f"@ ivol = $slurm_task_id + {self.nvol_start} - 2"
|
|
815
|
+
listed = " ".join(str(i) for i in self.ivols)
|
|
816
|
+
return (
|
|
817
|
+
f"set ivol_list = ( {listed} )\n@ ivol = $ivol_list[$slurm_task_id]"
|
|
818
|
+
)
|
|
819
|
+
|
|
768
820
|
def _create_tcsh_script(self, iz: int) -> str:
|
|
769
821
|
"""Generate the inner tcsh GALFORM script for snapshot iz.
|
|
770
822
|
|
|
@@ -834,7 +886,7 @@ set model = {self.model}
|
|
|
834
886
|
set Nbody_sim = {self.nbody_sim}
|
|
835
887
|
set iz = {iz}
|
|
836
888
|
@ slurm_task_id = ${{SLURM_ARRAY_TASK_ID}}
|
|
837
|
-
|
|
889
|
+
{self._ivol_assignment()}
|
|
838
890
|
|
|
839
891
|
# Change to GALFORM source directory (scripts use relative paths)
|
|
840
892
|
cd {self.galform_dir}
|
|
@@ -14,6 +14,7 @@ galform_execution/config/models.json
|
|
|
14
14
|
galform_execution/config/partition_configs.json
|
|
15
15
|
galform_execution/config/run_flags.json
|
|
16
16
|
galform_execution/config/redshift_lists/COLIBRE-L100m6.txt
|
|
17
|
+
galform_execution/config/redshift_lists/COLIBRE-L200m6.txt
|
|
17
18
|
galform_execution/config/redshift_lists/FLAMINGO-L1000N1800.txt
|
|
18
19
|
galform_execution/config/redshift_lists/L800.txt
|
|
19
20
|
galform_execution/config/redshift_lists/Mill1.txt
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "galform_execution"
|
|
3
|
-
version = "0.2.
|
|
3
|
+
version = "0.2.4"
|
|
4
4
|
description = "GALFORM execution helper utilities"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.8"
|
|
@@ -32,3 +32,11 @@ galform_execution = ["config/**/*.json", "config/**/*.txt"]
|
|
|
32
32
|
[tool.isort]
|
|
33
33
|
profile = "black"
|
|
34
34
|
|
|
35
|
+
[tool.ruff.lint]
|
|
36
|
+
# Pin the rule set explicitly. Without this, ruff's "no config" default
|
|
37
|
+
# selection changes between ruff releases (e.g. 0.15 -> 0.16 started
|
|
38
|
+
# enabling rules like FA100/BLE001), which broke CI even though nothing
|
|
39
|
+
# in this codebase changed. E4/E7/E9/F match ruff's long-standing
|
|
40
|
+
# original defaults (pyflakes + a handful of pycodestyle error checks).
|
|
41
|
+
select = ["E4", "E7", "E9", "F"]
|
|
42
|
+
|
|
@@ -878,3 +878,115 @@ def test_multi_output_respects_explicit_mgalmin_descendant_override():
|
|
|
878
878
|
"./replace_variable.csh $galform_inputs_file mgalmin_output_descendants .false."
|
|
879
879
|
in script
|
|
880
880
|
)
|
|
881
|
+
|
|
882
|
+
|
|
883
|
+
def test_params_file_path_is_job_unique():
|
|
884
|
+
"""The generated parameter file must not be shared between concurrent jobs.
|
|
885
|
+
|
|
886
|
+
Two jobs sharing (Nbody_sim, model, iz, ivol) each run
|
|
887
|
+
``cp $base_inputs_file $galform_inputs_file`` before substituting, so a
|
|
888
|
+
shared path lets one job wipe the other's substitutions and lets GALFORM
|
|
889
|
+
read a parameter set belonging to a different job. This silently mixed
|
|
890
|
+
x_imf branches across the 2026 redshift-ladder campaigns.
|
|
891
|
+
"""
|
|
892
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
893
|
+
gdir = _make_galform_dir(tmpdir)
|
|
894
|
+
|
|
895
|
+
submitter = GalformSubmitter(
|
|
896
|
+
galform_dir=gdir,
|
|
897
|
+
nbody_sim="L800",
|
|
898
|
+
model="gp14",
|
|
899
|
+
output_folder_name="Galform_Out_Test",
|
|
900
|
+
)
|
|
901
|
+
script_content = submitter._create_tcsh_script(iz=100)
|
|
902
|
+
|
|
903
|
+
line = next(
|
|
904
|
+
l
|
|
905
|
+
for l in script_content.splitlines()
|
|
906
|
+
if l.strip().startswith("set galform_inputs_file")
|
|
907
|
+
)
|
|
908
|
+
assert "${SLURM_JOB_ID}" in line, (
|
|
909
|
+
"parameter file path must be job-unique; got: " + line
|
|
910
|
+
)
|
|
911
|
+
|
|
912
|
+
|
|
913
|
+
def test_explicit_ivols_list_in_tcsh_script():
|
|
914
|
+
"""An explicit, non-contiguous ivol list is looked up by task id (tcsh arrays are 1-based)."""
|
|
915
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
916
|
+
gdir = _make_galform_dir(tmpdir)
|
|
917
|
+
|
|
918
|
+
submitter = GalformSubmitter(
|
|
919
|
+
galform_dir=gdir, nbody_sim="L800", model="gp14", iz=271, ivols=[5, 900, 17]
|
|
920
|
+
)
|
|
921
|
+
|
|
922
|
+
assert submitter.ivols == [5, 900, 17]
|
|
923
|
+
assert submitter.nvol_count == 3
|
|
924
|
+
script = submitter._create_tcsh_script(iz=271)
|
|
925
|
+
assert "set ivol_list = ( 5 900 17 )" in script
|
|
926
|
+
assert "@ ivol = $ivol_list[$slurm_task_id]" in script
|
|
927
|
+
assert "$slurm_task_id + " not in script
|
|
928
|
+
|
|
929
|
+
|
|
930
|
+
def test_explicit_ivols_sizes_job_wrapper():
|
|
931
|
+
"""The bash wrapper covers exactly len(ivols) task ids."""
|
|
932
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
933
|
+
gdir = _make_galform_dir(tmpdir)
|
|
934
|
+
ivols = list(range(0, 1024, 16)) # 64 ivols
|
|
935
|
+
|
|
936
|
+
submitter = GalformSubmitter(
|
|
937
|
+
galform_dir=gdir,
|
|
938
|
+
nbody_sim="L800",
|
|
939
|
+
model="gp14",
|
|
940
|
+
iz=271,
|
|
941
|
+
ivols=ivols,
|
|
942
|
+
log_path=str(Path(tmpdir) / "logs"),
|
|
943
|
+
partition="cosma8-shm",
|
|
944
|
+
)
|
|
945
|
+
|
|
946
|
+
script = submitter.create_job_script(iz=271, tcsh_path="/x.csh")
|
|
947
|
+
assert "#SBATCH --cpus-per-task=64" in script
|
|
948
|
+
assert "[ $task_id -le 64 ]" in script
|
|
949
|
+
|
|
950
|
+
|
|
951
|
+
def test_explicit_ivols_validation():
|
|
952
|
+
"""ivols must be unique, in range, non-empty and exclusive with nvol/nvol_range."""
|
|
953
|
+
import pytest
|
|
954
|
+
|
|
955
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
956
|
+
gdir = _make_galform_dir(tmpdir)
|
|
957
|
+
kw = dict(galform_dir=gdir, nbody_sim="L800", model="gp14", iz=271)
|
|
958
|
+
for bad in ([], [3, 3], [-1], [1024]):
|
|
959
|
+
with pytest.raises(ValueError):
|
|
960
|
+
GalformSubmitter(**kw, ivols=bad)
|
|
961
|
+
with pytest.raises(ValueError):
|
|
962
|
+
GalformSubmitter(**kw, ivols=[1, 2], nvol="1-2")
|
|
963
|
+
with pytest.raises(ValueError):
|
|
964
|
+
GalformSubmitter(**kw, ivols=[1, 2], nvol_range="1-2")
|
|
965
|
+
|
|
966
|
+
|
|
967
|
+
def test_contiguous_nvol_unchanged_without_ivols():
|
|
968
|
+
"""Without ivols the legacy contiguous mapping is untouched."""
|
|
969
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
970
|
+
gdir = _make_galform_dir(tmpdir)
|
|
971
|
+
submitter = GalformSubmitter(
|
|
972
|
+
galform_dir=gdir, nbody_sim="L800", model="gp14", iz=271, nvol="1-64"
|
|
973
|
+
)
|
|
974
|
+
assert submitter.ivols is None
|
|
975
|
+
script = submitter._create_tcsh_script(iz=271)
|
|
976
|
+
assert "@ ivol = $slurm_task_id + 1 - 2" in script
|
|
977
|
+
assert "ivol_list" not in script
|
|
978
|
+
|
|
979
|
+
|
|
980
|
+
def test_explicit_ivols_with_unknown_sim():
|
|
981
|
+
"""ivols stands in for nvol when the simulation is not in SIMULATION_CONFIGS."""
|
|
982
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
983
|
+
gdir = _make_galform_dir(tmpdir)
|
|
984
|
+
submitter = GalformSubmitter(
|
|
985
|
+
galform_dir=gdir,
|
|
986
|
+
nbody_sim="MyCustomSim",
|
|
987
|
+
model="gp14",
|
|
988
|
+
iz_list=[100],
|
|
989
|
+
ivols=[7, 2],
|
|
990
|
+
)
|
|
991
|
+
assert submitter.nvol_count == 2
|
|
992
|
+
assert submitter.nvol_range == "3-8"
|
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"COLIBRE-L100m6": {
|
|
3
|
-
"aquarius_particle_file": "/cosma8/data/dp004/jch/COLIBRE/HBT/L100_m6/DMO/trees/particle_lists/particle_list_127",
|
|
4
|
-
"aquarius_tree_file": "/cosma8/data/dp004/jch/COLIBRE/HBT/L100_m6/DMO/trees/tree_127",
|
|
5
|
-
"h0": 0.681,
|
|
6
|
-
"iz0": 127,
|
|
7
|
-
"iz_list": [
|
|
8
|
-
127,
|
|
9
|
-
102,
|
|
10
|
-
92
|
|
11
|
-
],
|
|
12
|
-
"lambda0": 0.693922,
|
|
13
|
-
"lbox": 68.1,
|
|
14
|
-
"mpart": 1569616.5,
|
|
15
|
-
"nbody_trees_dir": "/cosma8/data/dp004/jch/COLIBRE/HBT/L100_m6/DMO/trees",
|
|
16
|
-
"nvol_range": "1-64",
|
|
17
|
-
"omega0": 0.306078,
|
|
18
|
-
"omegab": 0.0486,
|
|
19
|
-
"pk_file": "Power_Spec/pk_FLAMINGO_DMO_FIDUCIAL.dat",
|
|
20
|
-
"sigma8": 0.807,
|
|
21
|
-
"snapshot_file": "COLIBRE-L100m6.txt",
|
|
22
|
-
"volume": 4934.706890625
|
|
23
|
-
},
|
|
24
|
-
"COLIBRE-L400m7": {
|
|
25
|
-
"_note": "GALFORM-format aquarius trees not yet built from COLIBRE DMO. dhalo trees at /cosma7/data/dp004/jch/COLIBRE/L400_m7/DMO/dhalo_trees/ (not yet in GALFORM aquarius format). Update tree paths and iz0/iz_list/nvol_range before submitting.",
|
|
26
|
-
"aquarius_particle_file": "/cosma5/data/jch/Galform/Merger_Trees/COLIBRE/L400_m7/particle_lists/particle_list_TBD",
|
|
27
|
-
"aquarius_tree_file": "/cosma5/data/jch/Galform/Merger_Trees/COLIBRE/L400_m7/treedir_TBD/tree_TBD",
|
|
28
|
-
"h0": 0.681,
|
|
29
|
-
"iz0": null,
|
|
30
|
-
"iz_list": [],
|
|
31
|
-
"lambda0": 0.693922,
|
|
32
|
-
"lbox": 272.4,
|
|
33
|
-
"mpart": 12556932.0,
|
|
34
|
-
"nbody_trees_dir": "/cosma5/data/jch/Galform/Merger_Trees/COLIBRE/L400_m7",
|
|
35
|
-
"nvol_range": "1-64",
|
|
36
|
-
"omega0": 0.306078,
|
|
37
|
-
"omegab": 0.0486,
|
|
38
|
-
"pk_file": "Power_Spec/pk_COLIBRE_norm.dat",
|
|
39
|
-
"sigma8": 0.807,
|
|
40
|
-
"snapshot_file": "/cosma5/data/jch/Galform/Merger_Trees/COLIBRE/L400_m7/redshift_list.txt",
|
|
41
|
-
"volume": null
|
|
42
|
-
}
|
|
43
|
-
}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{galform_execution-0.2.2 → galform_execution-0.2.4}/galform_execution/config/dust_params.json
RENAMED
|
File without changes
|
|
File without changes
|
{galform_execution-0.2.2 → galform_execution-0.2.4}/galform_execution/config/partition_configs.json
RENAMED
|
File without changes
|
|
File without changes
|
{galform_execution-0.2.2 → galform_execution-0.2.4}/galform_execution/config/redshift_lists/L800.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{galform_execution-0.2.2 → galform_execution-0.2.4}/galform_execution.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
{galform_execution-0.2.2 → galform_execution-0.2.4}/galform_execution.egg-info/entry_points.txt
RENAMED
|
File without changes
|
{galform_execution-0.2.2 → galform_execution-0.2.4}/galform_execution.egg-info/top_level.txt
RENAMED
|
File without changes
|
|
File without changes
|