pspipe 0.13__tar.gz → 0.15__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pspipe-0.13 → pspipe-0.15}/PKG-INFO +1 -1
- {pspipe-0.13 → pspipe-0.15}/pspipe/tasks.py +29 -9
- {pspipe-0.13 → pspipe-0.15}/pspipe/tools/pspipe.py +24 -0
- {pspipe-0.13 → pspipe-0.15}/pyproject.toml +1 -1
- {pspipe-0.13 → pspipe-0.15}/LICENSE +0 -0
- {pspipe-0.13 → pspipe-0.15}/README.md +0 -0
- {pspipe-0.13 → pspipe-0.15}/pspipe/__init__.py +0 -0
- {pspipe-0.13 → pspipe-0.15}/pspipe/database.py +0 -0
- {pspipe-0.13 → pspipe-0.15}/pspipe/obsdata.py +0 -0
- {pspipe-0.13 → pspipe-0.15}/pspipe/settings.py +0 -0
- {pspipe-0.13 → pspipe-0.15}/pspipe/templates/config/eor_bins_a12.parset +0 -0
- {pspipe-0.13 → pspipe-0.15}/pspipe/templates/config/eor_bins_alo.parset +0 -0
- {pspipe-0.13 → pspipe-0.15}/pspipe/templates/config/eor_bins_hba.parset +0 -0
- {pspipe-0.13 → pspipe-0.15}/pspipe/templates/config/eor_bins_nenufar.parset +0 -0
- {pspipe-0.13 → pspipe-0.15}/pspipe/templates/config/flagger.parset +0 -0
- {pspipe-0.13 → pspipe-0.15}/pspipe/templates/config/flagger_pre_combine.parset +0 -0
- {pspipe-0.13 → pspipe-0.15}/pspipe/templates/config/gpr_config_a12.parset +0 -0
- {pspipe-0.13 → pspipe-0.15}/pspipe/templates/config/gpr_config_hba.parset +0 -0
- {pspipe-0.13 → pspipe-0.15}/pspipe/templates/config/gpr_config_nenufar.parset +0 -0
- {pspipe-0.13 → pspipe-0.15}/pspipe/templates/config/gpr_config_v.parset +0 -0
- {pspipe-0.13 → pspipe-0.15}/pspipe/templates/config/ps_config_a12.parset +0 -0
- {pspipe-0.13 → pspipe-0.15}/pspipe/templates/config/ps_config_alo.parset +0 -0
- {pspipe-0.13 → pspipe-0.15}/pspipe/templates/config/ps_config_hba.parset +0 -0
- {pspipe-0.13 → pspipe-0.15}/pspipe/templates/config/ps_config_nenufar.parset +0 -0
- {pspipe-0.13 → pspipe-0.15}/pspipe/templates/default_a12.toml +0 -0
- {pspipe-0.13 → pspipe-0.15}/pspipe/templates/default_alo.toml +0 -0
- {pspipe-0.13 → pspipe-0.15}/pspipe/templates/default_hba.toml +0 -0
- {pspipe-0.13 → pspipe-0.15}/pspipe/templates/default_nenufar.toml +0 -0
- {pspipe-0.13 → pspipe-0.15}/pspipe/templates/default_settings.toml +0 -0
- {pspipe-0.13 → pspipe-0.15}/pspipe/tools/__init__.py +0 -0
- {pspipe-0.13 → pspipe-0.15}/pspipe/tools/psdb.py +0 -0
- {pspipe-0.13 → pspipe-0.15}/pspipe/utils.py +0 -0
|
@@ -84,6 +84,8 @@ class AbstractTask:
|
|
|
84
84
|
raise ValueError('Invalid worker settings.')
|
|
85
85
|
|
|
86
86
|
self.worker = self._worker_pool(name)
|
|
87
|
+
# Tasks of the pools already run by this task (see init_for_new_task)
|
|
88
|
+
self.past_tasks = []
|
|
87
89
|
|
|
88
90
|
def _worker_pool(self, name):
|
|
89
91
|
s_worker = self.settings.worker
|
|
@@ -96,6 +98,7 @@ class AbstractTask:
|
|
|
96
98
|
resources=settings.worker_resources(s_worker))
|
|
97
99
|
|
|
98
100
|
def init_for_new_task(self, post_task_name):
|
|
101
|
+
self.past_tasks += self.worker.tasks
|
|
99
102
|
self.worker = self._worker_pool(post_task_name)
|
|
100
103
|
|
|
101
104
|
def add_sub_task(self, sub_task, run_on_host=None):
|
|
@@ -120,7 +123,7 @@ class AbstractTask:
|
|
|
120
123
|
def execute(self):
|
|
121
124
|
self.worker.execute()
|
|
122
125
|
|
|
123
|
-
return self.
|
|
126
|
+
return self.get_pool_report().ok
|
|
124
127
|
|
|
125
128
|
def execute_single_local(self, cmd, log_file):
|
|
126
129
|
w = worker.WorkerPool([worker.localhost_shortname], name=self.name,
|
|
@@ -131,6 +134,11 @@ class AbstractTask:
|
|
|
131
134
|
w.execute()
|
|
132
135
|
|
|
133
136
|
def get_report(self):
|
|
137
|
+
'''Report of all the pools of the task.'''
|
|
138
|
+
return worker.WorkerPoolReport(self.past_tasks + self.worker.tasks)
|
|
139
|
+
|
|
140
|
+
def get_pool_report(self):
|
|
141
|
+
'''Report of the current pool only.'''
|
|
134
142
|
return worker.WorkerPoolReport(self.worker.tasks)
|
|
135
143
|
|
|
136
144
|
|
|
@@ -191,10 +199,8 @@ class ImagingSubTask(AbstractSubTask):
|
|
|
191
199
|
odd=self.odd if odd is None else odd, img_name=img_name)
|
|
192
200
|
meta.save()
|
|
193
201
|
|
|
194
|
-
def
|
|
195
|
-
for
|
|
196
|
-
os.remove(file)
|
|
197
|
-
|
|
202
|
+
def fits_keywords(self):
|
|
203
|
+
'''Integration time, total time and channel width of the images, for their headers.'''
|
|
198
204
|
img = self.settings.image
|
|
199
205
|
|
|
200
206
|
if img.time_start_index != img.time_end_index:
|
|
@@ -204,6 +210,15 @@ class ImagingSubTask(AbstractSubTask):
|
|
|
204
210
|
else:
|
|
205
211
|
total_time = self.obs_meta.total_time
|
|
206
212
|
|
|
213
|
+
return {'PEINTTIM': int(self.obs_meta.int_time), 'PETOTTIM': int(total_time),
|
|
214
|
+
'PECHWIDT': int(self.obs_meta.chan_width)}
|
|
215
|
+
|
|
216
|
+
def done_callback(self):
|
|
217
|
+
for file in glob.glob(f'{self.get_fits_prefix()}*-MFS-*.fits'):
|
|
218
|
+
os.remove(file)
|
|
219
|
+
|
|
220
|
+
img = self.settings.image
|
|
221
|
+
|
|
207
222
|
if self.even_odd:
|
|
208
223
|
parts = [('-even', True, False), ('-odd', False, True)]
|
|
209
224
|
else:
|
|
@@ -220,10 +235,12 @@ class ImagingSubTask(AbstractSubTask):
|
|
|
220
235
|
files = glob.glob(f'{prefix}*.fits')
|
|
221
236
|
img_name = 'image'
|
|
222
237
|
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
238
|
+
# psimager writes the keywords with the files (-fits-keyword)
|
|
239
|
+
if img.imager != 'psimager':
|
|
240
|
+
keywords = self.fits_keywords()
|
|
241
|
+
for file in files:
|
|
242
|
+
with pf.open(file, mode='update') as hdul:
|
|
243
|
+
hdul[0].header.update(keywords)
|
|
227
244
|
|
|
228
245
|
if not img.lst_bins:
|
|
229
246
|
self.register_files(files, img_name=img_name, even=even, odd=odd)
|
|
@@ -278,6 +295,9 @@ class ImagingSubTask(AbstractSubTask):
|
|
|
278
295
|
|
|
279
296
|
args.append(f'-temp-dir {self.temp_dir}')
|
|
280
297
|
|
|
298
|
+
if img.imager == 'psimager':
|
|
299
|
+
args.append(' '.join(f'-fits-keyword {k}={v}' for k, v in self.fits_keywords().items()))
|
|
300
|
+
|
|
281
301
|
if self.even_odd:
|
|
282
302
|
args.append('-even-odd')
|
|
283
303
|
elif self.even:
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import sys
|
|
4
4
|
|
|
5
5
|
import click
|
|
6
|
+
from libpipe import worker
|
|
6
7
|
|
|
7
8
|
from pspipe import __version__, database, tasks
|
|
8
9
|
|
|
@@ -32,6 +33,24 @@ def print_report(report):
|
|
|
32
33
|
print("==========================\n")
|
|
33
34
|
|
|
34
35
|
|
|
36
|
+
def print_run_summary(reports, task_names):
|
|
37
|
+
"""Recap of the run: one line per task (the tasks not run after a failure included), the
|
|
38
|
+
failed tasks with their logs, and the usage of all the tasks."""
|
|
39
|
+
print("\n=== Summary of the run ===")
|
|
40
|
+
for name in task_names:
|
|
41
|
+
report = dict(reports).get(name)
|
|
42
|
+
if report is None:
|
|
43
|
+
print(f"{name:20s} not run")
|
|
44
|
+
continue
|
|
45
|
+
state = "OK" if report.ok else "FAILED"
|
|
46
|
+
print(f"{name:20s} {state:7s} {report.n_success_rc0} / {report.n_total} tasks successful")
|
|
47
|
+
for t in report.success_nonzero + report.failed:
|
|
48
|
+
log = f" | log: {t.output_file}" if getattr(t, "output_file", None) else ""
|
|
49
|
+
print(f" - {t.name} | host={report.host_of(t)} | rc={t.returncode}{log}")
|
|
50
|
+
worker.WorkerPoolReport([t for _, r in reports for t in r.all_tasks]).print_usage()
|
|
51
|
+
print("==========================\n")
|
|
52
|
+
|
|
53
|
+
|
|
35
54
|
@click.command(epilog='\b\nAvailable tasks:\n' + tasks_help)
|
|
36
55
|
@click.version_option(__version__)
|
|
37
56
|
@click.argument('tasks_str', metavar='TASKS')
|
|
@@ -62,13 +81,18 @@ def main(tasks_str, settings_file, obs_ids, config_modifier):
|
|
|
62
81
|
print(f'Error: No obs_ids matching {obs_ids} in database.')
|
|
63
82
|
sys.exit(1)
|
|
64
83
|
|
|
84
|
+
reports = []
|
|
65
85
|
for task_name in tasks_str.split(','):
|
|
66
86
|
t = all_tasks[task_name](s)
|
|
67
87
|
obs_ids_set = t.run(obs_ids_set)
|
|
68
88
|
report = t.get_report()
|
|
89
|
+
reports.append((task_name, report))
|
|
69
90
|
|
|
70
91
|
if not obs_ids_set:
|
|
71
92
|
print(f'Task {task_name} finished with errors')
|
|
72
93
|
print_report(report)
|
|
94
|
+
print_run_summary(reports, tasks_str.split(','))
|
|
73
95
|
sys.exit(1)
|
|
74
96
|
print_report(report)
|
|
97
|
+
|
|
98
|
+
print_run_summary(reports, tasks_str.split(','))
|
|
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "pspipe"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.15"
|
|
8
8
|
description = "Power spectra pipeline for Cosmic Dawn, Epoch of Reionization radio interferometric experiments"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "GPL-3.0-or-later"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|