pspipe 0.13__tar.gz → 0.15__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. {pspipe-0.13 → pspipe-0.15}/PKG-INFO +1 -1
  2. {pspipe-0.13 → pspipe-0.15}/pspipe/tasks.py +29 -9
  3. {pspipe-0.13 → pspipe-0.15}/pspipe/tools/pspipe.py +24 -0
  4. {pspipe-0.13 → pspipe-0.15}/pyproject.toml +1 -1
  5. {pspipe-0.13 → pspipe-0.15}/LICENSE +0 -0
  6. {pspipe-0.13 → pspipe-0.15}/README.md +0 -0
  7. {pspipe-0.13 → pspipe-0.15}/pspipe/__init__.py +0 -0
  8. {pspipe-0.13 → pspipe-0.15}/pspipe/database.py +0 -0
  9. {pspipe-0.13 → pspipe-0.15}/pspipe/obsdata.py +0 -0
  10. {pspipe-0.13 → pspipe-0.15}/pspipe/settings.py +0 -0
  11. {pspipe-0.13 → pspipe-0.15}/pspipe/templates/config/eor_bins_a12.parset +0 -0
  12. {pspipe-0.13 → pspipe-0.15}/pspipe/templates/config/eor_bins_alo.parset +0 -0
  13. {pspipe-0.13 → pspipe-0.15}/pspipe/templates/config/eor_bins_hba.parset +0 -0
  14. {pspipe-0.13 → pspipe-0.15}/pspipe/templates/config/eor_bins_nenufar.parset +0 -0
  15. {pspipe-0.13 → pspipe-0.15}/pspipe/templates/config/flagger.parset +0 -0
  16. {pspipe-0.13 → pspipe-0.15}/pspipe/templates/config/flagger_pre_combine.parset +0 -0
  17. {pspipe-0.13 → pspipe-0.15}/pspipe/templates/config/gpr_config_a12.parset +0 -0
  18. {pspipe-0.13 → pspipe-0.15}/pspipe/templates/config/gpr_config_hba.parset +0 -0
  19. {pspipe-0.13 → pspipe-0.15}/pspipe/templates/config/gpr_config_nenufar.parset +0 -0
  20. {pspipe-0.13 → pspipe-0.15}/pspipe/templates/config/gpr_config_v.parset +0 -0
  21. {pspipe-0.13 → pspipe-0.15}/pspipe/templates/config/ps_config_a12.parset +0 -0
  22. {pspipe-0.13 → pspipe-0.15}/pspipe/templates/config/ps_config_alo.parset +0 -0
  23. {pspipe-0.13 → pspipe-0.15}/pspipe/templates/config/ps_config_hba.parset +0 -0
  24. {pspipe-0.13 → pspipe-0.15}/pspipe/templates/config/ps_config_nenufar.parset +0 -0
  25. {pspipe-0.13 → pspipe-0.15}/pspipe/templates/default_a12.toml +0 -0
  26. {pspipe-0.13 → pspipe-0.15}/pspipe/templates/default_alo.toml +0 -0
  27. {pspipe-0.13 → pspipe-0.15}/pspipe/templates/default_hba.toml +0 -0
  28. {pspipe-0.13 → pspipe-0.15}/pspipe/templates/default_nenufar.toml +0 -0
  29. {pspipe-0.13 → pspipe-0.15}/pspipe/templates/default_settings.toml +0 -0
  30. {pspipe-0.13 → pspipe-0.15}/pspipe/tools/__init__.py +0 -0
  31. {pspipe-0.13 → pspipe-0.15}/pspipe/tools/psdb.py +0 -0
  32. {pspipe-0.13 → pspipe-0.15}/pspipe/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pspipe
3
- Version: 0.13
3
+ Version: 0.15
4
4
  Summary: Power spectra pipeline for Cosmic Dawn, Epoch of Reionization radio interferometric experiments
5
5
  License-Expression: GPL-3.0-or-later
6
6
  License-File: LICENSE
@@ -84,6 +84,8 @@ class AbstractTask:
84
84
  raise ValueError('Invalid worker settings.')
85
85
 
86
86
  self.worker = self._worker_pool(name)
87
+ # Tasks of the pools already run by this task (see init_for_new_task)
88
+ self.past_tasks = []
87
89
 
88
90
  def _worker_pool(self, name):
89
91
  s_worker = self.settings.worker
@@ -96,6 +98,7 @@ class AbstractTask:
96
98
  resources=settings.worker_resources(s_worker))
97
99
 
98
100
  def init_for_new_task(self, post_task_name):
101
+ self.past_tasks += self.worker.tasks
99
102
  self.worker = self._worker_pool(post_task_name)
100
103
 
101
104
  def add_sub_task(self, sub_task, run_on_host=None):
@@ -120,7 +123,7 @@ class AbstractTask:
120
123
  def execute(self):
121
124
  self.worker.execute()
122
125
 
123
- return self.get_report().ok
126
+ return self.get_pool_report().ok
124
127
 
125
128
  def execute_single_local(self, cmd, log_file):
126
129
  w = worker.WorkerPool([worker.localhost_shortname], name=self.name,
@@ -131,6 +134,11 @@ class AbstractTask:
131
134
  w.execute()
132
135
 
133
136
  def get_report(self):
137
+ '''Report of all the pools of the task.'''
138
+ return worker.WorkerPoolReport(self.past_tasks + self.worker.tasks)
139
+
140
+ def get_pool_report(self):
141
+ '''Report of the current pool only.'''
134
142
  return worker.WorkerPoolReport(self.worker.tasks)
135
143
 
136
144
 
@@ -191,10 +199,8 @@ class ImagingSubTask(AbstractSubTask):
191
199
  odd=self.odd if odd is None else odd, img_name=img_name)
192
200
  meta.save()
193
201
 
194
- def done_callback(self):
195
- for file in glob.glob(f'{self.get_fits_prefix()}*-MFS-*.fits'):
196
- os.remove(file)
197
-
202
+ def fits_keywords(self):
203
+ '''Integration time, total time and channel width of the images, for their headers.'''
198
204
  img = self.settings.image
199
205
 
200
206
  if img.time_start_index != img.time_end_index:
@@ -204,6 +210,15 @@ class ImagingSubTask(AbstractSubTask):
204
210
  else:
205
211
  total_time = self.obs_meta.total_time
206
212
 
213
+ return {'PEINTTIM': int(self.obs_meta.int_time), 'PETOTTIM': int(total_time),
214
+ 'PECHWIDT': int(self.obs_meta.chan_width)}
215
+
216
+ def done_callback(self):
217
+ for file in glob.glob(f'{self.get_fits_prefix()}*-MFS-*.fits'):
218
+ os.remove(file)
219
+
220
+ img = self.settings.image
221
+
207
222
  if self.even_odd:
208
223
  parts = [('-even', True, False), ('-odd', False, True)]
209
224
  else:
@@ -220,10 +235,12 @@ class ImagingSubTask(AbstractSubTask):
220
235
  files = glob.glob(f'{prefix}*.fits')
221
236
  img_name = 'image'
222
237
 
223
- for file in files:
224
- pf.setval(file, 'PEINTTIM', value=int(self.obs_meta.int_time))
225
- pf.setval(file, 'PETOTTIM', value=int(total_time))
226
- pf.setval(file, 'PECHWIDT', value=int(self.obs_meta.chan_width))
238
+ # psimager writes the keywords with the files (-fits-keyword)
239
+ if img.imager != 'psimager':
240
+ keywords = self.fits_keywords()
241
+ for file in files:
242
+ with pf.open(file, mode='update') as hdul:
243
+ hdul[0].header.update(keywords)
227
244
 
228
245
  if not img.lst_bins:
229
246
  self.register_files(files, img_name=img_name, even=even, odd=odd)
@@ -278,6 +295,9 @@ class ImagingSubTask(AbstractSubTask):
278
295
 
279
296
  args.append(f'-temp-dir {self.temp_dir}')
280
297
 
298
+ if img.imager == 'psimager':
299
+ args.append(' '.join(f'-fits-keyword {k}={v}' for k, v in self.fits_keywords().items()))
300
+
281
301
  if self.even_odd:
282
302
  args.append('-even-odd')
283
303
  elif self.even:
@@ -3,6 +3,7 @@
3
3
  import sys
4
4
 
5
5
  import click
6
+ from libpipe import worker
6
7
 
7
8
  from pspipe import __version__, database, tasks
8
9
 
@@ -32,6 +33,24 @@ def print_report(report):
32
33
  print("==========================\n")
33
34
 
34
35
 
36
+ def print_run_summary(reports, task_names):
37
+ """Recap of the run: one line per task (the tasks not run after a failure included), the
38
+ failed tasks with their logs, and the usage of all the tasks."""
39
+ print("\n=== Summary of the run ===")
40
+ for name in task_names:
41
+ report = dict(reports).get(name)
42
+ if report is None:
43
+ print(f"{name:20s} not run")
44
+ continue
45
+ state = "OK" if report.ok else "FAILED"
46
+ print(f"{name:20s} {state:7s} {report.n_success_rc0} / {report.n_total} tasks successful")
47
+ for t in report.success_nonzero + report.failed:
48
+ log = f" | log: {t.output_file}" if getattr(t, "output_file", None) else ""
49
+ print(f" - {t.name} | host={report.host_of(t)} | rc={t.returncode}{log}")
50
+ worker.WorkerPoolReport([t for _, r in reports for t in r.all_tasks]).print_usage()
51
+ print("==========================\n")
52
+
53
+
35
54
  @click.command(epilog='\b\nAvailable tasks:\n' + tasks_help)
36
55
  @click.version_option(__version__)
37
56
  @click.argument('tasks_str', metavar='TASKS')
@@ -62,13 +81,18 @@ def main(tasks_str, settings_file, obs_ids, config_modifier):
62
81
  print(f'Error: No obs_ids matching {obs_ids} in database.')
63
82
  sys.exit(1)
64
83
 
84
+ reports = []
65
85
  for task_name in tasks_str.split(','):
66
86
  t = all_tasks[task_name](s)
67
87
  obs_ids_set = t.run(obs_ids_set)
68
88
  report = t.get_report()
89
+ reports.append((task_name, report))
69
90
 
70
91
  if not obs_ids_set:
71
92
  print(f'Task {task_name} finished with errors')
72
93
  print_report(report)
94
+ print_run_summary(reports, tasks_str.split(','))
73
95
  sys.exit(1)
74
96
  print_report(report)
97
+
98
+ print_run_summary(reports, tasks_str.split(','))
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
4
4
 
5
5
  [project]
6
6
  name = "pspipe"
7
- version = "0.13"
7
+ version = "0.15"
8
8
  description = "Power spectra pipeline for Cosmic Dawn, Epoch of Reionization radio interferometric experiments"
9
9
  readme = "README.md"
10
10
  license = "GPL-3.0-or-later"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes