pspipe 0.14__tar.gz → 0.16__tar.gz

{pspipe-0.14 → pspipe-0.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pspipe
-Version: 0.14
+Version: 0.16
 Summary: Power spectra pipeline for Cosmic Dawn, Epoch of Reionization radio interferometric experiments
 License-Expression: GPL-3.0-or-later
 License-File: LICENSE
@@ -20,7 +20,7 @@ Requires-Dist: astropy (>6.0)
 Requires-Dist: click (>=8.0)
 Requires-Dist: libpipe (>=0.9)
 Requires-Dist: myst-parser (>=2) ; extra == "docs"
-Requires-Dist: ps-eor (>=1.0)
+Requires-Dist: ps-eor (>=1.3)
 Requires-Dist: ps-eor[ml-gpr] (>=1.0) ; extra == "ml-gpr"
 Requires-Dist: pydata-sphinx-theme (>=0.15) ; extra == "docs"
 Requires-Dist: pytest (>=8.0) ; extra == "dev"
@@ -29,7 +29,6 @@ Requires-Dist: python-casacore (>=3.0)
 Requires-Dist: ruff (==0.16.1) ; extra == "dev"
 Requires-Dist: sphinx (>=7) ; extra == "docs"
 Requires-Dist: sphinx-click (>=6) ; extra == "docs"
-Requires-Dist: tables (>=3.2)
 Requires-Dist: tabulate (>=0.8)
 Requires-Dist: toml (>=0.10)
 Project-URL: Documentation, https://pspipe-eor.readthedocs.io

{pspipe-0.14 → pspipe-0.16}/pspipe/tasks.py

@@ -84,6 +84,8 @@ class AbstractTask:
             raise ValueError('Invalid worker settings.')
 
         self.worker = self._worker_pool(name)
+        # Tasks of the pools already run by this task (see init_for_new_task)
+        self.past_tasks = []
 
     def _worker_pool(self, name):
         s_worker = self.settings.worker
@@ -96,6 +98,7 @@ class AbstractTask:
                 resources=settings.worker_resources(s_worker))
 
     def init_for_new_task(self, post_task_name):
+        self.past_tasks += self.worker.tasks
         self.worker = self._worker_pool(post_task_name)
 
     def add_sub_task(self, sub_task, run_on_host=None):
@@ -120,7 +123,7 @@ class AbstractTask:
     def execute(self):
         self.worker.execute()
 
-        return self.get_report().ok
+        return self.get_pool_report().ok
 
     def execute_single_local(self, cmd, log_file):
         w = worker.WorkerPool([worker.localhost_shortname], name=self.name,
@@ -131,6 +134,11 @@ class AbstractTask:
         w.execute()
 
     def get_report(self):
+        '''Report of all the pools of the task.'''
+        return worker.WorkerPoolReport(self.past_tasks + self.worker.tasks)
+
+    def get_pool_report(self):
+        '''Report of the current pool only.'''
         return worker.WorkerPoolReport(self.worker.tasks)
 
 
@@ -289,6 +297,9 @@ class ImagingSubTask(AbstractSubTask):
 
         if img.imager == 'psimager':
             args.append(' '.join(f'-fits-keyword {k}={v}' for k, v in self.fits_keywords().items()))
+            # One file per image: on a parallel file system, opening a file costs more than
+            # reading it
+            args.append('-cube')
 
         if self.even_odd:
             args.append('-even-odd')

{pspipe-0.14 → pspipe-0.16}/pspipe/tools/pspipe.py

@@ -3,6 +3,7 @@
 import sys
 
 import click
+from libpipe import worker
 
 from pspipe import __version__, database, tasks
 
@@ -32,6 +33,24 @@ def print_report(report):
     print("==========================\n")
 
 
+def print_run_summary(reports, task_names):
+    """Recap of the run: one line per task (the tasks not run after a failure included), the
+    failed tasks with their logs, and the usage of all the tasks."""
+    print("\n=== Summary of the run ===")
+    for name in task_names:
+        report = dict(reports).get(name)
+        if report is None:
+            print(f"{name:20s} not run")
+            continue
+        state = "OK" if report.ok else "FAILED"
+        print(f"{name:20s} {state:7s} {report.n_success_rc0} / {report.n_total} tasks successful")
+        for t in report.success_nonzero + report.failed:
+            log = f" | log: {t.output_file}" if getattr(t, "output_file", None) else ""
+            print(f"    - {t.name} | host={report.host_of(t)} | rc={t.returncode}{log}")
+    worker.WorkerPoolReport([t for _, r in reports for t in r.all_tasks]).print_usage()
+    print("==========================\n")
+
+
 @click.command(epilog='\b\nAvailable tasks:\n' + tasks_help)
 @click.version_option(__version__)
 @click.argument('tasks_str', metavar='TASKS')
@@ -62,13 +81,18 @@ def main(tasks_str, settings_file, obs_ids, config_modifier):
         print(f'Error: No obs_ids matching {obs_ids} in database.')
         sys.exit(1)
 
+    reports = []
     for task_name in tasks_str.split(','):
         t = all_tasks[task_name](s)
         obs_ids_set = t.run(obs_ids_set)
         report = t.get_report()
+        reports.append((task_name, report))
 
         if not obs_ids_set:
             print(f'Task {task_name} finished with errors')
             print_report(report)
+            print_run_summary(reports, tasks_str.split(','))
             sys.exit(1)
         print_report(report)
+
+    print_run_summary(reports, tasks_str.split(','))

{pspipe-0.14 → pspipe-0.16}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [project]
 name = "pspipe"
-version = "0.14"
+version = "0.16"
 description = "Power spectra pipeline for Cosmic Dawn, Epoch of Reionization radio interferometric experiments"
 readme = "README.md"
 license = "GPL-3.0-or-later"
@@ -17,9 +17,8 @@ dependencies = [
     "toml>=0.10",
     "click>=8.0",
     "astropy>6.0",
-    "tables>=3.2",
     "tabulate>=0.8",
-    "ps-eor>=1.0",
+    "ps-eor>=1.3",
     "libpipe>=0.9",
     "python-casacore>=3.0",
 ]