pspipe 0.8__tar.gz → 0.9__tar.gz

{pspipe-0.8 → pspipe-0.9}/PKG-INFO

@@ -1,7 +1,8 @@
 Metadata-Version: 2.4
 Name: pspipe
-Version: 0.8
+Version: 0.9
 Summary: Power spectra pipeline for Cosmic Dawn, Epoch of Reionization radio interferometric experiments
+License-Expression: GPL-3.0-or-later
 License-File: LICENSE
 Author: Florent Mertens
 Author-email: florent.mertens@gmail.com
@@ -12,16 +13,23 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
+Provides-Extra: dev
 Provides-Extra: ml-gpr
 Requires-Dist: astropy (>6.0)
 Requires-Dist: click (>=8.0)
 Requires-Dist: libpipe (>=0.6)
 Requires-Dist: ps-eor (>=0.8)
 Requires-Dist: ps-eor[ml-gpr] (>=0.8) ; extra == "ml-gpr"
+Requires-Dist: pytest (>=8.0) ; extra == "dev"
+Requires-Dist: pytest-cov (>=5.0) ; extra == "dev"
 Requires-Dist: python-casacore (>=3.0)
+Requires-Dist: ruff (==0.16.1) ; extra == "dev"
 Requires-Dist: tables (>=3.2)
 Requires-Dist: tabulate (>=0.8)
 Requires-Dist: toml (>=0.10)
+Project-URL: Homepage, https://gitlab.com/flomertens/pspipe
+Project-URL: Issues, https://gitlab.com/flomertens/pspipe/-/issues
+Project-URL: Repository, https://gitlab.com/flomertens/pspipe
 Description-Content-Type: text/markdown
 
 Power Spectra Generation pipeline for CD/EoR experiments

{pspipe-0.8 → pspipe-0.9}/pspipe/database.py

@@ -2,24 +2,22 @@
 #
 # Author: F. Mertens
 
-import os
-import hashlib
+import fnmatch
 import glob
+import hashlib
 import json
-import fnmatch
+import os
 from collections import OrderedDict
 
 import numpy as np
-
-from libpipe import attrmap
-from libpipe import futils
+from libpipe import attrmap, futils
 from libpipe.attrmap import OrderedSet
 
+from . import utils
+
 # Lazy loaded:
 # - .obsdata.ObsData in VisRevision.get_data()
-
 from .settings import Settings
-from . import utils
 
 OBS_ID_JSON_POSTFIX = '_obs_id.json'
 IMG_REV_JSON = 'img_list.json'
@@ -49,25 +47,28 @@ def get_type_name(stokes_or_psf, even, odd):
 def listdir_isdir(path, ignore=None):
     if not os.path.exists(path):
         return []
-    return [f for f in os.listdir(path) if os.path.isdir(os.path.join(path, f)) and not f.startswith('.') and
-            (ignore is None or f not in ignore)]
+    return [f for f in os.listdir(path)
+            if os.path.isdir(os.path.join(path, f)) and not f.startswith('.')
+            and (ignore is None or f not in ignore)]
 
 
-class Database(object):
+class Database:
 
     def __init__(self, settings_obj):
         self.settings = settings_obj
 
     def list_obs_ids(self):
-        files = glob.glob(os.path.join(self.settings.get_path('data_dir'), 'obs_ids', f'*{OBS_ID_JSON_POSTFIX}'))
+        files = glob.glob(os.path.join(self.settings.get_path('data_dir'), 'obs_ids',
+                                       f'*{OBS_ID_JSON_POSTFIX}'))
         return [os.path.basename(k).split(OBS_ID_JSON_POSTFIX)[0] for k in files]
 
     def filter_obs_ids(self, obs_id_str):
         all_obs_ids = self.list_obs_ids()
-        return set(obs_id for k in obs_id_str.split(',') for obs_id in fnmatch.filter(all_obs_ids, k))
+        return {obs_id for k in obs_id_str.split(',') for obs_id in fnmatch.filter(all_obs_ids, k)}
 
     def get_obs_id_meta_path(self, obs_id):
-        return os.path.join(self.settings.get_path('data_dir'), 'obs_ids', obs_id + OBS_ID_JSON_POSTFIX)
+        return os.path.join(self.settings.get_path('data_dir'), 'obs_ids',
+                            obs_id + OBS_ID_JSON_POSTFIX)
 
     def get_obs_id_meta(self, obs_id):
         obs_id_meta_file = self.get_obs_id_meta_path(obs_id)
@@ -108,7 +109,8 @@ class Database(object):
 
     def list_image_revisions(self):
         return [ImageRevision(Settings.load_with_defaults(k))
-                for k in glob.glob(os.path.join(self.settings.get_path('data_dir'), '*', IMG_REV_TOML))]
+                for k in glob.glob(os.path.join(self.settings.get_path('data_dir'), '*',
+                                                IMG_REV_TOML))]
 
     def get_image_revision(self, img_rev_name=None):
         img_rev_conf_file = ImageRevision(self.settings, img_rev_name).get_settings_path()
@@ -130,7 +132,7 @@ class Database(object):
         os.remove(self.get_obs_id_meta_path(obs_id))
 
 
-class ImageRevision(object):
+class ImageRevision:
 
     def __init__(self, settings_obj, img_rev_name=None):
         self.settings = settings_obj.copy()
@@ -188,8 +190,11 @@ class ImageRevision(object):
         obs_meta = Database(self.settings).get_obs_id_meta(obs_id)
         start, end, interval = self.settings.image.lst_bins
 
-        s2day = lambda sec: sec / 3600. / 24.
-        hr2int = lambda hr: int(hr * 3600. / obs_meta.int_time)
+        def s2day(sec):
+            return sec / 3600. / 24.
+
+        def hr2int(hr):
+            return int(hr * 3600. / obs_meta.int_time)
 
         start_lst = utils.get_lst(s2day(obs_meta.start_time), self.settings.longitude)
         end_lst = utils.get_lst(s2day(obs_meta.end_time), self.settings.longitude)
@@ -199,7 +204,8 @@ class ImageRevision(object):
         bin_int = hr2int(interval)
 
         for i, (s, e) in enumerate(utils.pairwise(np.arange(start, end + interval, interval))):
-            if utils.is_in_lst_bin(s, start_lst, end_lst) and utils.is_in_lst_bin(e, start_lst, end_lst):
+            if (utils.is_in_lst_bin(s, start_lst, end_lst)
+                    and utils.is_in_lst_bin(e, start_lst, end_lst)):
                 lst_bins[i] = [s, e]
             if utils.is_in_lst_bin(start_lst, s, e):
                 start_index = hr2int(abs(e - start_lst))
@@ -209,7 +215,8 @@ class ImageRevision(object):
 
         if return_time_index:
             lst_idx = lst_bins.keys()
-            data_lst_idx = np.arange(start_lst_idx + 1, start_lst_idx + 1 + len(lst_idx)) % (max(lst_idx) + 1)
+            data_lst_idx = np.arange(start_lst_idx + 1,
+                    start_lst_idx + 1 + len(lst_idx)) % (max(lst_idx) + 1)
 
             return lst_bins, (start_index, end_index, bin_int), data_lst_idx
 
@@ -240,14 +247,15 @@ class ImageRevision(object):
 
     def filter_obs_ids(self, obs_id_str):
         all_obs_ids = self.list_obs_ids()
-        return set(obs_id for k in obs_id_str.split(',') for obs_id in fnmatch.filter(all_obs_ids, k))
+        return {obs_id for k in obs_id_str.split(',') for obs_id in fnmatch.filter(all_obs_ids, k)}
 
     def has_obs_id(self, obs_id):
         return obs_id in self.list_obs_ids()
 
     def new_obs_id(self, obs_id, lst_idx=None, data_dir_root=None):
-        img_meta = ImageRevisionMeta.new(self.get_meta_path(obs_id, lst_idx=lst_idx), obs_id, self.settings.image.name,
-                                         data_dir_root=data_dir_root)
+        img_meta = ImageRevisionMeta.new(
+                self.get_meta_path(obs_id, lst_idx=lst_idx), obs_id, self.settings.image.name,
+                data_dir_root=data_dir_root)
         os.makedirs(os.path.dirname(self.get_meta_path(obs_id, lst_idx=lst_idx)), exist_ok=True)
         img_meta.save()
 
@@ -286,7 +294,7 @@ class ImageRevision(object):
             futils.rm_if_exist(self.get_path())
 
 
-class VisRevision(object):
+class VisRevision:
 
     def __init__(self, settings_obj, img_rev_name=None, **vis_kargs):
         self.settings = settings_obj.copy()
@@ -332,7 +340,7 @@ class VisRevision(object):
 
     def filter_obs_ids(self, obs_id_str):
         all_obs_ids = self.list_obs_ids()
-        return set(obs_id for k in obs_id_str.split(',') for obs_id in fnmatch.filter(all_obs_ids, k))
+        return {obs_id for k in obs_id_str.split(',') for obs_id in fnmatch.filter(all_obs_ids, k)}
 
     def has_obs_id(self, obs_id):
         return obs_id in self.list_obs_ids()
@@ -345,7 +353,8 @@ class VisRevision(object):
         return img_meta
 
     def get_lst_bins(self, obs_id, return_time_index=False):
-        return ImageRevision(self.settings).get_lst_bins(obs_id, return_time_index=return_time_index)
+        return ImageRevision(self.settings).get_lst_bins(obs_id,
+                                                         return_time_index=return_time_index)
 
     def get_meta(self, obs_id, lst_idx=None):
         meta_file = self.get_meta_path(obs_id, lst_idx=lst_idx)
@@ -397,7 +406,7 @@ class ObsIDMeta(AbstractJsonMeta):
 
     @staticmethod
     def new(file, obs_id, **kargs):
-        assert utils.all_in_other(kargs, obs_id_info_keys + ['ms_files'])
+        assert utils.all_in_other(kargs, [*obs_id_info_keys, 'ms_files'])
 
         d = {'obs_id': obs_id, 'ms_files': []}
         d.update(dict.fromkeys(obs_id_info_keys, 0))
@@ -430,7 +439,7 @@ class ImageRevisionMeta(AbstractJsonMeta):
 
     def get_fits(self, stokes_or_psf, even=False, odd=False):
         assert (even != odd) or (even is False)
-        assert stokes_or_psf in all_stokes + ['psf', None]
+        assert stokes_or_psf in [*all_stokes, 'psf', None]
 
         name = get_type_name(stokes_or_psf, even, odd)
 
@@ -443,14 +452,15 @@ class ImageRevisionMeta(AbstractJsonMeta):
         return get_type_name(stokes, even, odd) in self
 
     def get_all_stokes(self):
-        return set(k.split('_')[0] for k in self if k.split('_')[0].upper() in all_stokes)
+        return {k.split('_')[0] for k in self if k.split('_')[0].upper() in all_stokes}
 
     def get_fits_stat(self):
-        return dict((k, len(v)) for k, v in self.items() if isinstance(v, list) and len(v) > 0)
+        return {k: len(v) for k, v in self.items() if isinstance(v, list) and len(v) > 0}
 
-    def add_fits(self, files, stokes_or_psf=None, even=False, odd=False, verbose=False, img_name='image'):
+    def add_fits(self, files, stokes_or_psf=None, even=False, odd=False, verbose=False,
+                 img_name='image'):
         assert (even != odd) or (even is False)
-        assert stokes_or_psf in all_stokes + ['psf', None]
+        assert stokes_or_psf in [*all_stokes, 'psf', None]
 
         if stokes_or_psf is not None:
             name = get_type_name(stokes_or_psf, even=even, odd=odd)
@@ -477,7 +487,7 @@ class ImageRevisionMeta(AbstractJsonMeta):
                 i_files = fnmatch.filter(files, f'*{img_name}.fits')
                 if i_files:
                     self.add_fits(i_files, 'I', even=even, odd=odd, verbose=verbose)
-            psf_files = fnmatch.filter(files, f'*psf.fits')
+            psf_files = fnmatch.filter(files, '*psf.fits')
             if psf_files:
                 self.add_fits(psf_files, 'psf', even=even, odd=odd, verbose=verbose)
 
@@ -498,5 +508,5 @@ class VisRevisionMeta(AbstractJsonMeta):
         return get_type_name(stokes, even, odd) in self
 
     def get_all_stokes(self):
-        return set(k.split('_')[0] for k in self if k.split('_')[0].upper() in all_stokes)
+        return {k.split('_')[0] for k in self if k.split('_')[0].upper() in all_stokes}
 

{pspipe-0.8 → pspipe-0.9}/pspipe/obsdata.py

@@ -5,8 +5,7 @@
 import os
 
 import numpy as np
-
-from ps_eor import pspec, datacube, flagger
+from ps_eor import datacube, flagger, pspec
 
 from . import database, utils
 
@@ -14,7 +13,7 @@ from . import database, utils
 # from ps_eor import ml_gpr
 
 
-class ObsData(object):
+class ObsData:
 
     def __init__(self, vis_rev, obs_id, lst_idx=None):
         self.vis_rev = vis_rev
@@ -28,13 +27,13 @@ class ObsData(object):
         self.all_stokes = {}
         for stokes in database.all_stokes:
             self.all_stokes[stokes.lower()] = stokes
-            self.all_stokes[stokes.lower() + '_even'] = 'even_%s' % stokes
-            self.all_stokes[stokes.lower() + '_odd'] = 'odd_%s' % stokes
-            self.all_stokes[stokes.lower() + '_dt'] = 'dt_%s' % stokes
+            self.all_stokes[stokes.lower() + '_even'] = f'even_{stokes}'
+            self.all_stokes[stokes.lower() + '_odd'] = f'odd_{stokes}'
+            self.all_stokes[stokes.lower() + '_dt'] = f'dt_{stokes}'
         self._ps_builder = None
 
     def __getattr__(self, key):
-        assert key in self.all_stokes.keys()
+        assert key in self.all_stokes
         return self.get_stokes(self.all_stokes[key])
 
     def _invalidate_cache(self):
@@ -131,30 +130,35 @@ class ObsData(object):
         return stokes in self.data or os.path.exists(self.get_h5_file(stokes))
 
     def get_ml_gpr_path(self, eor_bin, umin=50, umax=250, name='', sampler_method='mcmc'):
-        return os.path.join(os.path.dirname(self.get_h5_file('I')), name, f'eor{eor_bin}_u{umin}-{umax}')
+        return os.path.join(os.path.dirname(self.get_h5_file('I')), name,
+                            f'eor{eor_bin}_u{umin}-{umax}')
 
     def get_ml_gpr_res(self, eor_bin, umin=50, umax=250, name='', sampler_method='mcmc'):
         from ps_eor import ml_gpr
 
-        path = self.get_ml_gpr_path(eor_bin, umin=umin, umax=umax, name=name, sampler_method=sampler_method)
+        path = self.get_ml_gpr_path(eor_bin, umin=umin, umax=umax, name=name,
+                                    sampler_method=sampler_method)
         return ml_gpr.MLGPRResult.load(path, 'ml_gpr_res_I', sampler_method=sampler_method)
 
     def get_ml_gpr_inj_path(self, eor_bin, var_21_over_noise, x1, x2, umin=50, umax=250, name=''):
         d = f'inj_eor{eor_bin}_u{umin}-{umax}_a{var_21_over_noise:.3f}_x1{x1:.3f}_x2{x2:.3f}'
         return os.path.join(os.path.dirname(self.get_h5_file('I')), name, d)
 
-    def get_ml_gpr_inj_res(self, eor_bin, var_21_over_noise, x1, x2, umin=50, umax=250, name='', sampler_method='mcmc'):
+    def get_ml_gpr_inj_res(self, eor_bin, var_21_over_noise, x1, x2, umin=50, umax=250, name='',
+                           sampler_method='mcmc'):
         from ps_eor import ml_gpr
 
-        path = self.get_ml_gpr_inj_path(eor_bin, var_21_over_noise=var_21_over_noise, x1=x1, x2=x2, umin=umin, umax=umax, name=name)
+        path = self.get_ml_gpr_inj_path(eor_bin, var_21_over_noise=var_21_over_noise, x1=x1, x2=x2,
+                                        umin=umin, umax=umax, name=name)
         data_inj = datacube.CartDataCube.load(os.path.join(path, 'c_injected.h5'))
-        return ml_gpr.MLGPRInjResult.load(path, 'ml_gpr_res_I', data_inj, sampler_method=sampler_method)
+        return ml_gpr.MLGPRInjResult.load(path, 'ml_gpr_res_I', data_inj,
+                                          sampler_method=sampler_method)
 
     def get_ps_gen(self, **kargs):
         return self.ps_builder.get(self.get_stokes_i_like(), **kargs)
 
 
-class ObsDataGroup(object):
+class ObsDataGroup:
     """A group of observations exposed through a combined ObsData-like view."""
 
     def __init__(self, observations, name, umin, umax, weighting_mode='full',
@@ -184,9 +188,9 @@ class ObsDataGroup(object):
                 self.observation_weights = {
                     obs_id: float(weight)
                     for obs_id, weight in dict(observation_weights).items()}
-            except (TypeError, ValueError):
+            except (TypeError, ValueError) as err:
                 raise ValueError(
-                    'observation_weights must contain real scalar values')
+                    'observation_weights must contain real scalar values') from err
             missing = set(obs_ids) - set(self.observation_weights)
             unknown = set(self.observation_weights) - set(obs_ids)
             if missing or unknown:
@@ -257,7 +261,7 @@ class ObsDataGroup(object):
 
             for obs, cube in zip(
                     self.observations,
-                    self.get_observation_cubes(stokes)):
+                    self.get_observation_cubes(stokes), strict=False):
                 cube = cube.copy()
                 if cube.weights is None:
                     raise ValueError(
@@ -284,7 +288,7 @@ class ObsDataGroup(object):
             self.get_stokes_i_like(), **kargs)
 
 
-class ObsDataCollection(object):
+class ObsDataCollection:
     """Observation groups indexed by contiguous LST bins."""
 
     def __init__(self, lst_bins, groups):

{pspipe-0.8 → pspipe-0.9}/pspipe/settings.py

@@ -5,16 +5,13 @@
 import os
 
 import astropy.units as u
-
 from libpipe import futils
 from libpipe.settings import BaseSettings
 
 # Lazy loaded:
 # - ps_eor.datacube in Settings.validate_vis_cube()
-
 from . import utils
 
-
 TEMPLATE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'templates')
 
 TEMPLATE_NAMES = ['a12', 'nenufar', 'hba', 'alo']
@@ -29,7 +26,8 @@ class Settings(BaseSettings):
 
     def validate_image(self, val_fct):
         val_fct(u.Unit(self.image.scale).is_equivalent((u.rad, u.s)), 'scale')
-        val_fct(len(self.image.size.split()) == 2 and all(k.isdigit() for k in self.image.size.split()), 'size')
+        val_fct(len(self.image.size.split()) == 2
+                and all(k.isdigit() for k in self.image.size.split()), 'size')
         val_fct(self.image.channels_out == 'all' or str(self.image.channels_out).startswith(
             'every') or utils.is_number(self.image.channels_out), 'channels_out')
         val_fct(isinstance(self.image.umin, (int, float)), 'umin')
@@ -40,7 +38,8 @@ class Settings(BaseSettings):
         val_fct(isinstance(self.image.time_end_index, int), 'time_end_index')
         if self.image.time_end_index or self.image.time_start_index:
             val_fct(self.image.time_end_index >= self.image.time_start_index, 'time_end_index')
-        val_fct(isinstance(self.image.lst_bins, list) and len(self.image.lst_bins) in [0, 3], 'lst_bins')
+        val_fct(isinstance(self.image.lst_bins, list) and len(self.image.lst_bins) in [0, 3],
+                'lst_bins')
         val_fct(isinstance(self.image.clean_niter, (int, float)), 'clean_niter')
         val_fct(isinstance(self.image.clean_auto_threshold, (int, float)), 'clean_auto_threshold')
         val_fct(isinstance(self.image.clean_mgain, (int, float)), 'clean_mgain')
@@ -59,7 +58,8 @@ class Settings(BaseSettings):
         val_fct(isinstance(self.merge_ms.time_start_index, int), 'time_start_index')
         val_fct(isinstance(self.merge_ms.time_end_index, int), 'time_end_index')
         if self.merge_ms.time_end_index or self.merge_ms.time_start_index:
-            val_fct(self.merge_ms.time_end_index >= self.merge_ms.time_start_index, 'time_end_index')
+            val_fct(self.merge_ms.time_end_index >= self.merge_ms.time_start_index,
+                    'time_end_index')
 
     def validate_vis_cube(self, val_fct):
         from ps_eor import datacube
@@ -67,8 +67,9 @@ class Settings(BaseSettings):
         val_fct(isinstance(self.vis_cube.fov, (int, float, str)), 'fov')
         val_fct(isinstance(self.vis_cube.umin, (int, float)), 'umin')
         val_fct(isinstance(self.vis_cube.umax, (int, float)), 'umax')
-        val_fct(len(datacube.get_window(datacube.WindowFunction.parse_winfct_str(self.vis_cube.win_fct), 3)) == 3,
-                'win_fct')
+        val_fct(len(datacube.get_window(
+            datacube.WindowFunction.parse_winfct_str(self.vis_cube.win_fct), 3)) == 3,
+            'win_fct')
 
     def validate_vis_to_sph(self, val_fct):
         val_fct(futils.is_file(self.vis_to_sph.pre_flag), 'pre_flag')
@@ -103,6 +104,7 @@ class Settings(BaseSettings):
 
 
 def get_template_path(template_name):
-    assert template_name in TEMPLATE_NAMES, f'template_name should be one of: {" ".join(TEMPLATE_NAMES)}'   
+    assert template_name in TEMPLATE_NAMES, (f'template_name should be one of: '
+                                             f'{" ".join(TEMPLATE_NAMES)}')
 
     return os.path.join(TEMPLATE_DIR, f'default_{template_name}.toml')