pz-rail-astro-tools 1.0.6__py3-none-any.whl → 1.0.8__py3-none-any.whl

{pz_rail_astro_tools-1.0.6.dist-info → pz_rail_astro_tools-1.0.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: pz-rail-astro-tools
-Version: 1.0.6
+Version: 1.0.8
 Author-email: "LSST Dark Energy Science Collaboration (DESC)" <lsst-desc-rail-admin@slac.stanford.edu>
 License: MIT License
         
@@ -34,9 +34,9 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: deprecated
-Requires-Dist: pz-rail-base >=1.0.3
+Requires-Dist: pz-rail-base>=1.0.3
 Requires-Dist: numpy
-Requires-Dist: pandas >=2.2.2
+Requires-Dist: pandas>=2.2.2
 Requires-Dist: astropy
 Requires-Dist: healpy
 Requires-Dist: photerr
@@ -45,11 +45,11 @@ Requires-Dist: pz-hyperbolic-temp
 Requires-Dist: lsstdesc-gcr-catalogs
 Requires-Dist: FoFCatalogMatching
 Provides-Extra: dev
-Requires-Dist: tables-io[full] ; extra == 'dev'
-Requires-Dist: pytest ; extra == 'dev'
-Requires-Dist: pytest-cov ; extra == 'dev'
-Requires-Dist: pre-commit ; extra == 'dev'
-Requires-Dist: pylint ; extra == 'dev'
+Requires-Dist: tables-io[full]; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: pre-commit; extra == "dev"
+Requires-Dist: pylint; extra == "dev"
 
 # pz-rail-astro-tools
 

{pz_rail_astro_tools-1.0.6.dist-info → pz_rail_astro_tools-1.0.8.dist-info}/RECORD

@@ -1,11 +1,11 @@
 rail/astro_tools/__init__.py,sha256=4J-E0Ug0RP3Y38g8kJ6auAPDPEI4Osf19tuSOAACfxk,432
-rail/astro_tools/_version.py,sha256=T17ZEPR5Omt5RLA0TuZWTufRi58PzGIPek75nFcuMQY,411
+rail/astro_tools/_version.py,sha256=vy0P95Si-KzUl6NsixjrWEHfr5xcmkA_oO_S3bQRmvE,411
 rail/creation/degraders/grid_selection.py,sha256=gJKM0AYdG-HgvjTS3ovupbyoB5bxKhZn9FAAb94pQXY,12429
 rail/creation/degraders/observing_condition_degrader.py,sha256=y6Nal_kSdAz6N8amY4oV6mQ3upa2nIgVbV9P6F0PKwA,19977
-rail/creation/degraders/photometric_errors.py,sha256=xeLI5dH4P-kseClMsy7vTes6EBXh2Ca8_-9-nnFt3b0,4396
+rail/creation/degraders/photometric_errors.py,sha256=jx7JL5-lzKm6IbNTYwHqkEeGZduXC59Tad2UNwEJWlc,4703
 rail/creation/degraders/spectroscopic_degraders.py,sha256=27kylHksXf4MhgdfefSL7C2S3WCK-0YQ1Oc8dOHIBhc,5350
 rail/creation/degraders/spectroscopic_selections.py,sha256=TXlghBWLAniokbUpgw4IQVf3y3R6BwralS0ddiVgMp4,22205
-rail/creation/degraders/unrec_bl_model.py,sha256=8kDTDADMOu46BPRBKG9HtVCI9Q71WeQkxYcItzcQC8M,5192
+rail/creation/degraders/unrec_bl_model.py,sha256=E89DVV67FcHozCjtUz4ty3Ak3q8H9lnNnh5wNWi_R8M,7476
 rail/creation/engines/gcr_engine.py,sha256=LSp599uQGu8GcFNUmib792ScQwSB_Og8VxPLTt9aSNU,2981
 rail/examples_data/creation_data/data/HSC_grid_settings.pkl,sha256=zpJoSR9TSnDyL64DPXivIMam3fn6GOZoyoJoeBj2LAk,141
 rail/examples_data/creation_data/data/hsc_ratios_and_specz.hdf5,sha256=liH54wuuuHxToqjoAwQ_fFmEefk3haNweVThMwdDKN4,13823712
@@ -14,12 +14,13 @@ rail/examples_data/creation_data/data/survey_conditions/DC2-mask-neg-nside-128.f
 rail/examples_data/creation_data/data/survey_conditions/minion_1016_dc2_Median_airmass_i_and_nightlt1825_HEAL.fits,sha256=twvyce70G3h1MQnGGFQJG9wZc8UbuNqXc6mjtiuaL1s,1581120
 rail/examples_data/creation_data/data/survey_conditions/minion_1016_dc2_Median_fiveSigmaDepth_i_and_nightlt1825_HEAL.fits,sha256=RbMMFSVJQxGhZnL4BnISE9VTNljfTZwV6Bax6NxverY,1581120
 rail/examples_data/testdata/rubin_dm_dc2_example2.pq,sha256=_Iihbbcz1A332QAkaMiFacT4QAbBQBMTOwpfrS7AclE,25884
-rail/pipelines/degradation/apply_phot_errors.py,sha256=vj2avYWxfavORIXzJ7925V9YkuRmRMBSP0C8-DmgkSw,1958
+rail/pipelines/degradation/apply_phot_errors.py,sha256=E6H1Qza1IkO9VWqqxeny6MYkw3gURWscwKxzgUHEOko,2087
 rail/pipelines/degradation/blending.py,sha256=oxZHy1Tai70wT4g4xnBr24z2LxmR1p_J6lMoTN0nFc8,564
 rail/pipelines/degradation/spectroscopic_selection_pipeline.py,sha256=D0sDi6nrasv5CvQkLXgmMp0QbDrhGFRXzt6sWjlbD68,2028
+rail/pipelines/degradation/truth_to_observed.py,sha256=nfzMKdZd69_P1HnHy56UcjMUpFdRHf6g55oNhlW5Gzg,2995
 rail/tools/photometry_tools.py,sha256=wfKAeMz5T_CYDPC661MHtU1aJXVqomvahZl87KuE7uY,19541
-pz_rail_astro_tools-1.0.6.dist-info/LICENSE,sha256=tLMEN21HbzvT-7umOVVvPjaJZbQxCa-2bAeYLwG5Q04,1102
-pz_rail_astro_tools-1.0.6.dist-info/METADATA,sha256=FZnRuWE7ouiTsuikU28i2mrssyUgeGsNsEcYW6xzpGs,4040
-pz_rail_astro_tools-1.0.6.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
-pz_rail_astro_tools-1.0.6.dist-info/top_level.txt,sha256=6R6sqn-85I8YZTzmB9gv7sqckB2QaOTHbu-QLGTWBRE,5
-pz_rail_astro_tools-1.0.6.dist-info/RECORD,,
+pz_rail_astro_tools-1.0.8.dist-info/LICENSE,sha256=tLMEN21HbzvT-7umOVVvPjaJZbQxCa-2bAeYLwG5Q04,1102
+pz_rail_astro_tools-1.0.8.dist-info/METADATA,sha256=6crQYG2g1fqpab7nJGcE-I2pXAbhy0qkoiERUSj2NmM,4033
+pz_rail_astro_tools-1.0.8.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+pz_rail_astro_tools-1.0.8.dist-info/top_level.txt,sha256=6R6sqn-85I8YZTzmB9gv7sqckB2QaOTHbu-QLGTWBRE,5
+pz_rail_astro_tools-1.0.8.dist-info/RECORD,,

{pz_rail_astro_tools-1.0.6.dist-info → pz_rail_astro_tools-1.0.8.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.5.0)
+Generator: setuptools (75.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

rail/astro_tools/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.0.6'
-__version_tuple__ = version_tuple = (1, 0, 6)
+__version__ = version = '1.0.8'
+__version_tuple__ = version_tuple = (1, 0, 8)

rail/creation/degraders/photometric_errors.py

@@ -47,21 +47,27 @@ class PhotoErrorModel(Noisifier):
                 default = val.default
                 
             # Add this param to config_options
-            self.config[key] = Param(
+            # Use setattr() becuase ceci.StageConfig has
+            # implemented __setitem__ to just set the value
+            # rather than add the parameters
+            param = Param(
                 None,  # Let PhotErr handle type checking
                 default,  
                 msg="See the main docstring for details about this parameter.",
                 required=False,
             )
+            setattr(self.config, key, param)
 
     def reload_pars(self, args):
         """ This is needed b/c the parameters are dynamically defined, 
         so we have to reload them _after_ then have been defined """
-        copy_args = args.copy()
         if isinstance(args, dict):
+            # coming from python, add 'config' to the configuration
+            copy_args = args.copy()
             copy_args['config'] = args
         else:  # pragma: no cover
-            copy_args['config'] = vars(args)            
+            # coming from cli, just convert to a dict
+            copy_args = vars(args).copy()
         self.load_configs(copy_args)
         self._io_checked = False
         self.check_io()

rail/creation/degraders/unrec_bl_model.py

@@ -3,10 +3,13 @@
 from ceci.config import StageParameter as Param
 from rail.creation.degrader import Degrader
 from rail.core.data import PqHandle
+from rail.core.common_params import SHARED_PARAMS
 import numpy as np, pandas as pd
 import FoFCatalogMatching
 
 
+lsst_zp_dict = {'u':12.65, 'g':14.69, 'r':14.56, 'i': 14.38, 'z':13.99, 'y': 13.02}
+
 class UnrecBlModel(Degrader):
     """Model for Creating Unrecognized Blends.
 
@@ -20,7 +23,10 @@ class UnrecBlModel(Degrader):
     config_options.update(ra_label=Param(str, 'ra', msg='ra column name'),
                           dec_label=Param(str, 'dec', msg='dec column name'),
                           linking_lengths=Param(float, 1.0, msg='linking_lengths for FoF matching'),
-                          bands=Param(str, 'ugrizy', msg='name of filters'),
+                          bands=SHARED_PARAMS,
+                          zp_dict=Param(dict, lsst_zp_dict, msg='magnitude zeropoints dictionary'),
+                          ref_band=SHARED_PARAMS,
+                          redshift_col=SHARED_PARAMS,
                           match_size=Param(bool, False, msg='consider object size for finding blends'),
                           match_shape=Param(bool, False, msg='consider object shape for finding blends'),
                           obj_size=Param(str, 'obj_size', msg='object size column name'),
@@ -30,6 +36,8 @@ class UnrecBlModel(Degrader):
 
     outputs = [("output", PqHandle), ("compInd", PqHandle)]
 
+    blend_info_cols = ['group_id', 'n_obj', 'brightest_flux', 'total_flux', 'z_brightest', 'z_weighted', 'z_mean', 'z_stdev']
+
     def __call__(self, sample, seed: int = None):
         """The main interface method for ``Degrader``.
 
@@ -89,32 +97,73 @@ class UnrecBlModel(Degrader):
     def __merge_bl__(self, data):
 
         """Merge sources within a group into unrecognized blends."""
-        
+
         group_id = data['group_id']
         unique_id = np.unique(group_id)
 
         ra_label, dec_label = self.config.ra_label, self.config.dec_label
-        cols = [ra_label, dec_label] + [b for b in self.config.bands] + ['group_id']
+        cols = [ra_label, dec_label] + [b for b in self.config.bands] + [self.config.redshift_col] + self.blend_info_cols
 
         N_rows = len(unique_id)
         N_cols = len(cols)
 
+        # compute the fluxes once for all the galaxies
+        fluxes = {b:10**(-(data[b] - self.config.zp_dict[b])/2.5) for b in self.config.bands}
+
+        # pull the column indices
+        idx_ra = cols.index(ra_label)
+        idx_dec = cols.index(dec_label)
+        idx_redshift = cols.index(self.config.redshift_col)
+        idx_n_obj = cols.index('n_obj')
+        idx_brightest_flux = cols.index('brightest_flux')
+        idx_total_flux = cols.index('total_flux')
+        idx_z_brightest = cols.index('z_brightest')
+        idx_z_mean = cols.index('z_mean')
+        idx_z_weighted = cols.index('z_weighted')
+        idx_z_stdev = cols.index('z_stdev')
+
         mergeData = np.zeros((N_rows, N_cols))
         for i, id in enumerate(unique_id):
 
-            this_group = data.query(f'group_id=={id}')
+            # Get the mask for this grouping
+            mask = data['group_id'] == id
+
+            # Get the data and fluxes for this grouping
+            this_group = data[mask]
+            these_fluxes = {b:fluxes[b][mask] for b in self.config.bands}
+
+            # Pull put some useful stuff
+            n_obj = len(this_group)
+            ref_fluxes = these_fluxes[self.config.ref_band]
+            these_redshifts = this_group[self.config.redshift_col]
 
             ## take the average position for the blended source
-            mergeData[i, cols.index(ra_label)] = this_group[ra_label].mean()
-            mergeData[i, cols.index(dec_label)] = this_group[dec_label].mean()
+            mergeData[i, idx_ra] = this_group[ra_label].mean()
+            mergeData[i, idx_dec] = this_group[dec_label].mean()
 
             ## sum up the fluxes into the blended source
             for b in self.config.bands:
-                  mergeData[i, cols.index(b)] = -2.5*np.log10(np.sum(10**(-this_group[b]/2.5)))
+                mergeData[i, cols.index(b)] = -2.5*np.log10(np.sum(these_fluxes[b])) + self.config.zp_dict[b]
+
+            brighest_idx = np.argmax(ref_fluxes)
+            redshifts = these_redshifts.iloc[brighest_idx]
+
+            mergeData[i, idx_redshift] = redshifts
+            mergeData[i, idx_n_obj] = n_obj
+            mergeData[i, idx_brightest_flux] = ref_fluxes.max()
+            mergeData[i, idx_total_flux] = np.sum(ref_fluxes)
+            mergeData[i, idx_z_brightest] = redshifts
+            mergeData[i, idx_z_mean] = np.mean(these_redshifts)
+            mergeData[i, idx_z_weighted] = np.sum(these_redshifts*ref_fluxes)/np.sum(ref_fluxes)
+            if n_obj > 1:
+                mergeData[i, idx_z_stdev] = np.std(these_redshifts)
+            else:
+                mergeData[i, idx_z_stdev] = 0.
 
         mergeData[:,cols.index('group_id')] = unique_id
         mergeData_df = pd.DataFrame(data=mergeData, columns=cols)
         mergeData_df['group_id'] = mergeData_df['group_id'].astype(int)
+        mergeData_df['n_obj'] = mergeData_df['n_obj'].astype(int)
 
         return mergeData_df
 
@@ -133,4 +182,3 @@ class UnrecBlModel(Degrader):
         # Return the new catalog and component index in original catalog
         self.add_data("output", blData)
         self.add_data("compInd", compInd)
-

rail/pipelines/degradation/apply_phot_errors.py

@@ -13,6 +13,7 @@ from rail.core.stage import RailStage, RailPipeline
 import ceci
 
 from rail.core.utils import RAILDIR
+from rail.utils import catalog_utils
 
 if 'PZ_DUSTMAP_DIR' not in os.environ:
     os.environ['PZ_DUSTMAP_DIR'] = '.'
@@ -62,6 +63,7 @@ class ApplyPhotErrorsPipeline(RailPipeline):
                 name=f'error_model_{key}',
                 connections=dict(input=previous_stage.io.output),
                 hdf5_groupname='',
+                renameDict=catalog_utils.CatalogConfigBase.active_class().band_name_dict(),
             )
             self.add_stage(the_error_model)
             previous_stage = the_error_model

rail/pipelines/degradation/truth_to_observed.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# Prerquisites, os, and numpy
+import os
+import numpy as np
+
+# Various rail modules
+from rail.tools.photometry_tools import Dereddener, Reddener
+
+from rail.core.stage import RailStage, RailPipeline
+
+import ceci
+
+from rail.core.utils import RAILDIR
+from rail.utils import catalog_utils
+from rail.creation.degraders.unrec_bl_model import UnrecBlModel
+
+from .spectroscopic_selection_pipeline import SELECTORS, CommonConfigParams
+from .apply_phot_errors import ERROR_MODELS
+
+
+if 'PZ_DUSTMAP_DIR' not in os.environ:  # pragma: no cover
+    os.environ['PZ_DUSTMAP_DIR'] = '.'
+
+dustmap_dir = os.path.expandvars("${PZ_DUSTMAP_DIR}")
+
+
+class TruthToObservedPipeline(RailPipeline):
+
+    default_input_dict = dict(input='dummy.in')
+
+    def __init__(self, error_models=None, selectors=None, blending=False):
+        RailPipeline.__init__(self)
+
+        DS = RailStage.data_store
+        DS.__class__.allow_overwrite = True
+
+        active_catalog_config = catalog_utils.CatalogConfigBase.active_class()
+        band_name_dict = active_catalog_config.band_name_dict()
+
+        if error_models is None:
+            error_models = ERROR_MODELS.copy()
+
+        if selectors is None:
+            selectors = SELECTORS.copy()
+
+        config_pars = CommonConfigParams.copy()
+        config_pars['colnames'] = band_name_dict.copy()
+        config_pars['colnames']['redshift'] = active_catalog_config.redshift_col
+
+        self.reddener = Reddener.build(
+            dustmap_dir=dustmap_dir,
+            copy_all_cols=True,
+        )
+        previous_stage = self.reddener
+
+        if blending:
+            self.unrec_bl = UnrecBlModel.build()
+            previous_stage = self.unrec_bl
+
+        for key, val in error_models.items():
+            error_model_class = ceci.PipelineStage.get_stage(val['ErrorModel'], val['Module'])
+            the_error_model = error_model_class.make_and_connect(
+                name=f'error_model_{key}',
+                connections=dict(input=previous_stage.io.output),
+                hdf5_groupname='',
+                renameDict=band_name_dict,
+            )
+            self.add_stage(the_error_model)
+            previous_stage = the_error_model
+
+            dereddener_errors = Dereddener.make_and_connect(
+                name=f"deredden_{key}",
+                dustmap_dir=dustmap_dir,
+                connections=dict(input=previous_stage.io.output),
+                copy_all_cols=True,
+            )
+            self.add_stage(dereddener_errors)
+            previous_stage = dereddener_errors
+            
+            for key2, val2 in selectors.items():
+                the_class = ceci.PipelineStage.get_stage(val2['Select'], val2['Module'])
+                the_selector = the_class.make_and_connect(
+                    name=f'select_{key}_{key2}',
+                    connections=dict(input=previous_stage.io.output),
+                    **config_pars,
+                )            
+                self.add_stage(the_selector)