pz-rail-astro-tools 1.0.6__tar.gz → 1.0.7__tar.gz

{pz_rail_astro_tools-1.0.6 → pz_rail_astro_tools-1.0.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pz-rail-astro-tools
-Version: 1.0.6
+Version: 1.0.7
 Author-email: "LSST Dark Energy Science Collaboration (DESC)" <lsst-desc-rail-admin@slac.stanford.edu>
 License: MIT License
         

{pz_rail_astro_tools-1.0.6 → pz_rail_astro_tools-1.0.7}/src/pz_rail_astro_tools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pz-rail-astro-tools
-Version: 1.0.6
+Version: 1.0.7
 Author-email: "LSST Dark Energy Science Collaboration (DESC)" <lsst-desc-rail-admin@slac.stanford.edu>
 License: MIT License
         

{pz_rail_astro_tools-1.0.6 → pz_rail_astro_tools-1.0.7}/src/pz_rail_astro_tools.egg-info/SOURCES.txt

@@ -39,6 +39,7 @@ src/rail/examples_data/testdata/rubin_dm_dc2_example2.pq
 src/rail/pipelines/degradation/apply_phot_errors.py
 src/rail/pipelines/degradation/blending.py
 src/rail/pipelines/degradation/spectroscopic_selection_pipeline.py
+src/rail/pipelines/degradation/truth_to_observed.py
 src/rail/tools/photometry_tools.py
 tests/astro_tools/test_core.py
 tests/astro_tools/test_degraders.py

{pz_rail_astro_tools-1.0.6 → pz_rail_astro_tools-1.0.7}/src/rail/astro_tools/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.0.6'
-__version_tuple__ = version_tuple = (1, 0, 6)
+__version__ = version = '1.0.7'
+__version_tuple__ = version_tuple = (1, 0, 7)

{pz_rail_astro_tools-1.0.6 → pz_rail_astro_tools-1.0.7}/src/rail/creation/degraders/unrec_bl_model.py

@@ -3,6 +3,7 @@
 from ceci.config import StageParameter as Param
 from rail.creation.degrader import Degrader
 from rail.core.data import PqHandle
+from rail.core.common_params import SHARED_PARAMS
 import numpy as np, pandas as pd
 import FoFCatalogMatching
 
@@ -20,7 +21,9 @@ class UnrecBlModel(Degrader):
     config_options.update(ra_label=Param(str, 'ra', msg='ra column name'),
                           dec_label=Param(str, 'dec', msg='dec column name'),
                           linking_lengths=Param(float, 1.0, msg='linking_lengths for FoF matching'),
-                          bands=Param(str, 'ugrizy', msg='name of filters'),
+                          bands=SHARED_PARAMS,
+                          ref_band=SHARED_PARAMS,
+                          redshift_col=SHARED_PARAMS,
                           match_size=Param(bool, False, msg='consider object size for finding blends'),
                           match_shape=Param(bool, False, msg='consider object shape for finding blends'),
                           obj_size=Param(str, 'obj_size', msg='object size column name'),
@@ -30,6 +33,8 @@ class UnrecBlModel(Degrader):
 
     outputs = [("output", PqHandle), ("compInd", PqHandle)]
 
+    blend_info_cols = ['group_id', 'n_obj', 'brightest_flux', 'total_flux', 'z_brightest', 'z_weighted', 'z_mean', 'z_stdev']
+
     def __call__(self, sample, seed: int = None):
         """The main interface method for ``Degrader``.
 
@@ -89,32 +94,70 @@ class UnrecBlModel(Degrader):
     def __merge_bl__(self, data):
 
         """Merge sources within a group into unrecognized blends."""
-        
+
         group_id = data['group_id']
         unique_id = np.unique(group_id)
 
         ra_label, dec_label = self.config.ra_label, self.config.dec_label
-        cols = [ra_label, dec_label] + [b for b in self.config.bands] + ['group_id']
+        cols = [ra_label, dec_label] + [b for b in self.config.bands] + self.blend_info_cols
 
         N_rows = len(unique_id)
         N_cols = len(cols)
 
+        # compute the fluxes once for all the galaxies
+        fluxes = {b:10**(-data[b]/2.5) for b in self.config.bands}
+
+        # pull the column indices
+        idx_ra = cols.index(ra_label)
+        idx_dec = cols.index(dec_label)
+        idx_n_obj = cols.index('n_obj')
+        idx_brightest_flux = cols.index('brightest_flux')
+        idx_total_flux = cols.index('total_flux')
+        idx_z_brightest = cols.index('z_brightest')
+        idx_z_mean = cols.index('z_mean')
+        idx_z_weighted = cols.index('z_weighted')
+        idx_z_stdev = cols.index('z_stdev')
+
         mergeData = np.zeros((N_rows, N_cols))
         for i, id in enumerate(unique_id):
 
-            this_group = data.query(f'group_id=={id}')
+            # Get the mask for this grouping
+            mask = data['group_id'] == id
+
+            # Get the data and fluxes for this grouping
+            this_group = data[mask]
+            these_fluxes = {b:fluxes[b][mask] for b in self.config.bands}
+
+            # Pull put some useful stuff
+            n_obj = len(this_group)
+            ref_fluxes = these_fluxes[self.config.ref_band]
+            these_redshifts = this_group[self.config.redshift_col]
 
             ## take the average position for the blended source
-            mergeData[i, cols.index(ra_label)] = this_group[ra_label].mean()
-            mergeData[i, cols.index(dec_label)] = this_group[dec_label].mean()
+            mergeData[i, idx_ra] = this_group[ra_label].mean()
+            mergeData[i, idx_dec] = this_group[dec_label].mean()
 
             ## sum up the fluxes into the blended source
             for b in self.config.bands:
-                  mergeData[i, cols.index(b)] = -2.5*np.log10(np.sum(10**(-this_group[b]/2.5)))
+                mergeData[i, cols.index(b)] = -2.5*np.log10(np.sum(these_fluxes[b]))
+
+            brighest_idx = np.argmax(ref_fluxes)
+
+            mergeData[i, idx_n_obj] = n_obj
+            mergeData[i, idx_brightest_flux] = ref_fluxes.max()
+            mergeData[i, idx_total_flux] = np.sum(ref_fluxes)
+            mergeData[i, idx_z_brightest] = these_redshifts.iloc[brighest_idx]
+            mergeData[i, idx_z_mean] = np.mean(these_redshifts)
+            mergeData[i, idx_z_weighted] = np.sum(these_redshifts*ref_fluxes)/np.sum(ref_fluxes)
+            if n_obj > 1:
+                mergeData[i, idx_z_stdev] = np.std(these_redshifts)
+            else:
+                mergeData[i, idx_z_stdev] = 0.
 
         mergeData[:,cols.index('group_id')] = unique_id
         mergeData_df = pd.DataFrame(data=mergeData, columns=cols)
         mergeData_df['group_id'] = mergeData_df['group_id'].astype(int)
+        mergeData_df['n_obj'] = mergeData_df['n_obj'].astype(int)
 
         return mergeData_df
 
@@ -133,4 +176,3 @@ class UnrecBlModel(Degrader):
         # Return the new catalog and component index in original catalog
         self.add_data("output", blData)
         self.add_data("compInd", compInd)
-

pz_rail_astro_tools-1.0.7/src/rail/pipelines/degradation/truth_to_observed.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# Prerquisites, os, and numpy
+import os
+import numpy as np
+
+# Various rail modules
+from rail.tools.photometry_tools import Dereddener, Reddener
+
+from rail.core.stage import RailStage, RailPipeline
+
+import ceci
+
+from rail.core.utils import RAILDIR
+
+from rail.creation.degraders.unrec_bl_model import UnrecBlModel
+
+from .spectroscopic_selection_pipeline import SELECTORS, CommonConfigParams
+from .apply_phot_errors import ERROR_MODELS
+
+
+if 'PZ_DUSTMAP_DIR' not in os.environ:  # pragma: no cover
+    os.environ['PZ_DUSTMAP_DIR'] = '.'
+
+dustmap_dir = os.path.expandvars("${PZ_DUSTMAP_DIR}")
+
+
+class TruthToObservedPipeline(RailPipeline):
+
+    default_input_dict = dict(input='dummy.in')
+
+    def __init__(self, error_models=None, selectors=None, blending=False):
+        RailPipeline.__init__(self)
+
+        DS = RailStage.data_store
+        DS.__class__.allow_overwrite = True
+
+        if error_models is None:
+            error_models = ERROR_MODELS.copy()
+
+        if selectors is None:
+            selectors = SELECTORS.copy()
+
+        config_pars = CommonConfigParams.copy()
+
+        self.reddener = Reddener.build(
+            dustmap_dir=dustmap_dir,
+            copy_all_cols=True,
+        )
+        previous_stage = self.reddener
+
+        if blending:
+            self.unrec_bl = UnrecBlModel.build()
+            previous_stage = self.unrec_bl
+
+        for key, val in error_models.items():
+            error_model_class = ceci.PipelineStage.get_stage(val['ErrorModel'], val['Module'])
+            the_error_model = error_model_class.make_and_connect(
+                name=f'error_model_{key}',
+                connections=dict(input=previous_stage.io.output),
+                hdf5_groupname='',
+            )
+            self.add_stage(the_error_model)
+            previous_stage = the_error_model
+
+            dereddener_errors = Dereddener.make_and_connect(
+                name=f"deredden_{key}",
+                dustmap_dir=dustmap_dir,
+                connections=dict(input=previous_stage.io.output),
+                copy_all_cols=True,
+            )
+            self.add_stage(dereddener_errors)
+            previous_stage = dereddener_errors
+            
+            for key2, val2 in selectors.items():
+                the_class = ceci.PipelineStage.get_stage(val2['Select'], val2['Module'])
+                the_selector = the_class.make_and_connect(
+                    name=f'select_{key}_{key2}',
+                    connections=dict(input=previous_stage.io.output),
+                    **config_pars,
+                )            
+                self.add_stage(the_selector)

{pz_rail_astro_tools-1.0.6 → pz_rail_astro_tools-1.0.7}/tests/astro_tools/test_degraders.py

@@ -83,13 +83,14 @@ def data_for_bl():
     DS.__class__.allow_overwrite = True
 
     # generate random normal data
-    columns=['ra', 'dec', 'u', 'g', 'r', 'i', 'z', 'y']
+    columns=['ra', 'dec', 'u', 'g', 'r', 'i', 'z', 'y', 'redshift']
     rng = np.random.default_rng(0)
     x = rng.normal(loc=23, scale=3, size=(1000, len(columns)))
 
     # replace positions with constrained values
     x[:, 0] = np.random.uniform(low=0, high=0.02, size=1000)
     x[:, 1] = np.random.uniform(low=0, high=0.02, size=1000)
+    x[:, 8] = np.random.uniform(low=0, high=3.00, size=1000)
 
     # return data in handle wrapping a pandas DataFrame
     df = pd.DataFrame(x, columns=columns)
@@ -421,7 +422,14 @@ def test_EucliErrorModel(data):
 def test_BLModel(data_for_bl):
     # Setup the stage
 
-    degrader = UnrecBlModel.make_stage(name='unrec_bl_model', ra_label='ra', dec_label='dec', linking_lengths=1.0, bands='ugrizy', seed=1234)
+    degrader = UnrecBlModel.make_stage(
+        name='unrec_bl_model',
+        ra_label='ra',
+        dec_label='dec',
+        linking_lengths=1.0,
+        bands='ugrizy',
+        ref_band='i',
+        seed=1234)
 
     # Apply the degrader and get the data out
     outputs = degrader(data_for_bl)

pz_rail_astro_tools-1.0.7/tests/astro_tools/test_pipline.py

@@ -0,0 +1,17 @@
+import os
+from rail.utils.testing_utils import build_and_read_pipeline
+
+import pytest
+
+@pytest.mark.parametrize(
+    "pipeline_class, options",
+    [
+        ('rail.pipelines.degradation.apply_phot_errors.ApplyPhotErrorsPipeline', {}),
+        ('rail.pipelines.degradation.blending.BlendingPipeline', {}),
+        ('rail.pipelines.degradation.spectroscopic_selection_pipeline.SpectroscopicSelectionPipeline', {}),
+        ('rail.pipelines.degradation.truth_to_observed.TruthToObservedPipeline', {'blending':True}),
+    ]
+)
+def test_build_and_read_pipeline(pipeline_class, options):
+    build_and_read_pipeline(pipeline_class, **options)
+

pz_rail_astro_tools-1.0.6/tests/astro_tools/test_pipline.py

@@ -1,16 +0,0 @@
-import os
-from rail.utils.testing_utils import build_and_read_pipeline
-
-import pytest
-
-@pytest.mark.parametrize(
-    "pipeline_class",
-    [
-        'rail.pipelines.degradation.apply_phot_errors.ApplyPhotErrorsPipeline',
-        'rail.pipelines.degradation.blending.BlendingPipeline',
-        'rail.pipelines.degradation.spectroscopic_selection_pipeline.SpectroscopicSelectionPipeline',
-    ]
-)
-def test_build_and_read_pipeline(pipeline_class):
-    build_and_read_pipeline(pipeline_class)
-