pz-rail-astro-tools 1.0.4__tar.gz → 1.0.6__tar.gz

{pz_rail_astro_tools-1.0.4 → pz_rail_astro_tools-1.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pz-rail-astro-tools
-Version: 1.0.4
+Version: 1.0.6
 Author-email: "LSST Dark Energy Science Collaboration (DESC)" <lsst-desc-rail-admin@slac.stanford.edu>
 License: MIT License
         
@@ -43,6 +43,7 @@ Requires-Dist: photerr
 Requires-Dist: dustmaps
 Requires-Dist: pz-hyperbolic-temp
 Requires-Dist: lsstdesc-gcr-catalogs
+Requires-Dist: FoFCatalogMatching
 Provides-Extra: dev
 Requires-Dist: tables-io[full]; extra == "dev"
 Requires-Dist: pytest; extra == "dev"

{pz_rail_astro_tools-1.0.4 → pz_rail_astro_tools-1.0.6}/pyproject.toml

@@ -26,6 +26,7 @@ dependencies = [
     "dustmaps",
     "pz-hyperbolic-temp",
     "lsstdesc-gcr-catalogs",
+    "FoFCatalogMatching",
 ]
 
 # On a mac, install optional dependencies with `pip install '.[dev]'` (include the single quotes)

{pz_rail_astro_tools-1.0.4 → pz_rail_astro_tools-1.0.6}/src/pz_rail_astro_tools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pz-rail-astro-tools
-Version: 1.0.4
+Version: 1.0.6
 Author-email: "LSST Dark Energy Science Collaboration (DESC)" <lsst-desc-rail-admin@slac.stanford.edu>
 License: MIT License
         
@@ -43,6 +43,7 @@ Requires-Dist: photerr
 Requires-Dist: dustmaps
 Requires-Dist: pz-hyperbolic-temp
 Requires-Dist: lsstdesc-gcr-catalogs
+Requires-Dist: FoFCatalogMatching
 Provides-Extra: dev
 Requires-Dist: tables-io[full]; extra == "dev"
 Requires-Dist: pytest; extra == "dev"

{pz_rail_astro_tools-1.0.4 → pz_rail_astro_tools-1.0.6}/src/pz_rail_astro_tools.egg-info/SOURCES.txt

@@ -27,6 +27,7 @@ src/rail/creation/degraders/observing_condition_degrader.py
 src/rail/creation/degraders/photometric_errors.py
 src/rail/creation/degraders/spectroscopic_degraders.py
 src/rail/creation/degraders/spectroscopic_selections.py
+src/rail/creation/degraders/unrec_bl_model.py
 src/rail/creation/engines/gcr_engine.py
 src/rail/examples_data/creation_data/data/HSC_grid_settings.pkl
 src/rail/examples_data/creation_data/data/hsc_ratios_and_specz.hdf5
@@ -35,8 +36,9 @@ src/rail/examples_data/creation_data/data/survey_conditions/DC2-mask-neg-nside-1
 src/rail/examples_data/creation_data/data/survey_conditions/minion_1016_dc2_Median_airmass_i_and_nightlt1825_HEAL.fits
 src/rail/examples_data/creation_data/data/survey_conditions/minion_1016_dc2_Median_fiveSigmaDepth_i_and_nightlt1825_HEAL.fits
 src/rail/examples_data/testdata/rubin_dm_dc2_example2.pq
-src/rail/pipelines/utility/apply_phot_errors.py
-src/rail/pipelines/utility/spectroscopic_selection_pipeline.py
+src/rail/pipelines/degradation/apply_phot_errors.py
+src/rail/pipelines/degradation/blending.py
+src/rail/pipelines/degradation/spectroscopic_selection_pipeline.py
 src/rail/tools/photometry_tools.py
 tests/astro_tools/test_core.py
 tests/astro_tools/test_degraders.py

{pz_rail_astro_tools-1.0.4 → pz_rail_astro_tools-1.0.6}/src/pz_rail_astro_tools.egg-info/requires.txt

@@ -8,6 +8,7 @@ photerr
 dustmaps
 pz-hyperbolic-temp
 lsstdesc-gcr-catalogs
+FoFCatalogMatching
 
 [dev]
 tables-io[full]

{pz_rail_astro_tools-1.0.4 → pz_rail_astro_tools-1.0.6}/src/rail/astro_tools/__init__.py

@@ -4,4 +4,6 @@ from rail.creation.degraders.grid_selection import *
 from rail.creation.degraders.observing_condition_degrader import *
 from rail.creation.degraders.spectroscopic_degraders import *
 from rail.creation.degraders.spectroscopic_selections import *
+from rail.creation.degraders.photometric_errors import *
+from rail.creation.degraders.unrec_bl_model import *
 from rail.tools.photometry_tools import *

{pz_rail_astro_tools-1.0.4 → pz_rail_astro_tools-1.0.6}/src/rail/astro_tools/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.0.4'
-__version_tuple__ = version_tuple = (1, 0, 4)
+__version__ = version = '1.0.6'
+__version_tuple__ = version_tuple = (1, 0, 6)

{pz_rail_astro_tools-1.0.4 → pz_rail_astro_tools-1.0.6}/src/rail/creation/degraders/photometric_errors.py

@@ -45,7 +45,7 @@ class PhotoErrorModel(Noisifier):
                 default = val.default_factory()
             else:
                 default = val.default
-
+                
             # Add this param to config_options
             self.config[key] = Param(
                 None,  # Let PhotErr handle type checking
@@ -53,6 +53,18 @@ class PhotoErrorModel(Noisifier):
                 msg="See the main docstring for details about this parameter.",
                 required=False,
             )
+
+    def reload_pars(self, args):
+        """ This is needed b/c the parameters are dynamically defined, 
+        so we have to reload them _after_ then have been defined """
+        copy_args = args.copy()
+        if isinstance(args, dict):
+            copy_args['config'] = args
+        else:  # pragma: no cover
+            copy_args['config'] = vars(args)            
+        self.load_configs(copy_args)
+        self._io_checked = False
+        self.check_io()
         
     def _initNoiseModel(self):
         """
@@ -67,7 +79,6 @@ class PhotoErrorModel(Noisifier):
         """
         Add noise to the input catalog
         """
-        
         # Load the input catalog
         data = self.get_data("input")
 
@@ -95,10 +106,9 @@ class LSSTErrorModel(PhotoErrorModel):
         super().__init__(args, **kwargs)
         
         self.set_params(peLsstErrorParams)   
+        self.reload_pars(args)
         self.peNoiseModel = peLsstErrorModel
         
-
-        
         
 class RomanErrorModel(PhotoErrorModel):
     
@@ -113,6 +123,7 @@ class RomanErrorModel(PhotoErrorModel):
         super().__init__(args, **kwargs)
         
         self.set_params(peRomanErrorParams)    
+        self.reload_pars(args)
         self.peNoiseModel = peRomanErrorModel
 
                 
@@ -130,4 +141,5 @@ class EuclidErrorModel(PhotoErrorModel):
         super().__init__(args, **kwargs)
         
         self.set_params(peEuclidErrorParams)    
+        self.reload_pars(args)
         self.peNoiseModel = peEuclidErrorModel

{pz_rail_astro_tools-1.0.4 → pz_rail_astro_tools-1.0.6}/src/rail/creation/degraders/spectroscopic_degraders.py

@@ -2,6 +2,7 @@
 
 import numpy as np
 import pandas as pd
+from ceci.config import StageParameter as Param
 from rail.creation.selector import Selector
 from rail.creation.noisifier import Noisifier
 
@@ -40,9 +41,11 @@ class LineConfusion(Noisifier):
 
     name = 'LineConfusion'
     config_options = Noisifier.config_options.copy()
-    config_options.update(true_wavelen=float,
-                          wrong_wavelen=float,
-                          frac_wrong=float)
+    config_options.update(
+        true_wavelen=Param(float, required=True, msg="wavelength of the true emission line"),
+        wrong_wavelen=Param(float, required=True, msg="wavelength of the wrong emission line"),
+        frac_wrong=Param(float, required=True, msg="fraction of galaxies with confused emission lines"),
+    )
 
     def __init__(self, args, **kwargs):
         """
@@ -113,7 +116,9 @@ class InvRedshiftIncompleteness(Selector):
 
     name = 'InvRedshiftIncompleteness'
     config_options = Selector.config_options.copy()
-    config_options.update(pivot_redshift=float)
+    config_options.update(
+        pivot_redshift=Param(float, required=True, msg="redshift at which the incompleteness begins"),
+    )    
 
     def __init__(self, args, **kwargs):
         """

pz_rail_astro_tools-1.0.6/src/rail/creation/degraders/unrec_bl_model.py

@@ -0,0 +1,136 @@
+"""Model for Creating Unrecognized Blends"""
+
+from ceci.config import StageParameter as Param
+from rail.creation.degrader import Degrader
+from rail.core.data import PqHandle
+import numpy as np, pandas as pd
+import FoFCatalogMatching
+
+
+class UnrecBlModel(Degrader):
+    """Model for Creating Unrecognized Blends.
+
+    Finding objects nearby each other. Merge them into one blended
+    Use Friends of Friends for matching. May implement shape matching in the future.
+    Take avergaged Ra and Dec for blended source, and sum up fluxes in each band. May implement merged shapes in the future.
+
+    """
+    name = "UnrecBlModel"
+    config_options = Degrader.config_options.copy()
+    config_options.update(ra_label=Param(str, 'ra', msg='ra column name'),
+                          dec_label=Param(str, 'dec', msg='dec column name'),
+                          linking_lengths=Param(float, 1.0, msg='linking_lengths for FoF matching'),
+                          bands=Param(str, 'ugrizy', msg='name of filters'),
+                          match_size=Param(bool, False, msg='consider object size for finding blends'),
+                          match_shape=Param(bool, False, msg='consider object shape for finding blends'),
+                          obj_size=Param(str, 'obj_size', msg='object size column name'),
+                          a=Param(str, 'semi_major', msg='semi major axis column name'),
+                          b=Param(str, 'semi_minor', msg='semi minor axis column name'),
+                          theta=Param(str, 'orientation', msg='orientation angle column name'))
+
+    outputs = [("output", PqHandle), ("compInd", PqHandle)]
+
+    def __call__(self, sample, seed: int = None):
+        """The main interface method for ``Degrader``.
+
+        Applies degradation.
+
+        This will attach the sample to this `Degrader` (for introspection and
+        provenance tracking).
+
+        Then it will call the run() and finalize() methods, which need to be
+        implemented by the sub-classes.
+
+        The run() method will need to register the data that it creates to this
+        Estimator by using ``self.add_data('output', output_data)``.
+
+        Finally, this will return a PqHandle providing access to that output
+        data.
+
+        Parameters
+        ----------
+        sample : table-like
+            The sample to be degraded
+        seed : int, default=None
+            An integer to set the numpy random seed
+
+        Returns
+        -------
+        output_data : PqHandle
+            A handle giving access to a table with degraded sample
+        """
+        if seed is not None:
+            self.config.seed = seed
+
+        self.set_data("input", sample)
+        self.run()
+        self.finalize()
+
+        return {'output':self.get_handle("output"), 'compInd':self.get_handle("compInd")}
+
+    def __match_bl__(self, data):
+
+        """Group sources with friends of friends"""
+
+        ra_label, dec_label = self.config.ra_label, self.config.dec_label
+        linking_lengths = self.config.linking_lengths
+
+        results = FoFCatalogMatching.match({'truth': data}, linking_lengths=linking_lengths, ra_label=ra_label, dec_label=dec_label)
+        results.remove_column('catalog_key')
+
+        results = results.to_pandas(index='row_index')
+        results.sort_values(by='row_index', inplace=True)
+
+        ## adding the group id as the last column to data
+        matchData = pd.merge(data, results, left_index=True, right_index=True)
+
+        return matchData, results
+
+    def __merge_bl__(self, data):
+
+        """Merge sources within a group into unrecognized blends."""
+        
+        group_id = data['group_id']
+        unique_id = np.unique(group_id)
+
+        ra_label, dec_label = self.config.ra_label, self.config.dec_label
+        cols = [ra_label, dec_label] + [b for b in self.config.bands] + ['group_id']
+
+        N_rows = len(unique_id)
+        N_cols = len(cols)
+
+        mergeData = np.zeros((N_rows, N_cols))
+        for i, id in enumerate(unique_id):
+
+            this_group = data.query(f'group_id=={id}')
+
+            ## take the average position for the blended source
+            mergeData[i, cols.index(ra_label)] = this_group[ra_label].mean()
+            mergeData[i, cols.index(dec_label)] = this_group[dec_label].mean()
+
+            ## sum up the fluxes into the blended source
+            for b in self.config.bands:
+                  mergeData[i, cols.index(b)] = -2.5*np.log10(np.sum(10**(-this_group[b]/2.5)))
+
+        mergeData[:,cols.index('group_id')] = unique_id
+        mergeData_df = pd.DataFrame(data=mergeData, columns=cols)
+        mergeData_df['group_id'] = mergeData_df['group_id'].astype(int)
+
+        return mergeData_df
+
+    def run(self):
+        """Return pandas DataFrame with blending errors."""
+
+        # Load the input catalog
+        data = self.get_data("input")
+
+        # Match for close-by objects
+        matchData, compInd = self.__match_bl__(data)
+
+        # Merge matched objects into unrec-bl
+        blData = self.__merge_bl__(matchData)
+
+        # Return the new catalog and component index in original catalog
+        self.add_data("output", blData)
+        self.add_data("compInd", compInd)
+

{pz_rail_astro_tools-1.0.4/src/rail/pipelines/utility → pz_rail_astro_tools-1.0.6/src/rail/pipelines/degradation}/apply_phot_errors.py

@@ -52,6 +52,7 @@ class ApplyPhotErrorsPipeline(RailPipeline):
 
         self.reddener = Reddener.build(
             dustmap_dir=dustmap_dir,
+            copy_all_cols=True,
         )
 
         previous_stage = self.reddener
@@ -68,4 +69,5 @@ class ApplyPhotErrorsPipeline(RailPipeline):
         self.dereddener_errors = Dereddener.build(
             dustmap_dir=dustmap_dir,
             connections=dict(input=previous_stage.io.output),
+            copy_all_cols=True,
         )

pz_rail_astro_tools-1.0.6/src/rail/pipelines/degradation/blending.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# Prerquisites, os, and numpy
+import os
+import numpy as np
+
+# Various rail modules
+from rail.creation.degraders.unrec_bl_model import UnrecBlModel
+
+from rail.core.stage import RailStage, RailPipeline
+
+import ceci
+
+from rail.core.utils import RAILDIR
+
+
+class BlendingPipeline(RailPipeline):
+
+    default_input_dict = dict(input='dummy.in')
+
+    def __init__(self):
+        RailPipeline.__init__(self)
+
+        DS = RailStage.data_store
+        DS.__class__.allow_overwrite = True
+
+        self.unrec_bl = UnrecBlModel.build()

{pz_rail_astro_tools-1.0.4/src/rail/pipelines/utility → pz_rail_astro_tools-1.0.6/src/rail/pipelines/degradation}/spectroscopic_selection_pipeline.py

@@ -16,23 +16,23 @@ from rail.core.utils import RAILDIR
 
 SELECTORS = dict(
     GAMA = dict(
-        Selector='SpecSelection_GAMA',
+        Select='SpecSelection_GAMA',
         Module='rail.creation.degraders.spectroscopic_selections',
     ),
     BOSS = dict(
-        Selector='SpecSelection_BOSS',
+        Select='SpecSelection_BOSS',
         Module='rail.creation.degraders.spectroscopic_selections',
     ),
     VVDSf02 = dict(
-        Selector='SpecSelection_VVDSf02',
+        Select='SpecSelection_VVDSf02',
         Module='rail.creation.degraders.spectroscopic_selections',
     ),
     zCOSMOS = dict(
-        Selector='SpecSelection_zCOSMOS',
+        Select='SpecSelection_zCOSMOS',
         Module='rail.creation.degraders.spectroscopic_selections',
     ),
     HSC = dict(
-        Selector='SpecSelection_HSC',
+        Select='SpecSelection_HSC',
         Module='rail.creation.degraders.spectroscopic_selections',
     ),
 )
@@ -66,7 +66,7 @@ class SpectroscopicSelectionPipeline(RailPipeline):
             config_pars['colnames'] = colnames            
 
         for key, val in selectors.items():
-            the_class = ceci.PipelineStage.get_stage(val['Selector'], val['Module'])
+            the_class = ceci.PipelineStage.get_stage(val['Select'], val['Module'])
             the_selector = the_class.make_and_connect(
                 name=f'select_{key}',
                 **config_pars,

{pz_rail_astro_tools-1.0.4 → pz_rail_astro_tools-1.0.6}/src/rail/tools/photometry_tools.py

@@ -358,13 +358,15 @@ class LSSTFluxToMagConverter(RailStage):
     name = 'LSSTFluxToMagConverter'
 
     config_options = RailStage.config_options.copy()
-    config_options.update(bands='ugrizy')
-    config_options.update(flux_name="{band}_gaap1p0Flux")
-    config_options.update(flux_err_name="{band}_gaap1p0FluxErr")
-    config_options.update(mag_name="mag_{band}_lsst")
-    config_options.update(mag_err_name="mag_err_{band}_lsst")
-    config_options.update(copy_cols={})
-    config_options.update(mag_offset=31.4)
+    config_options.update(
+        bands=Param(str, default='ugrizy', msg="Names of the bands"),
+        flux_name=Param(str, default="{band}_gaap1p0Flux", msg="Template for band names"),
+        flux_err_name=Param(str, default="{band}_gaap1p0FluxErr", msg="Template for band error column names"),
+        mag_name=Param(str, default="mag_{band}_lsst", msg="Template for magnitude column names"),
+        mag_err_name=Param(str, default="mag_err_{band}_lsst", msg="Template for magnitude error column names"),
+        copy_cols=Param(dict, default={}, msg="Map of other columns to copy"),
+        mag_offset=Param(float, default=31.4, msg="Magntidue offset value"),
+    )
 
     mag_conv = np.log(10)*0.4
 
@@ -417,16 +419,17 @@ class DustMapBase(RailStage):
     name = 'DustMapBase'
 
     config_options = RailStage.config_options.copy()
-    config_options.update(bands='ugrizy')
-    config_options.update(ra_name='ra')
-    config_options.update(dec_name='dec')
-    config_options.update(mag_name="mag_{band}_lsst")
-    config_options.update(band_a_env=[4.81,3.64,2.70,2.06,1.58,1.31])
-    config_options.update(dustmap_name='sfd')
-    config_options.update(dustmap_dir=str)
-    config_options.update(copy_cols=[])
-    config_options.update(copy_all_cols=False)
-
+    config_options.update(
+        ra_name=Param(str, default='ra', msg="Name of the RA column"),
+        dec_name=Param(str, default='dec', msg="Name of the DEC column"),
+        mag_name=Param(str, default="mag_{band}_lsst", msg="Template for the magnitude columns"),
+        band_a_env=SHARED_PARAMS,
+        dustmap_name=Param(str, default='sfd', msg="Name of the dustmap in question"),
+        dustmap_dir=Param(str, required=True, msg="Directory with dustmaps"),
+        copy_cols=Param(list, default=[], msg="Additional columns to copy"),
+        copy_all_cols=Param(bool, default=False, msg="Copy all the columns"),
+    )
+        
     inputs = [('input', PqHandle)]
     outputs = [('output', PqHandle)]
 
@@ -467,10 +470,9 @@ class DustMapBase(RailStage):
             raise KeyError(f"Unknown dustmap {self.config.dustmap_name}, options are {list(dust_map_dict.keys())}") from msg
         ebvvec = dust_map(coords)
         band_mag_name_list=[]
-        for i, band_ in enumerate(self.config.bands):
-            band_mag_name = self.config.mag_name.format(band=band_)
+        for band_mag_name, a_env_value in self.config.band_a_env.items():
             mag_vals = data[band_mag_name]
-            out_data[band_mag_name] = self._calc_values(mag_vals, ebvvec, self.config.band_a_env[i])
+            out_data[band_mag_name] = self._calc_values(mag_vals, ebvvec, a_env_value)
             band_mag_name_list.append(band_mag_name)
        
         # check if copy_all_cols set to true:

{pz_rail_astro_tools-1.0.4 → pz_rail_astro_tools-1.0.6}/tests/astro_tools/test_degraders.py

@@ -14,6 +14,7 @@ from rail.creation.degraders.spectroscopic_selections import *
 from rail.creation.degraders.observing_condition_degrader import ObsCondition
 from rail.creation.degraders.grid_selection import GridSelection
 from rail.creation.degraders.photometric_errors import EuclidErrorModel, LSSTErrorModel
+from rail.creation.degraders.unrec_bl_model import UnrecBlModel
 
 
 @pytest.fixture
@@ -74,6 +75,26 @@ def data_with_radec():
     return DS.add_data("data_with_radec", df, TableHandle, path="dummy_with_radec.pd")
 
 
+@pytest.fixture
+def data_for_bl():
+    """Some dummy data to use below."""
+
+    DS = DATA_STORE()
+    DS.__class__.allow_overwrite = True
+
+    # generate random normal data
+    columns=['ra', 'dec', 'u', 'g', 'r', 'i', 'z', 'y']
+    rng = np.random.default_rng(0)
+    x = rng.normal(loc=23, scale=3, size=(1000, len(columns)))
+
+    # replace positions with constrained values
+    x[:, 0] = np.random.uniform(low=0, high=0.02, size=1000)
+    x[:, 1] = np.random.uniform(low=0, high=0.02, size=1000)
+
+    # return data in handle wrapping a pandas DataFrame
+    df = pd.DataFrame(x, columns=columns)
+    return DS.add_data("data_for_bl", df, TableHandle, path="dummy_for_bl.pd")
+
 
 @pytest.mark.parametrize("pivot_redshift,errortype", [("fake", TypeError), (-1, ValueError)])
 def test_InvRedshiftIncompleteness_bad_params(pivot_redshift, errortype):
@@ -396,4 +417,22 @@ def test_EucliErrorModel(data):
     # Setup the stage
     degrader = EuclidErrorModel.make_stage()
 
-    
+
+def test_BLModel(data_for_bl):
+    # Setup the stage
+
+    degrader = UnrecBlModel.make_stage(name='unrec_bl_model', ra_label='ra', dec_label='dec', linking_lengths=1.0, bands='ugrizy', seed=1234)
+
+    # Apply the degrader and get the data out
+    outputs = degrader(data_for_bl)
+    degraded_data = outputs['output'].data
+    truth_components = outputs['compInd'].data
+
+    # Check output data has less rows than input data
+    assert degraded_data.shape[0] < data_for_bl.data.shape[0]
+
+    # Check components has the same rows as input data
+    assert truth_components.shape[0] == data_for_bl.data.shape[0]
+
+    os.remove(degrader.get_output(degrader.get_aliased_tag("output"), final_name=True))
+    os.remove(degrader.get_output(degrader.get_aliased_tag("compInd"), final_name=True))

pz_rail_astro_tools-1.0.6/tests/astro_tools/test_pipline.py

@@ -0,0 +1,16 @@
+import os
+from rail.utils.testing_utils import build_and_read_pipeline
+
+import pytest
+
+@pytest.mark.parametrize(
+    "pipeline_class",
+    [
+        'rail.pipelines.degradation.apply_phot_errors.ApplyPhotErrorsPipeline',
+        'rail.pipelines.degradation.blending.BlendingPipeline',
+        'rail.pipelines.degradation.spectroscopic_selection_pipeline.SpectroscopicSelectionPipeline',
+    ]
+)
+def test_build_and_read_pipeline(pipeline_class):
+    build_and_read_pipeline(pipeline_class)
+

pz_rail_astro_tools-1.0.4/tests/astro_tools/test_pipline.py

@@ -1,15 +0,0 @@
-import os
-from rail.utils.testing_utils import build_and_read_pipeline
-
-import pytest
-
-@pytest.mark.parametrize(
-    "pipeline_class",
-    [
-        'rail.pipelines.utility.apply_phot_errors.ApplyPhotErrorsPipeline',
-        'rail.pipelines.utility.spectroscopic_selection_pipeline.SpectroscopicSelectionPipeline',
-    ]
-)
-def test_build_and_read_pipeline(pipeline_class):
-    build_and_read_pipeline(pipeline_class)
-