pz-rail-astro-tools 1.0.3__tar.gz → 1.0.5__tar.gz

{pz_rail_astro_tools-1.0.3 → pz_rail_astro_tools-1.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pz-rail-astro-tools
-Version: 1.0.3
+Version: 1.0.5
 Author-email: "LSST Dark Energy Science Collaboration (DESC)" <lsst-desc-rail-admin@slac.stanford.edu>
 License: MIT License
         
@@ -43,6 +43,7 @@ Requires-Dist: photerr
 Requires-Dist: dustmaps
 Requires-Dist: pz-hyperbolic-temp
 Requires-Dist: lsstdesc-gcr-catalogs
+Requires-Dist: FoFCatalogMatching
 Provides-Extra: dev
 Requires-Dist: tables-io[full]; extra == "dev"
 Requires-Dist: pytest; extra == "dev"

{pz_rail_astro_tools-1.0.3 → pz_rail_astro_tools-1.0.5}/pyproject.toml

@@ -26,6 +26,7 @@ dependencies = [
     "dustmaps",
     "pz-hyperbolic-temp",
     "lsstdesc-gcr-catalogs",
+    "FoFCatalogMatching",
 ]
 
 # On a mac, install optional dependencies with `pip install '.[dev]'` (include the single quotes)

{pz_rail_astro_tools-1.0.3 → pz_rail_astro_tools-1.0.5}/src/pz_rail_astro_tools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pz-rail-astro-tools
-Version: 1.0.3
+Version: 1.0.5
 Author-email: "LSST Dark Energy Science Collaboration (DESC)" <lsst-desc-rail-admin@slac.stanford.edu>
 License: MIT License
         
@@ -43,6 +43,7 @@ Requires-Dist: photerr
 Requires-Dist: dustmaps
 Requires-Dist: pz-hyperbolic-temp
 Requires-Dist: lsstdesc-gcr-catalogs
+Requires-Dist: FoFCatalogMatching
 Provides-Extra: dev
 Requires-Dist: tables-io[full]; extra == "dev"
 Requires-Dist: pytest; extra == "dev"

{pz_rail_astro_tools-1.0.3 → pz_rail_astro_tools-1.0.5}/src/pz_rail_astro_tools.egg-info/SOURCES.txt

@@ -22,12 +22,13 @@ src/pz_rail_astro_tools.egg-info/requires.txt
 src/pz_rail_astro_tools.egg-info/top_level.txt
 src/rail/astro_tools/__init__.py
 src/rail/astro_tools/_version.py
+src/rail/creation/degraders/bl_demo.ipynb
 src/rail/creation/degraders/grid_selection.py
-src/rail/creation/degraders/lsst_error_model.py
 src/rail/creation/degraders/observing_condition_degrader.py
 src/rail/creation/degraders/photometric_errors.py
 src/rail/creation/degraders/spectroscopic_degraders.py
 src/rail/creation/degraders/spectroscopic_selections.py
+src/rail/creation/degraders/unrec_bl_model.py
 src/rail/creation/engines/gcr_engine.py
 src/rail/examples_data/creation_data/data/HSC_grid_settings.pkl
 src/rail/examples_data/creation_data/data/hsc_ratios_and_specz.hdf5
@@ -36,9 +37,13 @@ src/rail/examples_data/creation_data/data/survey_conditions/DC2-mask-neg-nside-1
 src/rail/examples_data/creation_data/data/survey_conditions/minion_1016_dc2_Median_airmass_i_and_nightlt1825_HEAL.fits
 src/rail/examples_data/creation_data/data/survey_conditions/minion_1016_dc2_Median_fiveSigmaDepth_i_and_nightlt1825_HEAL.fits
 src/rail/examples_data/testdata/rubin_dm_dc2_example2.pq
+src/rail/pipelines/degradation/apply_phot_errors.py
+src/rail/pipelines/degradation/blending.py
+src/rail/pipelines/degradation/spectroscopic_selection_pipeline.py
 src/rail/tools/photometry_tools.py
 tests/astro_tools/test_core.py
 tests/astro_tools/test_degraders.py
 tests/astro_tools/test_gcr_engine.py
+tests/astro_tools/test_pipline.py
 tests/astro_tools/gcr_test_data/schema.yaml
 tests/astro_tools/gcr_test_data/test_object_tract_4850.hdf5

{pz_rail_astro_tools-1.0.3 → pz_rail_astro_tools-1.0.5}/src/pz_rail_astro_tools.egg-info/requires.txt

@@ -8,6 +8,7 @@ photerr
 dustmaps
 pz-hyperbolic-temp
 lsstdesc-gcr-catalogs
+FoFCatalogMatching
 
 [dev]
 tables-io[full]

{pz_rail_astro_tools-1.0.3 → pz_rail_astro_tools-1.0.5}/src/rail/astro_tools/__init__.py

@@ -4,5 +4,6 @@ from rail.creation.degraders.grid_selection import *
 from rail.creation.degraders.observing_condition_degrader import *
 from rail.creation.degraders.spectroscopic_degraders import *
 from rail.creation.degraders.spectroscopic_selections import *
-from rail.creation.degraders.lsst_error_model import *
+from rail.creation.degraders.photometric_errors import *
+from rail.creation.degraders.unrec_bl_model import *
 from rail.tools.photometry_tools import *

{pz_rail_astro_tools-1.0.3 → pz_rail_astro_tools-1.0.5}/src/rail/astro_tools/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.0.3'
-__version_tuple__ = version_tuple = (1, 0, 3)
+__version__ = version = '1.0.5'
+__version_tuple__ = version_tuple = (1, 0, 5)

pz_rail_astro_tools-1.0.5/src/rail/creation/degraders/bl_demo.ipynb

@@ -0,0 +1,206 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "2610a0f0-0c71-4401-896f-734442bcd66d",
+   "metadata": {},
+   "source": [
+    "## Blending Degrader demo\n",
+    "\n",
+    "author: Shuang Liang\n",
+    "\n",
+    "This notebook demonstrate the use of `rail.creation.degradation.unrec_bl_model`, which uses Friends of Friends to finds sources close to each other and merge them into unrecognized blends"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f7a6adc3-68e8-4a1d-842f-bfb0960a1c4a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from rail.creation.degraders.unrec_bl_model import UnrecBlModel\n",
+    "\n",
+    "from rail.core.data import PqHandle\n",
+    "from rail.core.stage import RailStage\n",
+    "\n",
+    "import matplotlib.pyplot as plt\n",
+    "import pandas as pd, numpy as np"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6912a740-31ea-4987-b06d-81ff17cd895a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "DS = RailStage.data_store\n",
+    "DS.__class__.allow_overwrite = True\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a282c2ed-141b-4507-8254-dc8fbc9864dc",
+   "metadata": {},
+   "source": [
+    "### Create a random catalog with ugrizy+YJHF bands as the the true input"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1078bc2a-fc54-41c3-bd30-6c447bb971d4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data = np.random.normal(23, 3, size = (1000,12))\n",
+    "data[:, 0] = np.random.uniform(low=0, high=0.03, size=1000)\n",
+    "data[:, 1] = np.random.uniform(low=0, high=0.03, size=1000)\n",
+    "\n",
+    "data_df = pd.DataFrame(data=data,    # values\n",
+    "            columns=['ra', 'dec', 'u', 'g', 'r', 'i', 'z', 'y', 'Y', 'J', 'H', 'F'])\n",
+    "\n",
+    "data_truth = DS.add_data('input', data_df, PqHandle)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "33c99a4d-8375-4003-9a9a-70fa85a3eb82",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#data_df.to_parquet('bl_test.pq')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a5636721-a734-4746-bd93-8101bc558b6e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "plt.scatter(data_truth.data['ra'], data_truth.data['dec'], s=5)\n",
+    "plt.xlabel(\"Ra [Deg]\", fontsize=14)\n",
+    "plt.ylabel(\"Dec [Deg]\", fontsize=14)\n",
+    "plt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1da27deb-d167-4f38-8c59-f270184d6ab3",
+   "metadata": {},
+   "source": [
+    "### The blending model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a07f72a0-e24c-4844-90f0-d5a49ac4362b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## radius cut is in arcsecs\n",
+    "blModel = UnrecBlModel.make_stage(name='unrec_bl_model', ra_label='ra', dec_label='dec', linking_lengths=1.0, \\\n",
+    "                                  bands='ugrizyYJHF')\n",
+    "blModel.get_config_dict()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e5f4862a-0621-46d4-8901-7e84b461c424",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "samples_w_bl = blModel(data_truth)\n",
+    "samples_w_bl()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bc5158dd-f474-4731-b847-b4a7358656b9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fig, ax = plt.subplots(figsize=(6, 5), dpi=100)\n",
+    "\n",
+    "ax.scatter(data_truth.data['ra'],   data_truth.data['dec'],   s=10, facecolors='none', edgecolors='b', label='Original')\n",
+    "ax.scatter(samples_w_bl.data['ra'], samples_w_bl.data['dec'], s=5, c='r', label='w. Unrec-BL')\n",
+    "\n",
+    "ax.legend(loc=2, fontsize=12)\n",
+    "ax.set_xlabel(\"Ra [Deg]\", fontsize=14)\n",
+    "ax.set_ylabel(\"Dec [Deg]\", fontsize=14)\n",
+    "\n",
+    "plt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "268b3d37-b7fd-4ac1-8457-2104a87c9e6d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "b = 'r'\n",
+    "plt.hist(data_truth.data[b], bins=np.linspace(10, 30, 20), label='Original')\n",
+    "plt.hist(samples_w_bl.data[b], bins=np.linspace(10, 30, 20),  fill=False, label='w. Unrec-BL')\n",
+    "\n",
+    "plt.xlabel(fr'Magnitude ${b}$', fontsize=14)\n",
+    "plt.legend(fontsize=12)\n",
+    "plt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a1d51c15-1e04-4b22-9abb-9b267965dbeb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "flux = 10**(-(data_truth.data[b]-28.10)/2.5)       # r band zp for lsst is 28.10\n",
+    "flux_bl = 10**(-(samples_w_bl.data[b]-28.10)/2.5)\n",
+    "\n",
+    "plt.hist(flux, bins=np.linspace(0, 10000, 40), label='Original')\n",
+    "plt.hist(flux_bl, bins=np.linspace(0, 10000, 40), fill=False, label='w. Unrec-BL')\n",
+    "\n",
+    "plt.xlabel(fr'Flux ${b}$', fontsize=14)\n",
+    "plt.yscale('log')\n",
+    "plt.legend(fontsize=12)\n",
+    "plt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "da2b37c8-4667-4ad2-ad09-b1c68d2612b8",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

{pz_rail_astro_tools-1.0.3 → pz_rail_astro_tools-1.0.5}/src/rail/creation/degraders/photometric_errors.py

@@ -4,6 +4,7 @@ error models defined in the package photerr
 
 Author: John Franklin Crenshaw, Tianqing Zhang
 """
+import numpy as np
 from dataclasses import MISSING
 
 from ceci.config import StageParameter as Param
@@ -44,7 +45,7 @@ class PhotoErrorModel(Noisifier):
                 default = val.default_factory()
             else:
                 default = val.default
-
+                
             # Add this param to config_options
             self.config[key] = Param(
                 None,  # Let PhotErr handle type checking
@@ -52,6 +53,18 @@ class PhotoErrorModel(Noisifier):
                 msg="See the main docstring for details about this parameter.",
                 required=False,
             )
+
+    def reload_pars(self, args):
+        """ This is needed b/c the parameters are dynamically defined, 
+        so we have to reload them _after_ then have been defined """
+        copy_args = args.copy()
+        if isinstance(args, dict):
+            copy_args['config'] = args
+        else:  # pragma: no cover
+            copy_args['config'] = vars(args)            
+        self.load_configs(copy_args)
+        self._io_checked = False
+        self.check_io()
         
     def _initNoiseModel(self):
         """
@@ -66,12 +79,15 @@ class PhotoErrorModel(Noisifier):
         """
         Add noise to the input catalog
         """
-        
         # Load the input catalog
         data = self.get_data("input")
 
         # Add photometric errors
-        obsData = self.noiseModel(data, random_state=self.config.seed)
+        if self.config.seed is not None:  # pragma: no cover
+            seed = int(self.config.seed)
+        else:
+            seed = np.random.Generator(np.random.PCG64())
+        obsData = self.noiseModel(data, random_state=seed)
         
         # Return the new catalog
         self.add_data("output", obsData)
@@ -90,10 +106,9 @@ class LSSTErrorModel(PhotoErrorModel):
         super().__init__(args, **kwargs)
         
         self.set_params(peLsstErrorParams)   
+        self.reload_pars(args)
         self.peNoiseModel = peLsstErrorModel
         
-
-        
         
 class RomanErrorModel(PhotoErrorModel):
     
@@ -108,6 +123,7 @@ class RomanErrorModel(PhotoErrorModel):
         super().__init__(args, **kwargs)
         
         self.set_params(peRomanErrorParams)    
+        self.reload_pars(args)
         self.peNoiseModel = peRomanErrorModel
 
                 
@@ -125,4 +141,5 @@ class EuclidErrorModel(PhotoErrorModel):
         super().__init__(args, **kwargs)
         
         self.set_params(peEuclidErrorParams)    
+        self.reload_pars(args)
         self.peNoiseModel = peEuclidErrorModel

{pz_rail_astro_tools-1.0.3 → pz_rail_astro_tools-1.0.5}/src/rail/creation/degraders/spectroscopic_degraders.py

@@ -2,6 +2,7 @@
 
 import numpy as np
 import pandas as pd
+from ceci.config import StageParameter as Param
 from rail.creation.selector import Selector
 from rail.creation.noisifier import Noisifier
 
@@ -40,9 +41,11 @@ class LineConfusion(Noisifier):
 
     name = 'LineConfusion'
     config_options = Noisifier.config_options.copy()
-    config_options.update(true_wavelen=float,
-                          wrong_wavelen=float,
-                          frac_wrong=float)
+    config_options.update(
+        true_wavelen=Param(float, required=True, msg="wavelength of the true emission line"),
+        wrong_wavelen=Param(float, required=True, msg="wavelength of the wrong emission line"),
+        frac_wrong=Param(float, required=True, msg="fraction of galaxies with confused emission lines"),
+    )
 
     def __init__(self, args, **kwargs):
         """
@@ -113,7 +116,9 @@ class InvRedshiftIncompleteness(Selector):
 
     name = 'InvRedshiftIncompleteness'
     config_options = Selector.config_options.copy()
-    config_options.update(pivot_redshift=float)
+    config_options.update(
+        pivot_redshift=Param(float, required=True, msg="redshift at which the incompleteness begins"),
+    )    
 
     def __init__(self, args, **kwargs):
         """

{pz_rail_astro_tools-1.0.3 → pz_rail_astro_tools-1.0.5}/src/rail/creation/degraders/spectroscopic_selections.py

@@ -36,7 +36,7 @@ class SpecSelection(Selector):
         Random seed for reproducibility
     """
 
-    name = "specselection"
+    name = "SpecSelection"
     config_options = Selector.config_options.copy()
     config_options.update(
         N_tot=Param(int, 10000, msg="Number of selected sources"),
@@ -121,9 +121,9 @@ class SpecSelection(Selector):
             if band not in self.config.colnames.keys():
                 continue
             colname = self.config.colnames[band]
-            self.mask &= (np.abs(data[colname]) < nondetect_val) & (
-                ~np.isnan(data[colname])
-            )
+            self.mask &= data[colname] != nondetect_val
+            self.mask &= ~np.isnan(data[colname])
+            self.mask &= np.isfinite(data[colname])
 
     def downsampling_N_tot(self):
         """Randomly sample down the objects to a given number of data objects.
@@ -180,7 +180,7 @@ class SpecSelection_GAMA(SpecSelection):
     The necessary column is r band.
     """
     
-    name = "specselection_gama"
+    name = "SpecSelection_GAMA"
 
     def selection(self, data):
         """GAMA selection function. """
@@ -208,7 +208,7 @@ class SpecSelection_BOSS(SpecSelection):
     For BOSS selection, the data should at least include gri bands.
     """
 
-    name = "specselection_boss"
+    name = "SpecSelection_BOSS"
 
     def selection(self, data):
         """The BOSS selection function."""
@@ -268,7 +268,7 @@ class SpecSelection_DEEP2(SpecSelection):
     conversion degrader is ready, this subclass will be updated accordingly.
     """
 
-    name = "specselection_deep2"
+    name = "SpecSelection_DEEP2"
 
     def photometryCut(self, data):
         """Applies DEEP2 photometric cut based on Newman+13.
@@ -355,7 +355,7 @@ class SpecSelection_VVDSf02(SpecSelection):
     Necessary columns are i band magnitude and redshift.
     """
 
-    name = "specselection_VVDSf02"
+    name = "SpecSelection_VVDSf02"
 
     def photometryCut(self, data):
         """Photometric cut of VVDS 2h-field based on LeFèvre+05.
@@ -466,7 +466,7 @@ class SpecSelection_zCOSMOS(SpecSelection):
     For zCOSMOS, the data should at least include i band and redshift.
     """
 
-    name = "specselection_zCOSMOS"
+    name = "SpecSelection_zCOSMOS"
 
     def photometryCut(self, data):
         """Photometry cut for zCOSMOS based on Lilly+09.
@@ -527,7 +527,7 @@ class SpecSelection_HSC(SpecSelection):
     For HSC, the data should at least include giz bands and redshift.
     """
 
-    name = "specselection_HSC"
+    name = "SpecSelection_HSC"
 
     def photometryCut(self, data):
         """HSC galaxies were binned in color magnitude space with i-band mag
@@ -604,4 +604,4 @@ class SpecSelection_HSC(SpecSelection):
         # start message
         printMsg = "Applying the HSC selection."
 
-        return printMsg
+        return printMsg

pz_rail_astro_tools-1.0.5/src/rail/creation/degraders/unrec_bl_model.py

@@ -0,0 +1,98 @@
+"""Model for Creating Unrecognized Blends"""
+
+from ceci.config import StageParameter as Param
+from rail.creation.degrader import Degrader
+import numpy as np, pandas as pd
+import FoFCatalogMatching
+
+
+class UnrecBlModel(Degrader):
+    """Model for Creating Unrecognized Blends.
+
+    Finding objects nearby each other. Merge them into one blended
+    Use Friends of Friends for matching. May implement shape matching in the future.
+    Take avergaged Ra and Dec for blended source, and sum up fluxes in each band. May implement merged shapes in the future.
+
+    """
+    name = "UnrecBlModel"
+    config_options = Degrader.config_options.copy()
+    config_options.update(ra_label=Param(str, 'ra', msg='ra column name'),
+                          dec_label=Param(str, 'dec', msg='dec column name'),
+                          linking_lengths=Param(float, 1.0, msg='linking_lengths for FoF matching'),
+                          bands=Param(str, 'ugrizy', msg='name of filters'),
+                          match_size=Param(bool, False, msg='consider object size for finding blends'),
+                          match_shape=Param(bool, False, msg='consider object shape for finding blends'),
+                          obj_size=Param(str, 'obj_size', msg='object size column name'),
+                          a=Param(str, 'semi_major', msg='semi major axis column name'),
+                          b=Param(str, 'semi_minor', msg='semi minor axis column name'),
+                          theta=Param(str, 'orientation', msg='orientation angle column name'))
+
+    def __match_bl__(self, data):
+
+        """Group sources with friends of friends"""
+
+        ra_label, dec_label = self.config.ra_label, self.config.dec_label
+        linking_lengths = self.config.linking_lengths
+
+        results = FoFCatalogMatching.match({'truth': data}, linking_lengths=linking_lengths, ra_label=ra_label, dec_label=dec_label)
+        results.remove_column('catalog_key')
+
+        results = results.to_pandas(index='row_index')
+        results.sort_values(by='row_index', inplace=True)
+
+        ## adding the group id as the last column to data
+        matchData = pd.merge(data, results, left_index=True, right_index=True)
+
+        return matchData
+
+    def __merge_bl__(self, data):
+
+        """Merge sources within a group into unrecognized blends."""
+        
+        group_id = data['group_id']
+        unique_id = np.unique(group_id)
+
+        ra_label, dec_label = self.config.ra_label, self.config.dec_label
+
+        cols = list(data.columns)
+        ra_ind = cols.index(ra_label)
+        dec_ind = cols.index(dec_label)
+        bands_ind = {b:cols.index(b) for b in self.config.bands}
+
+        N_rows = len(unique_id)
+        N_cols = len(cols)
+
+        mergeData = np.zeros((N_rows, N_cols))
+        
+        for i, id in enumerate(unique_id):
+
+            this_group = data.query(f'group_id=={id}')
+
+            ## take the average position for the blended source
+            mergeData[i, ra_ind] = this_group[ra_label].mean()
+            mergeData[i, dec_ind] = this_group[dec_label].mean()
+
+            ## sum up the fluxes into the blended source
+            for b in self.config.bands:
+                  mergeData[i, bands_ind[b]] = -2.5*np.log10(np.sum(10**(-this_group[b]/2.5)))
+
+        mergeData[:,-1] = unique_id
+        mergeData_df = pd.DataFrame(data=mergeData, columns=cols)
+        mergeData_df['group_id'] = mergeData_df['group_id'].astype(int)
+
+        return mergeData_df
+
+    def run(self):
+        """Return pandas DataFrame with blending errors."""
+
+        # Load the input catalog
+        data = self.get_data("input")
+
+        # Match for close-by objects
+        matchData = self.__match_bl__(data)
+
+        # Merge matched objects into unrec-bl
+        blData = self.__merge_bl__(matchData)
+
+        # Return the new catalog
+        self.add_data("output", blData)

pz_rail_astro_tools-1.0.5/src/rail/pipelines/degradation/apply_phot_errors.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# Prerquisites, os, and numpy
+import os
+import numpy as np
+
+# Various rail modules
+from rail.tools.photometry_tools import Dereddener, Reddener
+
+from rail.core.stage import RailStage, RailPipeline
+
+import ceci
+
+from rail.core.utils import RAILDIR
+
+if 'PZ_DUSTMAP_DIR' not in os.environ:
+    os.environ['PZ_DUSTMAP_DIR'] = '.'
+
+dustmap_dir = os.path.expandvars("${PZ_DUSTMAP_DIR}")
+
+
+ERROR_MODELS = dict(
+    lsst = dict(
+        ErrorModel='LSSTErrorModel',
+        Module='rail.creation.degraders.photometric_errors',
+    ),
+    roman = dict(
+        ErrorModel='RomanErrorModel',
+        Module='rail.creation.degraders.photometric_errors',
+    ),
+    #euclid = dict(
+    #    ErrorModel='EuclidErrorModel',
+    #    Module='rail.creation.degraders.photometric_errors',
+    #),
+)
+
+
+
+class ApplyPhotErrorsPipeline(RailPipeline):
+
+    default_input_dict = dict(input='dummy.in')
+
+    def __init__(self, error_models=None):
+        RailPipeline.__init__(self)
+
+        DS = RailStage.data_store
+        DS.__class__.allow_overwrite = True
+
+        if error_models is None:
+            error_models = ERROR_MODELS
+
+        self.reddener = Reddener.build(
+            dustmap_dir=dustmap_dir,
+            copy_all_cols=True,
+        )
+
+        previous_stage = self.reddener
+        for key, val in error_models.items():
+            error_model_class = ceci.PipelineStage.get_stage(val['ErrorModel'], val['Module'])
+            the_error_model = error_model_class.make_and_connect(
+                name=f'error_model_{key}',
+                connections=dict(input=previous_stage.io.output),
+                hdf5_groupname='',
+            )
+            self.add_stage(the_error_model)
+            previous_stage = the_error_model
+
+        self.dereddener_errors = Dereddener.build(
+            dustmap_dir=dustmap_dir,
+            connections=dict(input=previous_stage.io.output),
+            copy_all_cols=True,
+        )

pz_rail_astro_tools-1.0.5/src/rail/pipelines/degradation/blending.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# Prerquisites, os, and numpy
+import os
+import numpy as np
+
+# Various rail modules
+from rail.creation.degraders.unrec_bl_model import UnrecBlModel
+
+from rail.core.stage import RailStage, RailPipeline
+
+import ceci
+
+from rail.core.utils import RAILDIR
+
+
+class BlendingPipeline(RailPipeline):
+
+    default_input_dict = dict(input='dummy.in')
+
+    def __init__(self):
+        RailPipeline.__init__(self)
+
+        DS = RailStage.data_store
+        DS.__class__.allow_overwrite = True
+
+        self.unrec_bl = UnrecBlModel.build()

pz_rail_astro_tools-1.0.5/src/rail/pipelines/degradation/spectroscopic_selection_pipeline.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# Prerquisites, os, and numpy
+import os
+import numpy as np
+
+# Various rail modules
+
+from rail.core.stage import RailStage, RailPipeline
+
+import ceci
+
+from rail.utils.catalog_utils import CatalogConfigBase
+from rail.core.utils import RAILDIR
+
+SELECTORS = dict(
+    GAMA = dict(
+        Select='SpecSelection_GAMA',
+        Module='rail.creation.degraders.spectroscopic_selections',
+    ),
+    BOSS = dict(
+        Select='SpecSelection_BOSS',
+        Module='rail.creation.degraders.spectroscopic_selections',
+    ),
+    VVDSf02 = dict(
+        Select='SpecSelection_VVDSf02',
+        Module='rail.creation.degraders.spectroscopic_selections',
+    ),
+    zCOSMOS = dict(
+        Select='SpecSelection_zCOSMOS',
+        Module='rail.creation.degraders.spectroscopic_selections',
+    ),
+    HSC = dict(
+        Select='SpecSelection_HSC',
+        Module='rail.creation.degraders.spectroscopic_selections',
+    ),
+)
+
+
+CommonConfigParams = dict(
+    N_tot = 100_000,
+    nondetect_val = -np.inf,
+    downsample= False,
+)
+
+
+class SpectroscopicSelectionPipeline(RailPipeline):
+
+    default_input_dict = dict(input='dummy.in')
+    
+    def __init__(self, selectors=None):
+        RailPipeline.__init__(self)
+
+        DS = RailStage.data_store
+        DS.__class__.allow_overwrite = True
+
+        if selectors is None:
+            selectors = SELECTORS.copy()
+
+        config_pars = CommonConfigParams.copy()
+        active_catalog = CatalogConfigBase.active_class()        
+        if active_catalog:
+            colnames = active_catalog.band_name_dict()
+            colnames['redshift'] = active_catalog.redshift_col
+            config_pars['colnames'] = colnames            
+
+        for key, val in selectors.items():
+            the_class = ceci.PipelineStage.get_stage(val['Select'], val['Module'])
+            the_selector = the_class.make_and_connect(
+                name=f'select_{key}',
+                **config_pars,
+            )
+            
+            self.add_stage(the_selector)

{pz_rail_astro_tools-1.0.3 → pz_rail_astro_tools-1.0.5}/src/rail/tools/photometry_tools.py

@@ -358,13 +358,15 @@ class LSSTFluxToMagConverter(RailStage):
     name = 'LSSTFluxToMagConverter'
 
     config_options = RailStage.config_options.copy()
-    config_options.update(bands='ugrizy')
-    config_options.update(flux_name="{band}_gaap1p0Flux")
-    config_options.update(flux_err_name="{band}_gaap1p0FluxErr")
-    config_options.update(mag_name="mag_{band}_lsst")
-    config_options.update(mag_err_name="mag_err_{band}_lsst")
-    config_options.update(copy_cols={})
-    config_options.update(mag_offset=31.4)
+    config_options.update(
+        bands=Param(str, default='ugrizy', msg="Names of the bands"),
+        flux_name=Param(str, default="{band}_gaap1p0Flux", msg="Template for band names"),
+        flux_err_name=Param(str, default="{band}_gaap1p0FluxErr", msg="Template for band error column names"),
+        mag_name=Param(str, default="mag_{band}_lsst", msg="Template for magnitude column names"),
+        mag_err_name=Param(str, default="mag_err_{band}_lsst", msg="Template for magnitude error column names"),
+        copy_cols=Param(dict, default={}, msg="Map of other columns to copy"),
+        mag_offset=Param(float, default=31.4, msg="Magntidue offset value"),
+    )
 
     mag_conv = np.log(10)*0.4
 
@@ -417,16 +419,17 @@ class DustMapBase(RailStage):
     name = 'DustMapBase'
 
     config_options = RailStage.config_options.copy()
-    config_options.update(bands='ugrizy')
-    config_options.update(ra_name='ra')
-    config_options.update(dec_name='dec')
-    config_options.update(mag_name="mag_{band}_lsst")
-    config_options.update(band_a_env=[4.81,3.64,2.70,2.06,1.58,1.31])
-    config_options.update(dustmap_name='sfd')
-    config_options.update(dustmap_dir=str)
-    config_options.update(copy_cols=[])
-    config_options.update(copy_all_cols=False)
-
+    config_options.update(
+        ra_name=Param(str, default='ra', msg="Name of the RA column"),
+        dec_name=Param(str, default='dec', msg="Name of the DEC column"),
+        mag_name=Param(str, default="mag_{band}_lsst", msg="Template for the magnitude columns"),
+        band_a_env=SHARED_PARAMS,
+        dustmap_name=Param(str, default='sfd', msg="Name of the dustmap in question"),
+        dustmap_dir=Param(str, required=True, msg="Directory with dustmaps"),
+        copy_cols=Param(list, default=[], msg="Additional columns to copy"),
+        copy_all_cols=Param(bool, default=False, msg="Copy all the columns"),
+    )
+        
     inputs = [('input', PqHandle)]
     outputs = [('output', PqHandle)]
 
@@ -453,7 +456,10 @@ class DustMapBase(RailStage):
     def run(self):
         data = self.get_data('input', allow_missing=True)
         out_data = {}
-        coords = SkyCoord(data[self.config.ra_name], data[self.config.dec_name], unit = 'deg',frame='fk5')
+        coords = SkyCoord(
+            np.array(data[self.config.ra_name]),
+            np.array(data[self.config.dec_name]),
+            unit = 'deg',frame='fk5')
         dust_map_dict = dict(sfd=dustmaps_sfd.SFDQuery)
         try:
             dust_map_class = dust_map_dict[self.config.dustmap_name]
@@ -464,10 +470,9 @@ class DustMapBase(RailStage):
             raise KeyError(f"Unknown dustmap {self.config.dustmap_name}, options are {list(dust_map_dict.keys())}") from msg
         ebvvec = dust_map(coords)
         band_mag_name_list=[]
-        for i, band_ in enumerate(self.config.bands):
-            band_mag_name = self.config.mag_name.format(band=band_)
+        for band_mag_name, a_env_value in self.config.band_a_env.items():
             mag_vals = data[band_mag_name]
-            out_data[band_mag_name] = self._calc_values(mag_vals, ebvvec, self.config.band_a_env[i])
+            out_data[band_mag_name] = self._calc_values(mag_vals, ebvvec, a_env_value)
             band_mag_name_list.append(band_mag_name)
        
         # check if copy_all_cols set to true:

{pz_rail_astro_tools-1.0.3 → pz_rail_astro_tools-1.0.5}/tests/astro_tools/test_degraders.py

@@ -13,7 +13,8 @@ from rail.creation.degraders.spectroscopic_degraders import InvRedshiftIncomplet
 from rail.creation.degraders.spectroscopic_selections import *
 from rail.creation.degraders.observing_condition_degrader import ObsCondition
 from rail.creation.degraders.grid_selection import GridSelection
-from rail.creation.degraders.lsst_error_model import LSSTErrorModel
+from rail.creation.degraders.photometric_errors import EuclidErrorModel, LSSTErrorModel
+from rail.creation.degraders.unrec_bl_model import UnrecBlModel
 
 
 @pytest.fixture
@@ -74,6 +75,29 @@ def data_with_radec():
     return DS.add_data("data_with_radec", df, TableHandle, path="dummy_with_radec.pd")
 
 
+@pytest.fixture
+def data_for_bl():
+    """Some dummy data to use below."""
+
+    DS = DATA_STORE()
+    DS.__class__.allow_overwrite = True
+
+    # generate random normal data
+    columns=['ra', 'dec', 'redshift', 'u', 'g', 'r', 'i', 'z', 'y']
+    rng = np.random.default_rng(0)
+    x = rng.normal(loc=23, scale=3, size=(1000, len(columns)))
+
+    # replace positions with constrained values
+    x[:, 0] = np.random.uniform(low=0, high=0.02, size=1000)
+    x[:, 1] = np.random.uniform(low=0, high=0.02, size=1000)
+
+    # replace redshifts with reasonable values
+    x[:, 2] = np.linspace(0, 2, x.shape[0])
+
+    # return data in handle wrapping a pandas DataFrame
+    df = pd.DataFrame(x, columns=columns)
+    return DS.add_data("data_for_bl", df, TableHandle, path="dummy_for_bl.pd")
+
 
 @pytest.mark.parametrize("pivot_redshift,errortype", [("fake", TypeError), (-1, ValueError)])
 def test_InvRedshiftIncompleteness_bad_params(pivot_redshift, errortype):
@@ -391,3 +415,20 @@ def test_LSSTErrorModel_returns_correct_columns(data):
         assert f"{band}_err" in degraded_data.columns
     os.remove(degrader.get_output(degrader.get_aliased_tag("output"), final_name=True))
 
+
+def test_EucliErrorModel(data):
+    # Setup the stage
+    degrader = EuclidErrorModel.make_stage()
+
+
+def test_BLModel(data_for_bl):
+    # Setup the stage
+    degrader = UnrecBlModel.make_stage()
+
+    # Apply the degrader and get the data out
+    degraded_data = degrader(data_for_bl).data
+
+    # Check output data has less rows than input data
+    assert degraded_data.shape[0] < data_for_bl.data.shape[0]
+
+    os.remove(degrader.get_output(degrader.get_aliased_tag("output"), final_name=True))

pz_rail_astro_tools-1.0.5/tests/astro_tools/test_pipline.py

@@ -0,0 +1,16 @@
+import os
+from rail.utils.testing_utils import build_and_read_pipeline
+
+import pytest
+
+@pytest.mark.parametrize(
+    "pipeline_class",
+    [
+        'rail.pipelines.degradation.apply_phot_errors.ApplyPhotErrorsPipeline',
+        'rail.pipelines.degradation.blending.BlendingPipeline',
+        'rail.pipelines.degradation.spectroscopic_selection_pipeline.SpectroscopicSelectionPipeline',
+    ]
+)
+def test_build_and_read_pipeline(pipeline_class):
+    build_and_read_pipeline(pipeline_class)
+

pz_rail_astro_tools-1.0.3/src/rail/creation/degraders/lsst_error_model.py

@@ -1,61 +0,0 @@
-"""The LSST Model for photometric errors."""
-from dataclasses import MISSING
-
-from ceci.config import StageParameter as Param
-from photerr import LsstErrorModel as PhotErrErrorModel
-from photerr import LsstErrorParams as PhotErrErrorParams
-from rail.creation.degrader import Degrader
-
-
-class LSSTErrorModel(Degrader):
-    """The LSST Model for photometric errors.
-
-    This is a wrapper around the error model from PhotErr. The parameter
-    docstring below is dynamically added by the installed version of PhotErr:
-
-    """
-
-    # Dynamically add the parameter docstring from PhotErr
-    __doc__ += PhotErrErrorParams.__doc__
-
-    name = "LSSTErrorModel"
-    config_options = Degrader.config_options.copy()
-
-    # Dynamically add all parameters from PhotErr
-    _photerr_params = PhotErrErrorParams.__dataclass_fields__
-    for key, val in _photerr_params.items():
-        # Get the default value
-        if val.default is MISSING:
-            default = val.default_factory()
-        else:
-            default = val.default
-
-        # Add this param to config_options
-        config_options[key] = Param(
-            None,  # Let PhotErr handle type checking
-            default,  
-            msg="See the main docstring for details about this parameter.",
-            required=False,
-        )
-
-    def __init__(self, args, **kwargs):
-        """
-        Constructor
-
-        Does standard Degrader initialization and sets up the error model.
-        """
-        super().__init__(args, **kwargs)
-        self.error_model = PhotErrErrorModel(
-            **{key: self.config[key] for key in self._photerr_params}
-        )
-
-    def run(self):
-        """Return pandas DataFrame with photometric errors."""
-        # Load the input catalog
-        data = self.get_data("input")
-
-        # Add photometric errors
-        obsData = self.error_model(data, random_state=self.config.seed)
-
-        # Return the new catalog
-        self.add_data("output", obsData)