pz-rail-astro-tools 0.1.2__py3-none-any.whl → 0.1.3__py3-none-any.whl

{pz_rail_astro_tools-0.1.2.dist-info → pz_rail_astro_tools-0.1.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pz-rail-astro-tools
-Version: 0.1.2
+Version: 0.1.3
 Author-email: "LSST Dark Energy Science Collaboration (DESC)" <lsst-desc-rail-admin@slac.stanford.edu>
 License: MIT License
         

{pz_rail_astro_tools-0.1.2.dist-info → pz_rail_astro_tools-0.1.3.dist-info}/RECORD

@@ -1,10 +1,12 @@
 rail/astro_tools/__init__.py,sha256=CwIb_-fmJ0DTHnnxVNTwXYvof69yPcBxp6RiHGx6f4M,386
-rail/astro_tools/_version.py,sha256=SFCDdrYA67D1Je-jHgVVh4LOopkPvuV6NMtqSJ7Tfhg,411
-rail/creation/degradation/grid_selection.py,sha256=aJ8tJsR9DiqgEhDEUKTo-O0T1uQXlN119yji8tT9dYU,11974
+rail/astro_tools/_version.py,sha256=L5DCMp1QAlSqy-8bW7d51bLubTxNjZGYc5fMQkb752U,411
+rail/creation/degradation/grid_selection.py,sha256=zUT75TuFLUzyPg9loSkGEO1HM35a5OtpbXRYSk1W9w8,12432
 rail/creation/degradation/lsst_error_model.py,sha256=WWQxPO3w1S6sALQxl7x7_mUI1f3o1eOGpBk3Yzi-1Q0,1969
-rail/creation/degradation/observing_condition_degrader.py,sha256=9aJe2vlDhO2exL2UTfxN5h_FrLp-q5wXtv32g0MSuys,19794
-rail/creation/degradation/spectroscopic_degraders.py,sha256=-Qko0oFVjPSnC5RKIJ7InMsVJkPAUnt3lRnuhTFWTzI,4935
-rail/creation/degradation/spectroscopic_selections.py,sha256=f-87GGSDIORaupuU-7TZG_IgNjTIWQS1hx6BbTz-7Yc,22251
+rail/creation/degradation/observing_condition_degrader.py,sha256=jelIfFHzrgmRmRARm5uC36ZohepfMHq5djWT5wGEaZI,19987
+rail/creation/degradation/photerr_demo.ipynb,sha256=ZT_CX2v88Qqxq8OS8MEk2BT8qiZRXJ2xNB2EDYrf9cA,6650
+rail/creation/degradation/photometric_errors.py,sha256=HbQaEKHEcSo4ab2J3qZAiVFxlJpLAQFwjGKUnSUmG8U,3927
+rail/creation/degradation/spectroscopic_degraders.py,sha256=zpCxK0jbKTKlU-eqe05pW-BM2O743_ocEQy2TQPP3EA,5038
+rail/creation/degradation/spectroscopic_selections.py,sha256=nzn8KSOEsQRX-FORu0VEUa303TpR6i4VvGf9vb3ip8Q,22176
 rail/creation/engines/gcr_engine.py,sha256=MTzbV-FDRAoWfTpUzKCrdyOGK3hsSJ5atSLpwslBE5Y,2989
 rail/examples_data/creation_data/data/HSC_grid_settings.pkl,sha256=zpJoSR9TSnDyL64DPXivIMam3fn6GOZoyoJoeBj2LAk,141
 rail/examples_data/creation_data/data/hsc_ratios_and_specz.hdf5,sha256=liH54wuuuHxToqjoAwQ_fFmEefk3haNweVThMwdDKN4,13823712
@@ -12,9 +14,9 @@ rail/examples_data/creation_data/data/survey_conditions/DC2-dr6-galcounts-i20-i2
 rail/examples_data/creation_data/data/survey_conditions/DC2-mask-neg-nside-128.fits,sha256=79Kg4xRP4WHlOCcS-3SKqf9aP_LZRAq-izIM0DFL6bw,1581120
 rail/examples_data/creation_data/data/survey_conditions/minion_1016_dc2_Median_airmass_i_and_nightlt1825_HEAL.fits,sha256=twvyce70G3h1MQnGGFQJG9wZc8UbuNqXc6mjtiuaL1s,1581120
 rail/examples_data/creation_data/data/survey_conditions/minion_1016_dc2_Median_fiveSigmaDepth_i_and_nightlt1825_HEAL.fits,sha256=RbMMFSVJQxGhZnL4BnISE9VTNljfTZwV6Bax6NxverY,1581120
-rail/tools/util_photometry.py,sha256=8EyuOMrJ8ResIMWJZlPooEOkl110ZKM2DMclqF4OO0E,18365
-pz_rail_astro_tools-0.1.2.dist-info/LICENSE,sha256=tLMEN21HbzvT-7umOVVvPjaJZbQxCa-2bAeYLwG5Q04,1102
-pz_rail_astro_tools-0.1.2.dist-info/METADATA,sha256=nRvcFb0CjKs6LVYSqA8L1zl0vxRhSt-9sBbDXASwOzU,3946
-pz_rail_astro_tools-0.1.2.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
-pz_rail_astro_tools-0.1.2.dist-info/top_level.txt,sha256=6R6sqn-85I8YZTzmB9gv7sqckB2QaOTHbu-QLGTWBRE,5
-pz_rail_astro_tools-0.1.2.dist-info/RECORD,,
+rail/tools/util_photometry.py,sha256=canY8t7rxX7ng_PfPEtfCsF1SCnv-BwGBRSGel97aPw,18425
+pz_rail_astro_tools-0.1.3.dist-info/LICENSE,sha256=tLMEN21HbzvT-7umOVVvPjaJZbQxCa-2bAeYLwG5Q04,1102
+pz_rail_astro_tools-0.1.3.dist-info/METADATA,sha256=1LT110NObmr7eEDKUiznHSuAtrP6pzQsobBTKedFY0M,3946
+pz_rail_astro_tools-0.1.3.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+pz_rail_astro_tools-0.1.3.dist-info/top_level.txt,sha256=6R6sqn-85I8YZTzmB9gv7sqckB2QaOTHbu-QLGTWBRE,5
+pz_rail_astro_tools-0.1.3.dist-info/RECORD,,

{pz_rail_astro_tools-0.1.2.dist-info → pz_rail_astro_tools-0.1.3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.3)
+Generator: bdist_wheel (0.43.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

rail/astro_tools/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.1.2'
-__version_tuple__ = version_tuple = (0, 1, 2)
+__version__ = version = '0.1.3'
+__version_tuple__ = version_tuple = (0, 1, 3)

rail/creation/degradation/grid_selection.py

@@ -1,16 +1,16 @@
-"""Degraders that emulate spectroscopic effects on photometry"""
+"""Selector that emulate spectroscopic effects on photometry"""
 
 import os
 import numpy as np
 import pandas as pd
 import pickle
 import tables_io
-from rail.creation.degrader import Degrader
+from rail.creation.selector import Selector
 from rail.core.utils import find_rail_file
 from ceci.config import StageParameter as Param
 
 
-class GridSelection(Degrader):
+class GridSelection(Selector):
     """
     Uses the ratio of HSC spectroscpic galaxies to photometric galaxies to portion a sample
     into training and application samples. Option to implement a color-based redshift cut off in each pixel.
@@ -39,12 +39,14 @@ class GridSelection(Degrader):
     Second HSC Data Release, details of which can be found here:
     Aihara, H., AlSayyad, Y., Ando, M., et al. 2019, PASJ, 71, 114
     doi: 10.1093/pasj/psz103
+    
+    Update(Apr 16 2024): Now inherit from selector and implement the _select() instead of run()
     """
     def_ratio_file = find_rail_file("examples_data/creation_data/data/hsc_ratios_and_specz.hdf5")
     def_set_file = find_rail_file("examples_data/creation_data/data/HSC_grid_settings.pkl" )
 
     name = 'GridSelection'
-    config_options = Degrader.config_options.copy()
+    config_options = Selector.config_options.copy()
     config_options.update(color_redshift_cut=Param(bool, True, msg='using color-based redshift cut'),
                           percentile_cut=Param(float, 99.0, msg='percentile cut-off for each pixel in color-based redshift cut off'),
                           redshift_cut=Param(float, 100.0, msg="cut redshifts above this value"),
@@ -57,14 +59,14 @@ class GridSelection(Degrader):
 
     def __init__(self, args, comm=None):
 
-        Degrader.__init__(self, args, comm=comm)
+        Selector.__init__(self, args, comm=comm)
 
         if self.config.redshift_cut < 0:
             raise ValueError("redshift cut must be positive")
         if (self.config.percentile_cut < 0) | (self.config.percentile_cut > 100):
             raise ValueError('percentile cut off must be between 0 and 100')
 
-    def run(self):
+    def _select(self):
         """
         HSC galaxies were binned in color magnitude space with i-band mag from -2 to 6 and g-z color from 13 to 26
         200 bins in each direction. The ratio of of galaxies with spectroscopic redshifts (training galaxies) to
@@ -104,8 +106,9 @@ class GridSelection(Degrader):
             raise ValueError("x limits should have two elements!")
         if len(y_lims) != 2:  # pragma: no cover
             raise ValueError("y limits should have two elements!")
-        data_hsc_like = data[(data['x_vals'] >= x_lims[0]) & (data['x_vals'] <= x_lims[1]) & (data['y_vals'] >= y_lims[0]) & (data['y_vals'] <= y_lims[1])]
-
+        # data_hsc_like = data[(data['x_vals'] >= x_lims[0]) & (data['x_vals'] <= x_lims[1]) & (data['y_vals'] >= y_lims[0]) & (data['y_vals'] <= y_lims[1])]
+        
+        data_hsc_like = data
         x_vals = data_hsc_like['x_vals'].to_numpy()
         y_vals = data_hsc_like['y_vals'].to_numpy()
 
@@ -155,9 +158,15 @@ class GridSelection(Degrader):
 
         ratio_list = []
         max_specz_list = []
+                
         for pix_x, pix_y in zip(pixels_x, pixels_y):
-            ratio_list.append(ratios[pix_y][pix_x])
-            max_specz_list.append(max_specz[pix_y][pix_x])
+            # cover the galaxy outside the grid of ratio
+            if (pix_x in [-1,ratios.shape[0]]) or (pix_y in [-1, ratios.shape[1]]):
+                ratio_list.append(0.0)
+                max_specz_list.append(99.0)
+            else:
+                ratio_list.append(ratios[pix_y][pix_x])
+                max_specz_list.append(max_specz[pix_y][pix_x])
 
         data_hsc_like['ratios'] = ratio_list
         data_hsc_like['max_specz'] = max_specz_list
@@ -204,9 +213,13 @@ class GridSelection(Degrader):
             else:  # pragma: no cover
                 for j in range(int(number_to_keep) + 1):
                     keep_inds.append(indices_to_list[j])
-
-        training_data = data_hsc_like_redshift_cut.loc[keep_inds, :]
-        training_data = training_data[training_data['redshift'] > 0]
-        training_data = training_data.drop(['x_vals', 'y_vals', 'ratios'], axis=1)
-
-        self.add_data('output', training_data)
+        
+        finalcut = data_hsc_like_redshift_cut.loc[keep_inds, :]
+        finalcut = finalcut[finalcut['redshift'] > 0]
+        
+        final_keep_inds = np.array(finalcut.index)
+        
+        total_mask = np.zeros(len(data), dtype=bool)
+        total_mask[final_keep_inds] = True
+
+        return total_mask

rail/creation/degradation/observing_condition_degrader.py

@@ -9,10 +9,10 @@ import pandas as pd
 from ceci.config import StageParameter as Param
 from photerr import LsstErrorModel, LsstErrorParams
 
-from rail.creation.degrader import Degrader
+from rail.creation.noisifier import Noisifier
 
 
-class ObsCondition(Degrader):
+class ObsCondition(Noisifier):
     """Photometric errors based on observation conditions
 
     This degrader calculates spatially-varying photometric errors
@@ -60,7 +60,7 @@ class ObsCondition(Degrader):
     """
 
     name = "ObsCondition"
-    config_options = Degrader.config_options.copy()
+    config_options = Noisifier.config_options.copy()
     config_options.update(
         nside=Param(
             int,
@@ -117,7 +117,7 @@ class ObsCondition(Degrader):
     
 
     def __init__(self, args, comm=None):
-        Degrader.__init__(self, args, comm=comm)
+        Noisifier.__init__(self, args, comm=comm)
 
         # store a list of keys relevant for
         # survey conditions;
@@ -407,10 +407,17 @@ class ObsCondition(Degrader):
 
         return pixel_cat
 
+    
+    def _initNoiseModel(self):
+        """
+        Initialise the error model: LSSTerrorModel
+        """
+        self.default_errorModel = LsstErrorModel()
 
-    def run(self):
+        
+    def _addNoise(self):
         """
-        Run the degrader.
+        Run the noisifier.
         """
         self.rng = np.random.default_rng(seed=self.config["random_seed"])
 
@@ -437,8 +444,8 @@ class ObsCondition(Degrader):
                 
                 # first, check if pixel is -99 - these objects have default obs_conditions:
                 if pixel==-99:
-                    # creating the error model for this pixel
-                    errorModel = LsstErrorModel()
+                    # use the default error model for this pixel
+                    errorModel = self.default_errorModel
                 
                 else:            
                     # get the observing conditions for this pixel

rail/creation/degradation/photerr_demo.ipynb

@@ -0,0 +1,238 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "2610a0f0-0c71-4401-896f-734442bcd66d",
+   "metadata": {},
+   "source": [
+    "## Photometric error stage demo\n",
+    "\n",
+    "author: Tianqing Zhang, John-Franklin Crenshaw\n",
+    "\n",
+    "This notebook demonstrate the use of `rail.creation.degradation.photometric_errors`, which adds column for the  photometric noise to the catalog based on the package PhotErr developed by John-Franklin Crenshaw. The RAIL stage PhotoErrorModel inherit from the Noisifier base classes, and the LSST, Roman, Euclid child classes inherit from the PhotoErrorModel"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f7a6adc3-68e8-4a1d-842f-bfb0960a1c4a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "from rail.creation.degradation.photometric_errors import LSSTErrorModel\n",
+    "from rail.creation.degradation.photometric_errors import RomanErrorModel\n",
+    "from rail.creation.degradation.photometric_errors import EuclidErrorModel\n",
+    "\n",
+    "from rail.core.data import PqHandle\n",
+    "from rail.core.stage import RailStage\n",
+    "\n",
+    "import matplotlib.pyplot as plt\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6912a740-31ea-4987-b06d-81ff17cd895a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "DS = RailStage.data_store\n",
+    "DS.__class__.allow_overwrite = True\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a282c2ed-141b-4507-8254-dc8fbc9864dc",
+   "metadata": {},
+   "source": [
+    "### Create a random catalog with ugrizy+YJHF bands as the the true input"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1078bc2a-fc54-41c3-bd30-6c447bb971d4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data = np.random.normal(23, 3, size = (1000,10))\n",
+    "\n",
+    "data_df = pd.DataFrame(data=data,    # values\n",
+    "            columns=['u', 'g', 'r', 'i', 'z', 'y', 'Y', 'J', 'H', 'F'])\n",
+    "data_truth = PqHandle('input')\n",
+    "data_truth.set_data(data_df)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1da27deb-d167-4f38-8c59-f270184d6ab3",
+   "metadata": {},
+   "source": [
+    "### The LSST error model adds noise to the optical bands"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e5f4862a-0621-46d4-8901-7e84b461c424",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "errorModel_lsst = LSSTErrorModel.make_stage(name=\"error_model\")\n",
+    "\n",
+    "samples_w_errs = errorModel_lsst(data_truth)\n",
+    "samples_w_errs()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6282c42e-1a6f-480a-aa2b-817bd30d372f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fig, ax = plt.subplots(figsize=(5, 4), dpi=100)\n",
+    "\n",
+    "for band in \"ugrizy\":\n",
+    "    # pull out the magnitudes and errors\n",
+    "    mags = samples_w_errs.data[band].to_numpy()\n",
+    "    errs = samples_w_errs.data[band + \"_err\"].to_numpy()\n",
+    "\n",
+    "    # sort them by magnitude\n",
+    "    mags, errs = mags[mags.argsort()], errs[mags.argsort()]\n",
+    "\n",
+    "    # plot errs vs mags\n",
+    "    ax.plot(mags, errs, label=band)\n",
+    "\n",
+    "ax.legend()\n",
+    "ax.set(xlabel=\"Magnitude (AB)\", ylabel=\"Error (mags)\")\n",
+    "plt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "50927ccb-4492-4bdd-a29b-0907704b2c59",
+   "metadata": {},
+   "source": [
+    "### The Roman error model adds noise to the infrared bands"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d484a34b-cf10-45bd-8571-b78dc1818180",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "errorModel_Roman = RomanErrorModel.make_stage(name=\"error_model\")\n",
+    "\n",
+    "samples_w_errs_roman = errorModel_Roman(data_truth)\n",
+    "samples_w_errs_roman()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a12362dc-0689-43f3-b990-edff734010bb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fig, ax = plt.subplots(figsize=(5, 4), dpi=100)\n",
+    "\n",
+    "for band in \"YJHF\":\n",
+    "    # pull out the magnitudes and errors\n",
+    "    mags = samples_w_errs_roman.data[band].to_numpy()\n",
+    "    errs = samples_w_errs_roman.data[band + \"_err\"].to_numpy()\n",
+    "\n",
+    "    # sort them by magnitude\n",
+    "    mags, errs = mags[mags.argsort()], errs[mags.argsort()]\n",
+    "\n",
+    "    # plot errs vs mags\n",
+    "    ax.plot(mags, errs, label=band)\n",
+    "\n",
+    "ax.legend()\n",
+    "ax.set(xlabel=\"Magnitude (AB)\", ylabel=\"Error (mags)\")\n",
+    "plt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "359de4ad-a47c-4dc5-8c81-53aeb92bdf42",
+   "metadata": {},
+   "source": [
+    "### The Euclid error model adds noise to YJH bands"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "775d0a23-7435-4b01-83db-2234c3d24f57",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "errorModel_Euclid = EuclidErrorModel.make_stage(name=\"error_model\")\n",
+    "\n",
+    "samples_w_errs_Euclid = errorModel_Euclid(data_truth)\n",
+    "samples_w_errs_Euclid()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7fcfe07a-5571-4dd3-8ad0-358964c1d493",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fig, ax = plt.subplots(figsize=(5, 4), dpi=100)\n",
+    "\n",
+    "for band in \"YJH\":\n",
+    "    # pull out the magnitudes and errors\n",
+    "    mags = samples_w_errs_Euclid.data[band].to_numpy()\n",
+    "    errs = samples_w_errs_Euclid.data[band + \"_err\"].to_numpy()\n",
+    "\n",
+    "    # sort them by magnitude\n",
+    "    mags, errs = mags[mags.argsort()], errs[mags.argsort()]\n",
+    "\n",
+    "    # plot errs vs mags\n",
+    "    ax.plot(mags, errs, label=band)\n",
+    "\n",
+    "ax.legend()\n",
+    "ax.set(xlabel=\"Magnitude (AB)\", ylabel=\"Error (mags)\")\n",
+    "plt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "268b3d37-b7fd-4ac1-8457-2104a87c9e6d",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "rail_env",
+   "language": "python",
+   "name": "rail_env"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

rail/creation/degradation/photometric_errors.py

@@ -0,0 +1,136 @@
+"""
+The Photometric error Model LSST, Roman, and Euclid. Based on photometric 
+error models defined in the package photerr
+
+Author: John Franklin Crenshaw, Tianqing Zhang
+"""
+from dataclasses import MISSING
+
+from ceci.config import StageParameter as Param
+from photerr import LsstErrorModel as peLsstErrorModel
+from photerr import LsstErrorParams as peLsstErrorParams
+from photerr import RomanErrorModel as peRomanErrorModel
+from photerr import RomanErrorParams as peRomanErrorParams
+from photerr import EuclidErrorModel as peEuclidErrorModel
+from photerr import EuclidErrorParams as peEuclidErrorParams
+from rail.creation.noisifier import Noisifier
+
+class PhotoErrorModel(Noisifier):
+    """The Base Model for photometric errors.
+
+    This is a wrapper around the error model from PhotErr. The parameter
+    docstring below is dynamically added by the installed version of PhotErr:
+    
+    
+    """
+    
+    name = "PhotoErrorModel"
+    
+    def set_params(self, peparams):
+        """
+        Set the photometric error parameters from photerr to 
+        the ceci config
+        """
+        PhotErrErrorParams = peparams
+                
+        config_options = Noisifier.config_options.copy()
+
+        # Dynamically add all parameters from PhotErr
+        _photerr_params = PhotErrErrorParams.__dataclass_fields__
+        self._photerr_params = _photerr_params
+        for key, val in _photerr_params.items():
+            # Get the default value
+            if val.default is MISSING:
+                default = val.default_factory()
+            else:
+                default = val.default
+
+            # Add this param to config_options
+            self.config[key] = Param(
+                None,  # Let PhotErr handle type checking
+                default,  
+                msg="See the main docstring for details about this parameter.",
+                required=False,
+            )
+        
+    def __init__(self, args, comm=None):
+        """
+        Constructor
+
+        Does standard Degrader initialization and sets up the error model.
+        """
+        Noisifier.__init__(self, args, comm=comm)
+        
+    def _initNoiseModel(self):
+        """
+        Initialize the noise model by the peNoiseModel
+        """
+        self.noiseModel = self.peNoiseModel(
+            **{key: self.config[key] for key in self._photerr_params}
+        )
+        
+    def _addNoise(self):
+        
+        """
+        Add noise to the input catalog
+        """
+        
+        # Load the input catalog
+        data = self.get_data("input")
+
+        # Add photometric errors
+        obsData = self.noiseModel(data, random_state=self.config.seed)
+        
+        # Return the new catalog
+        self.add_data("output", obsData)
+ 
+        
+class LSSTErrorModel(PhotoErrorModel):
+
+    """
+    The LSST Error model, defined by peLsstErrorParams and peLsstErrorModel
+    """
+    
+    name = "LSSTErrorModel"
+    
+    def __init__(self, args, comm=None):
+
+        PhotoErrorModel.__init__(self, args, comm=comm)
+        
+        self.set_params(peLsstErrorParams)   
+        self.peNoiseModel = peLsstErrorModel
+        
+
+        
+        
+class RomanErrorModel(PhotoErrorModel):
+    
+    """
+    The Roman Error model, defined by peRomanErrorParams and peRomanErrorModel
+    """
+    
+    name = "RomanErrorModel"
+    
+    def __init__(self, args, comm=None):
+
+        PhotoErrorModel.__init__(self, args, comm=comm)
+        
+        self.set_params(peRomanErrorParams)    
+        self.peNoiseModel = peRomanErrorModel
+
+                
+        
+class EuclidErrorModel(PhotoErrorModel):
+    
+    """
+    The Roman Error model, defined by peRomanErrorParams and peRomanErrorModel
+    """
+    
+    name = "EuclidErrorModel"
+    
+    def __init__(self, args, comm=None):
+
+        PhotoErrorModel.__init__(self, args, comm=comm)
+        
+        self.set_params(peEuclidErrorParams)    
+        self.peNoiseModel = peEuclidErrorModel

rail/creation/degradation/spectroscopic_degraders.py

@@ -2,10 +2,12 @@
 
 import numpy as np
 import pandas as pd
-from rail.creation.degrader import Degrader
+from rail.creation.selector import Selector
+from rail.creation.noisifier import Noisifier
 
 
-class LineConfusion(Degrader):
+
+class LineConfusion(Noisifier):
     """Degrader that simulates emission line confusion.
 
     .. code-block:: python
@@ -37,7 +39,7 @@ class LineConfusion(Degrader):
     """
 
     name = 'LineConfusion'
-    config_options = Degrader.config_options.copy()
+    config_options = Noisifier.config_options.copy()
     config_options.update(true_wavelen=float,
                           wrong_wavelen=float,
                           frac_wrong=float)
@@ -45,7 +47,7 @@ class LineConfusion(Degrader):
     def __init__(self, args, comm=None):
         """
         """
-        Degrader.__init__(self, args, comm=comm)
+        Noisifier.__init__(self, args, comm=comm)
         # validate parameters
         if self.config.true_wavelen < 0:
             raise ValueError("true_wavelen must be positive, not {self.config.true_wavelen}")
@@ -53,8 +55,12 @@ class LineConfusion(Degrader):
             raise ValueError("wrong_wavelen must be positive, not {self.config.wrong_wavelen}")
         if self.config.frac_wrong < 0 or self.config.frac_wrong > 1:
             raise ValueError("frac_wrong must be between 0 and 1., not {self.config.wrong_wavelen}")
+            
+            
+    def _initNoiseModel(self):
+        self.rng = np.random.default_rng(self.config.seed)
 
-    def run(self):
+    def _addNoise(self):
         """ Run method
 
         Applies line confusion
@@ -75,8 +81,7 @@ class LineConfusion(Degrader):
         zmin = self.config.wrong_wavelen / self.config.true_wavelen - 1
 
         # select the random fraction of galaxies whose lines are confused
-        rng = np.random.default_rng(self.config.seed)
-        idx = rng.choice(
+        idx = self.rng.choice(
             np.where(values[:, 0] > zmin)[0],
             size=int(self.config.frac_wrong * values.shape[0]),
             replace=False,
@@ -92,7 +97,7 @@ class LineConfusion(Degrader):
         self.add_data('output', outData)
 
 
-class InvRedshiftIncompleteness(Degrader):
+class InvRedshiftIncompleteness(Selector):
     """Degrader that simulates incompleteness with a selection function
     inversely proportional to redshift.
 
@@ -107,17 +112,17 @@ class InvRedshiftIncompleteness(Degrader):
     """
 
     name = 'InvRedshiftIncompleteness'
-    config_options = Degrader.config_options.copy()
+    config_options = Selector.config_options.copy()
     config_options.update(pivot_redshift=float)
 
     def __init__(self, args, comm=None):
         """
         """
-        Degrader.__init__(self, args, comm=comm)
+        Selector.__init__(self, args, comm=comm)
         if self.config.pivot_redshift < 0:
             raise ValueError("pivot redshift must be positive, not {self.config.pivot_redshift}")
 
-    def run(self):
+    def _select(self):
         """ Run method
 
         Applies incompleteness
@@ -136,4 +141,4 @@ class InvRedshiftIncompleteness(Degrader):
         rng = np.random.default_rng(self.config.seed)
         mask = rng.random(size=data.shape[0]) <= survival_prob
 
-        self.add_data('output', data[mask])
+        return mask

rail/creation/degradation/spectroscopic_selections.py

@@ -1,39 +1,43 @@
-""" Applying selection functions to catalog """
+"""Degrader that applies selection functions to catalog."""
 
 import os
 
 import numpy as np
 from ceci.config import StageParameter as Param
-from rail.creation.degrader import Degrader
+from rail.creation.selector import Selector
 from scipy.interpolate import interp1d
-from rail.core.utils import find_rail_file
+from rail.core.utils import RAILDIR
 
 
-class SpecSelection(Degrader):
-    """
-    The super class of spectroscopic selections.
+class SpecSelection(Selector):
+    """The super class of spectroscopic selections.
 
     Parameters
     ----------
-    N_tot: integer
-        total number of down-sampled, spec-selected galaxies.
-        If N_tot is greater than the number of spec-sepected galaxies, then
-        it will be ignored.
-    nondetect_val: value to be removed for non detects
-    downsample: bool
-        If True, then downsample the pre-selected galaxies
-        to N_tot galaxies.
-    success_rate_dir: string, the path to the success rate files.
-    percentile_cut: If using color-based redshift cut, percentile in redshifts above which redshifts will be cut from the sample. Default is 100 (no cut)
-    colnames: a dictionary that includes necessary columns
-                         (magnitudes, colors and redshift) for selection. For magnitudes, the keys are ugrizy; for colors, the keys are,
-                         for example, gr standing for g-r; for redshift, the key is 'redshift'.
-    random_seed: random seed for reproducibility.
-
+    N_tot : int
+        The total number of down-sampled, spec-selected galaxies. If N_tot is 
+        greater than the number of spec-sepected galaxies, then it will be 
+        ignored.
+    nondetect_val : float
+        The value to be removed for non detects
+    downsample : bool
+        If True, then downsample the pre-selected galaxies to N_tot galaxies.
+    success_rate_dir : string
+        The path to the success rate files
+    percentile_cut: int, default=100
+        If using color-based redshift cut, percentile in redshifts above which 
+        redshifts will be cut from the sample. Default is 100 (no cut).
+    colnames: dict
+        A dictionary that includes necessary columns (magnitudes, colors and 
+        redshift) for selection. For magnitudes, the keys are ugrizy; for 
+        colors, the keys are, for example, gr standing for g-r; for redshift, 
+        the key is 'redshift'.
+    random_seed : int
+        Random seed for reproducibility
     """
 
     name = "specselection"
-    config_options = Degrader.config_options.copy()
+    config_options = Selector.config_options.copy()
     config_options.update(
         N_tot=Param(int, 10000, msg="Number of selected sources"),
         nondetect_val=Param(float, 99.0, msg="value to be removed for non detects"),
@@ -44,7 +48,9 @@ class SpecSelection(Degrader):
         ),
         success_rate_dir=Param(
             str,
-            find_rail_file("examples_data/creation_data/data/success_rate_data"),
+            os.path.join(RAILDIR,
+                "rail/examples_data/creation_data/data/success_rate_data",
+            ),
             msg="The path to the directory containing success rate files.",
         ),
         percentile_cut=Param(int, 100, msg="cut redshifts above this percentile"),
@@ -62,17 +68,13 @@ class SpecSelection(Degrader):
     )
 
     def __init__(self, args, comm=None):
-        Degrader.__init__(self, args, comm=comm)
-        # validate the settings
+        Selector.__init__(self, args, comm=comm)
         self._validate_settings()
         self.mask = None
         self.rng = None
 
     def _validate_settings(self):
-        """
-        Validate all the settings.
-        """
-
+        """Validate all the settings."""
         if self.config.N_tot < 0:
             raise ValueError(
                 "Total number of selected sources must be a " "positive integer."
@@ -85,10 +87,13 @@ class SpecSelection(Degrader):
             )
 
     def validate_colnames(self, data):
-        """
-        Validate the column names of data table to make sure they have necessary information
-        for each selection.
-        colnames: a list of column names.
+        """Validate the column names of data table to make sure they have 
+        necessary information for each selection.
+
+        Parameters
+        ----------
+        colnames : list of str
+            A list of column names
         """
         colnames = self.config.colnames.values()
         check = all(item in data.columns for item in colnames)
@@ -101,15 +106,15 @@ class SpecSelection(Degrader):
             )
 
     def selection(self, data):
-        """
-        Selection functions. This should be overwritten by the subclasses
-        corresponding to different spec selections.
+        """Selection functions. 
+        
+        This should be overwritten by the subclasses corresponding to different 
+        spec selections.
         """
 
     def invalid_cut(self, data):
-        """
-        This function removes entries in the data that have invalid magnitude values
-        (nondetect_val or NaN)
+        """Removes entries in the data that have invalid magnitude values (NaN
+        or nondetect_val).
         """
         nondetect_val = self.config.nondetect_val
         for band in "ugrizy":
@@ -121,9 +126,7 @@ class SpecSelection(Degrader):
             )
 
     def downsampling_N_tot(self):
-        """
-        Method to randomly sample down the objects to a given
-        number of data objects.
+        """Randomly sample down the objects to a given number of data objects.
         """
         N_tot = self.config.N_tot
         N_selected = np.count_nonzero(self.mask)
@@ -146,11 +149,9 @@ class SpecSelection(Degrader):
             # update the internal state
             self.mask &= mask
 
-    def run(self):
-        """
-        Run the selection
-        """
-        self.rng = np.random.default_rng(seed=self.config.seed)
+    def _select(self):
+        """Run the selection."""
+        self.rng = np.random.default_rng(seed=self.config.random_seed)
         # get the bands and bandNames present in the data
         data = self.get_data("input", allow_missing=True)
         self.validate_colnames(data)
@@ -159,10 +160,11 @@ class SpecSelection(Degrader):
         self.selection(data)
         if self.config.downsample is True:
             self.downsampling_N_tot()
+        
+        return self.mask
+#         data_selected = data.iloc[np.where(self.mask == 1)[0]]
 
-        data_selected = data.iloc[np.where(self.mask == 1)[0]]
-
-        self.add_data("output", data_selected)
+#         self.add_data("output", data_selected)
 
     def __repr__(self):
         """
@@ -171,25 +173,22 @@ class SpecSelection(Degrader):
 
 
 class SpecSelection_GAMA(SpecSelection):
-    """
-    The class of spectroscopic selections with GAMA.
-    The GAMA survey covers an area of 286 deg^2, with ~238000 objects
-    The necessary column is r band
-    """
+    """The class of spectroscopic selections with GAMA.
 
+    The GAMA survey covers an area of 286 deg^2, with ~238000 objects.
+    
+    The necessary column is r band.
+    """
+    
     name = "specselection_gama"
 
     def selection(self, data):
-        """
-        GAMA selection function
-        """
+        """GAMA selection function. """
         print("Applying the selection from GAMA survey...")
         self.mask *= data[self.config.colnames["r"]] < 19.87
 
     def __repr__(self):
-        """
-        Define how the model is represented and printed.
-        """
+        """Define how the model is represented and printed."""
         # start message
         printMsg = "Applying the GAMA selection."
 
@@ -197,22 +196,22 @@ class SpecSelection_GAMA(SpecSelection):
 
 
 class SpecSelection_BOSS(SpecSelection):
-    """
-    The class of spectroscopic selections with BOSS.
+    """The class of spectroscopic selections with BOSS.
+    
     BOSS selection function is based on
     http://www.sdss3.org/dr9/algorithms/boss_galaxy_ts.php
-    The selection has changed slightly compared to Dawson+13
+
+    The selection has changed slightly compared to Dawson+13.
+
     BOSS covers an area of 9100 deg^2 with 893,319 galaxies.
+
     For BOSS selection, the data should at least include gri bands.
     """
 
     name = "specselection_boss"
 
     def selection(self, data):
-        """
-        The BOSS selection function.
-        """
-
+        """The BOSS selection function."""
         print("Applying the selection from BOSS survey...")
         # cut quantities (unchanged)
         c_p = 0.7 * (
@@ -252,9 +251,7 @@ class SpecSelection_BOSS(SpecSelection):
         self.mask *= low_z | cmass
 
     def __repr__(self):
-        """
-        Define how the model is represented and printed.
-        """
+        """ Define how the model is represented and printed."""
         # start message
         printMsg = "Applying the BOSS selection."
 
@@ -262,26 +259,27 @@ class SpecSelection_BOSS(SpecSelection):
 
 
 class SpecSelection_DEEP2(SpecSelection):
-    """
-    The class of spectroscopic selections with DEEP2.
+    """The class of spectroscopic selections with DEEP2.
+
     DEEP2 has a sky coverage of 2.8 deg^2 with ~53000 spectra.
-    For DEEP2, one needs R band magnitude, B-R/R-I colors which are not available for the time being.
-    So we use LSST gri bands now. When the conversion degrader is ready, this subclass will be updated
-    accordingly.
+
+    For DEEP2, one needs R band magnitude, B-R/R-I colors--which are not 
+    available for the time being, so we use LSST gri bands now. When the 
+    conversion degrader is ready, this subclass will be updated accordingly.
     """
 
     name = "specselection_deep2"
 
     def photometryCut(self, data):
-        """
-        Applying DEEP2 photometric cut based on Newman+13.
-        This modified selection gives the best match to the data n(z) with
-        its cut at z~0.75 and the B-R/R-I distribution (Newman+13, Fig. 12)
+        """Applies DEEP2 photometric cut based on Newman+13.
+
+        This modified selection gives the best match to the data n(z) with its 
+        cut at z~0.75 and the B-R/R-I distribution (Newman+13, Fig. 12).
 
         Notes
         -----
-        We cannot apply the surface brightness cut and do not apply the
-        Gaussian weighted sampling near the original colour cuts.
+        We cannot apply the surface brightness cut and do not apply the Gaussian
+        weighted sampling near the original colour cuts.
         """
         mask = (
             (data[self.config.colnames["r"]] > 18.5)
@@ -312,8 +310,7 @@ class SpecSelection_DEEP2(SpecSelection):
         self.mask &= mask
 
     def speczSuccess(self, data):
-        """
-        Spec-z success rate as function of r_AB for Q>=3 read of Figure 13 in
+        """Spec-z success rate as function of r_AB for Q>=3 read of Figure 13 in
         Newman+13 for DEEP2 fields 2-4. Values are binned in steps of 0.2 mag
         with the first and last bin centered on 19 and 24.
         """
@@ -338,16 +335,12 @@ class SpecSelection_DEEP2(SpecSelection):
         self.mask &= mask
 
     def selection(self, data):
-        """
-        DEEP2 selection function
-        """
+        """DEEP2 selection function."""
         self.photometryCut(data)
         self.speczSuccess(data)
 
     def __repr__(self):
-        """
-        Define how the model is represented and printed.
-        """
+        """Define how the model is represented and printed."""
         # start message
         printMsg = "Applying the DEEP2 selection."
 
@@ -355,21 +348,22 @@ class SpecSelection_DEEP2(SpecSelection):
 
 
 class SpecSelection_VVDSf02(SpecSelection):
-    """
-    The class of spectroscopic selections with VVDSf02.
+    """The class of spectroscopic selections with VVDSf02.
+
     It covers an area of 0.5 deg^2 with ~10000 sources.
+
     Necessary columns are i band magnitude and redshift.
     """
 
     name = "specselection_VVDSf02"
 
     def photometryCut(self, data):
-        """
-        Photometric cut of VVDS 2h-field based on LeFèvre+05.
+        """Photometric cut of VVDS 2h-field based on LeFèvre+05.
 
         Notes
         -----
-        The oversight of 1.0 magnitudes on the bright end misses 0.2% of galaxies.
+        The oversight of 1.0 magnitudes on the bright end misses 0.2% of 
+        galaxies.
         """
         mask = (data[self.config.colnames["i"]] > 17.5) & (
             data[self.config.colnames["i"]] < 24.0
@@ -378,18 +372,17 @@ class SpecSelection_VVDSf02(SpecSelection):
         self.mask &= mask
 
     def speczSuccess(self, data):
-        """
-        Success rate of VVDS 2h-field
+        """Success rate of VVDS 2h-field.
 
         Notes
         -----
-        We use a redshift-based and I-band based success rate
-        independently here since we do not know their correlation,
-        which makes the success rate worse than in reality.
+        We use a redshift-based and I-band based success rate independently here
+        since we do not know their correlation, which makes the success rate 
+        worse than in reality.
 
-        Spec-z success rate as function of i_AB read of Figure 16 in
-        LeFevre+05 for the VVDS 2h field. Values are binned in steps of
-        0.5 mag with the first starting at 17 and the last bin ending at 24.
+        Spec-z success rate as function of i_AB read of Figure 16 in LeFevre+05
+        for the VVDS 2h field. Values are binned in steps of 0.5 mag with the 
+        first starting at 17 and the last bin ending at 24.
         """
         success_I_bins = np.arange(17.0, 24.0 + 0.01, 0.5)
         success_I_centers = (success_I_bins[1:] + success_I_bins[:-1]) / 2.0
@@ -458,9 +451,7 @@ class SpecSelection_VVDSf02(SpecSelection):
         self.speczSuccess(data)
 
     def __repr__(self):
-        """
-        Define how the model is represented and printed.
-        """
+        """Define how the model is represented and printed."""
         # start message
         printMsg = "Applying the VVDSf02 selection."
 
@@ -468,19 +459,21 @@ class SpecSelection_VVDSf02(SpecSelection):
 
 
 class SpecSelection_zCOSMOS(SpecSelection):
-    """
-    The class of spectroscopic selections with zCOSMOS
+    """The class of spectroscopic selections with zCOSMOS.
+
     It covers an area of 1.7 deg^2 with ~20000 galaxies.
+
     For zCOSMOS, the data should at least include i band and redshift.
     """
 
     name = "specselection_zCOSMOS"
 
     def photometryCut(self, data):
-        """
-        Photometry cut for zCOSMOS based on Lilly+09.
+        """Photometry cut for zCOSMOS based on Lilly+09.
+
+        Updates the internal state.
+
         NOTE: This only includes zCOSMOS bright.
-        update the internal state
         """
         mask = (data[self.config.colnames["i"]] > 15.0) & (
             data[self.config.colnames["i"]] < 22.5
@@ -489,8 +482,7 @@ class SpecSelection_zCOSMOS(SpecSelection):
         self.mask &= mask
 
     def speczSuccess(self, data):
-        """
-        Spec-z success rate as function of redshift (x) and I_AB (y) read of
+        """Spec-z success rate as function of redshift (x) and I_AB (y) read of
         Figure 3 in Lilly+09 for zCOSMOS bright sample.
         """
         success_rate_dir = self.config.success_rate_dir
@@ -522,9 +514,7 @@ class SpecSelection_zCOSMOS(SpecSelection):
         self.speczSuccess(data)
 
     def __repr__(self):
-        """
-        Define how the model is represented and printed.
-        """
+        """Defines how the model is represented and printed."""
         # start message
         printMsg = "Applying the zCOSMOS selection."
 
@@ -532,16 +522,16 @@ class SpecSelection_zCOSMOS(SpecSelection):
 
 
 class SpecSelection_HSC(SpecSelection):
-    """
-    The class of spectroscopic selections with HSC
-    or HSC, the data should at least include giz bands and redshift.
+    """The class of spectroscopic selections with HSC.
+
+    For HSC, the data should at least include giz bands and redshift.
     """
 
     name = "specselection_HSC"
 
     def photometryCut(self, data):
-        """
-        HSC galaxies were binned in color magnitude space with i-band mag from -2 to 6 and g-z color from 13 to 26.
+        """HSC galaxies were binned in color magnitude space with i-band mag
+        from -2 to 6 and g-z color from 13 to 26.
         """
         mask = (data[self.config.colnames["i"]] > 13.0) & (
             data[self.config.colnames["i"]] < 26.0
@@ -552,11 +542,13 @@ class SpecSelection_HSC(SpecSelection):
         self.mask &= mask
 
     def speczSuccess(self, data):
-        """
-        HSC galaxies were binned in color magnitude space with i-band mag from -2 to 6 and g-z color from 13 to 26
-        200 bins in each direction. The ratio of of galaxies with spectroscopic redshifts (training galaxies) to
-        galaxies with only photometry in HSC wide field (application galaxies) was computed for each pixel. We divide
-        the data into the same pixels and randomly select galaxies into the training sample based on the HSC ratios
+        """HSC galaxies were binned in color magnitude space with i-band mag 
+        from -2 to 6 and g-z color from 13 to 26 (200 bins in each direction). 
+        The ratio of galaxies with spectroscopic redshifts (training galaxies) 
+        to galaxies with only photometry in HSC wide field (application 
+        galaxies) was computed for each pixel. We divide the data into the same
+        pixels and randomly select galaxies into the training sample based on 
+        the HSC ratios.
         """
         success_rate_dir = self.config.success_rate_dir
         x_edge = np.linspace(13, 26, 201, endpoint=True)
@@ -608,10 +600,8 @@ class SpecSelection_HSC(SpecSelection):
         self.speczSuccess(data)
 
     def __repr__(self):
-        """
-        Define how the model is represented and printed.
-        """
+        """Defines how the model is represented and printed."""
         # start message
         printMsg = "Applying the HSC selection."
 
-        return printMsg
+        return printMsg

rail/tools/util_photometry.py

@@ -415,8 +415,8 @@ class LSSTFluxToMagConverter(RailStage):
 class Dereddener(RailStage):
     """Utility stage that does dereddening
     
-    Note: set copy_cols="ALL" to copy all 
-    columns in data
+    Note: set copy_all_cols=True to copy all 
+    columns in data, copy_cols will be ignored
     """
     name = 'Dereddener'
 
@@ -427,6 +427,7 @@ class Dereddener(RailStage):
     config_options.update(dustmap_name='sfd')
     config_options.update(dustmap_dir=str)
     config_options.update(copy_cols=[])
+    config_options.update(copy_all_cols=False)
 
     inputs = [('input', Hdf5Handle)]
     outputs = [('output', Hdf5Handle)]
@@ -470,12 +471,12 @@ class Dereddener(RailStage):
             mag_vals = data[band_mag_name]
             out_data[band_mag_name] = mag_vals - ebvvec*self.config.band_a_env[i]
             band_mag_name_list.append(band_mag_name)
-        
-        # check if copy_cols is a list, or is set to ALL:
-        if isinstance(self.config.copy_cols, list): # pragma: no cover
+       
+        # check if copy_all_cols set to true:
+        if self.config.copy_all_cols==False: # pragma: no cover
             for col_ in self.config.copy_cols:  # pragma: no cover
                 out_data[col_] = data[col_]
-        elif self.config.copy_cols=="ALL": # pragma: no cover
+        elif self.config.copy_all_cols==True: # pragma: no cover
             for col_ in data:
                 # make sure we do not overwrite the photometry columns
                 if col_ not in band_mag_name_list: