corgidrp 0.1__py3-none-any.whl

corgidrp/__init__.py

@@ -0,0 +1,63 @@
+import configparser
+import os
+import pathlib
+import configparser
+
+__version__ = "0.1"
+version = __version__ # temporary backwards compatability 
+
+#### Create a configuration file for the corgidrp if it doesn't exist. 
+def create_config_dir():
+    """
+    Checks if the default .corgidrp directory exists, and if not, it sets it up
+    """
+    homedir = pathlib.Path.home()
+    config_folder = os.path.join(homedir, ".corgidrp")
+    # replace legacy file with folder if needed
+    if os.path.isfile(config_folder):
+        oldconfig = configparser.ConfigParser()
+        oldconfig.read(config_folder)
+        os.remove(config_folder)
+    else:
+        oldconfig = None
+
+    # make folder if doesn't exist
+    if not os.path.isdir(config_folder):
+        os.mkdir(config_folder)
+
+        # make default calibrations folder
+        default_cal_dir = os.path.join(config_folder, "default_calibs")
+        if not os.path.exists(default_cal_dir):
+            os.mkdir(default_cal_dir)
+
+        # write config 
+        config_filepath = os.path.join(config_folder, "corgidrp.cfg")
+        if not os.path.exists(config_filepath):
+            config = configparser.ConfigParser()
+            config["PATH"] = {}
+            config["PATH"]["caldb"] = os.path.join(config_folder, "corgidrp_caldb.csv") # location to store caldb
+            config["PATH"]["default_calibs"] = default_cal_dir
+            config["DATA"] = {}
+            config["DATA"]["track_individual_errors"] = "False"
+            # overwrite with old settings if needed
+            if oldconfig is not None:
+                config["PATH"]["caldb"] = oldconfig["PATH"]["caldb"]
+
+            with open(config_filepath, 'w') as f:
+                config.write(f)
+
+            print("corgidrp: Configuration file written to {0}. Please edit if you want things stored in different locations.".format(config_filepath))
+create_config_dir()
+    
+_bool_map = {"true" : True, "false" : False}
+
+# borrowed from the kpicdrp caldb
+# load in default caldbs based on configuration file
+config_filepath = os.path.join(pathlib.Path.home(), ".corgidrp", "corgidrp.cfg")
+config = configparser.ConfigParser()
+config.read(config_filepath)
+
+## pipeline settings
+caldb_filepath = config.get("PATH", "caldb", fallback=None)
+default_cal_dir = config.get("PATH", "default_calibs", fallback=None)
+track_individual_errors = _bool_map[config.get("DATA", "track_individual_errors").lower()]

corgidrp/caldb.py

@@ -0,0 +1,304 @@
+"""
+Calibration tracking system. Modified from kpicdrp caldb implmentation (Copyright (c) 2024, KPIC Team)
+"""
+import os
+import numpy as np
+import pandas as pd
+import corgidrp
+import corgidrp.data as data
+import astropy.time as time
+
+
+column_names = [
+    "Filepath",
+    "Type",
+    "MJD",
+    "EXPTIME",
+    "Files Used",
+    "Date Created",
+    "Hash",
+    "DRPVERSN",
+    "OBSID",
+    "NAXIS1",
+    "NAXIS2",
+    "OPMODE",
+    "CMDGAIN",
+    "EXCAMT",
+]
+
+labels = {data.Dark: "Dark", 
+          data.NonLinearityCalibration: "NonLinearityCalibration", 
+          data.BadPixelMap: "BadPixelMap", 
+          data.KGain : "KGain", 
+          data.DetectorParams : "DetectorParams"}
+
+class CalDB:
+    """
+    Database for tracking calibration files saved to disk. Modified from the kpicdrp version
+
+    Note that database is not parallelism-safe, but should be ok in most cases.
+    (Jason: look at using posix_ipc to guarantee thread safety if we really need it)
+
+    Args:
+        filepath (str): [optional] filepath to a CSV file with an existing database
+
+    Fields:
+        columns (list): column names of dataframe
+        filepath(str): full filepath to data
+    """
+
+    def __init__(self, filepath=""):
+        """
+        Args:
+            filepath (str): [optional] filepath to a CSV file with an existing database
+        """
+
+        # If filepath is not passed in, use the default (majority of case)
+        if len(filepath) == 0:
+            self.filepath = corgidrp.caldb_filepath
+        else:
+            # possibly edge case where we want to use specialized caldb
+            self.filepath = filepath
+
+        # if database does't exist, create a blank one
+        if not os.path.exists(self.filepath):
+            # new database
+            self.columns = column_names
+            self._db = pd.DataFrame(columns=self.columns)
+            self.save()
+        else:
+            # already a database exists
+            self.load()
+            self.columns = list(self._db.columns.values)
+
+    def load(self):
+        """
+        Load/update db from filepath
+        """
+        self._db = pd.read_csv(self.filepath)
+
+    def save(self):
+        """
+        Save file without numbered index to disk with user specified filepath as a CSV file
+        """
+        self._db.to_csv(self.filepath, index=False)
+
+    def _get_values_from_entry(self, entry, is_calib=True):
+        """
+        Extract the properties from this data entry to ingest them into the database
+
+        Args:
+            entry (corgidrp.data.Image subclass): calibration frame to add to the database
+            is_calib (bool): is a calibration frame. if Not, it won't look up filetype.
+                             Only used in get_calib() to grab metadata for science frames
+
+        Returns:
+            tuple:
+                row (list):
+                    List of data entry properties
+                row_dict (dict):
+                    Dictionary of data entry properties keyed by column names
+
+        """
+        filepath = os.path.abspath(entry.filepath)
+        if is_calib:
+            datatype = labels[entry.__class__]  # get the database str representation
+        else:
+            datatype = "Sci"
+        mjd = time.Time(entry.ext_hdr["SCTSRT"]).mjd
+        exptime = entry.ext_hdr["EXPTIME"]
+
+        # check if this exists. will be a keyword written by corgidrp
+        if "DRPNFILE" in entry.ext_hdr:
+            files_used = entry.ext_hdr["DRPNFILE"]
+        else:
+            files_used = 0
+
+        if "DRPCTIME" in entry.ext_hdr:
+            date_created = time.Time(entry.ext_hdr["DRPCTIME"]).mjd
+        else:
+            date_created = -1
+
+        if "DRPVERSN" in entry.ext_hdr:
+            drp_version = entry.ext_hdr["DRPVERSN"]
+        else:
+            drp_version = ""
+
+        obsid = entry.pri_hdr["OBSID"]
+
+        hash_val = entry.get_hash()
+
+        # this only works for 2D images. may need to adapt for non-2D calibration frames
+        naxis1 = entry.data.shape[-1]
+        naxis2 = entry.data.shape[-2]
+
+        row = [
+            filepath,
+            datatype,
+            mjd,
+            exptime,
+            files_used,
+            date_created,
+            hash_val,
+            drp_version,
+            obsid,
+            naxis1,
+            naxis2,
+        ]
+
+        # rest are ext_hdr keys we can copy
+        start_index = len(row)
+        for i in range(start_index, len(self.columns)):
+            row.append(entry.ext_hdr[self.columns[i]])  # add value staright from header
+
+        row_dict = {}
+        for key, val in zip(self.columns, row):
+            row_dict[key] = val
+
+        return row, row_dict
+
+    def create_entry(self, entry, to_disk=True):
+        """
+        Add a new entry to or update an existing one in the database. Note that function by default will load and save db to disk
+
+        Args:
+            entry (corgidrp.data.Image subclass): calibration frame to add to the database
+            to_disk (bool): True by default, will update DB from disk before adding entry and saving it back to disk
+        """
+        new_row, row_dict = self._get_values_from_entry(entry)
+
+        # update database from disk in case anything changed
+        if to_disk:
+            self.load()
+
+        # use filepath as key to see if it's already in database
+        if row_dict["Filepath"] in self._db.values:
+            row_index = self._db[
+                self._db["Filepath"] == row_dict["Filepath"]
+            ].index.values
+            self._db.loc[row_index, self.columns] = new_row
+        # otherwise create new entry
+        else:
+            new_entry = pd.DataFrame([new_row], columns=self.columns)
+            self._db = pd.concat([self._db, new_entry], ignore_index=True)
+
+        # save to disk to update changes
+        if to_disk:
+            self.save()
+
+    def remove_entry(self, entry, to_disk=True):
+        """
+        Remove an entry from the database. Removes the entire row
+
+        Args:
+            entry (corgidrp.data.Image subclass): calibration frame to add to the database
+            to_disk (bool): True by default, will update DB from disk before adding entry and saving it back to disk
+        """
+        new_row, row_dict = self._get_values_from_entry(entry)
+
+        # update database from disk in case anything changed
+        if to_disk:
+            self.load()
+
+        if row_dict["Filepath"] in self._db.values:
+            entry_index = self._db[
+                self._db["Filepath"] == row_dict["Filepath"]
+            ].index.values
+            self._db = self._db.drop(self._db.index[entry_index])
+            self._db = self._db.reset_index(drop=True)
+        else:
+            raise ValueError("No filepath found so could not remove.")
+
+        # save to disk to update changes
+        if to_disk:
+            self.save()
+
+    def get_calib(self, frame, dtype, to_disk=True):
+        """
+        Outputs the best calibration file of the given type for the input sciene frame.
+
+        Args:
+            frame (corgidrp.data.Image): an image frame to request a calibratio for
+            dtype (corgidrp.data Class): for example: corgidrp.data.Dark (TODO: document the entire list of options)
+            to_disk (bool): True by default, will update DB from disk before matching
+
+        Returns:
+            corgidrp.data.*: an instance of the appropriate calibration type (Exact type depends on calibration type)
+        """
+        if dtype not in labels:
+            raise ValueError(
+                "Requested calibration dtype of {0} not a valid option".format(dtype)
+            )
+        dtype_label = labels[dtype]
+
+        # get values for this science frame
+        _, frame_dict = self._get_values_from_entry(frame, is_calib=False)
+
+        # update database from disk in case anything changed
+        if to_disk:
+            self.load()
+
+        # downselect to only calibs of this type
+        calibdf = self._db[self._db["Type"] == dtype_label]
+
+        if dtype_label in ["Dark"]:
+            # general selection criteria for 2D image frames. Can use different selection criteria for different dtypes
+            options = calibdf.loc[
+                (
+                    (calibdf["EXPTIME"] == frame_dict["EXPTIME"])
+                    & (calibdf["NAXIS1"] == frame_dict["NAXIS1"])
+                    & (calibdf["NAXIS2"] == frame_dict["NAXIS2"])
+                )
+            ]
+        else:
+            options = calibdf
+
+        # select the one closest in time
+        result_index = np.abs(options["MJD"] - frame_dict["MJD"]).argmin()
+        calib_filepath = options.iloc[result_index, 0]
+
+        # load the object from disk and return it
+        return dtype(calib_filepath)
+
+    def scan_dir_for_new_entries(self, filedir, look_in_subfolders=True, to_disk=True):
+        """
+        Scan a folder and subfolder for calibration files and add them all to the caldb
+
+        Args:
+            filedir (str): path to folder to scan (includes all subfolders by default)
+            look_in_subfolders (bool): whether to look in subfolders for files. True by default
+            to_disk (bool): True by default, will update DB from disk before adding entry and saving it back to disk
+        """
+        calib_frames = []
+        # walk the directory to find all the calibration files
+        for dirpath, subfolders, filenames in os.walk(filedir):
+            for filename in filenames:
+                # hard coded check only for files that end in .fits
+                if filename[-5:] != ".fits":
+                    continue
+
+                filepath = os.path.join(dirpath, filename)
+                frame = data.autoload(filepath)
+
+                # check what class it has been loaded as. only save frames that fall into calibration classes
+                if frame.__class__ in labels:
+                    calib_frames.append(frame)
+
+            # the first iteration looks in the basedir
+            # if we don't wnat to look in subdirs now, we should break
+            if not look_in_subfolders:
+                break
+
+        # load all these files into the caldb
+        for calib_frame in calib_frames:
+            self.create_entry(calib_frame, to_disk=to_disk)
+
+### Create set of default calibrations
+# Add default detector_params calibration file if it doesn't exist
+if not os.path.exists(os.path.join(corgidrp.default_cal_dir, "DetectorParams_2023-11-01T00:00:00.000.fits")):
+    default_detparams = data.DetectorParams({}, date_valid=time.Time("2023-11-01 00:00:00", scale='utc'))
+    default_detparams.save(filedir=corgidrp.default_cal_dir)
+
+    # add default caldb entries
+    default_caldb = CalDB()
+    default_caldb.scan_dir_for_new_entries(corgidrp.default_cal_dir)