pandorafits 0.1.0__tar.gz

pandorafits-0.1.0/PKG-INFO

@@ -0,0 +1,55 @@
+Metadata-Version: 2.1
+Name: pandorafits
+Version: 0.1.0
+Summary: Tools to work with fits files for Pandora
+License: MIT
+Author: Christina Hedges
+Author-email: christina.l.hedges@nasa.gov
+Requires-Python: >=3.9,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: astropy (>=5)
+Requires-Dist: lxml (>=5.3.0,<6.0.0)
+Requires-Dist: openpyxl (>=3.1.5,<4.0.0)
+Requires-Dist: pandas (>=2.2.2,<3.0.0)
+Description-Content-Type: text/markdown
+
+# `pandora-fits`
+
+Tools to work with fits files from Pandora.
+
+`pandora-fits` wraps `astropy.io.fits.HDUList` classes to ensure that files conform to Pandora FITS standards.
+
+The standards are defined using excel files in the `src/pandorasat/fileformats/` folder. Changing these files will change the standards that this tool checks against.
+
+## Pandora Detectors
+
+Pandora has two detectors, VISDA and NIRDA. You can read more about each of these in [pandora-sat](https://github.com/PandoraMission/pandora-sat/tree/main).
+
+## Pandora File Levels
+
+Pandora will have the following levels of files for each detector
+
+| Level | Description                                                  |
+|-------|--------------------------------------------------------------|
+| 0     | Raw data from spacecraft                                     |
+| 1     | Reorganized raw data, with potential for additional keywords |
+| 2     | Calibrated image data products                               |
+| 3     | Spectral time-series data, ready for science.                |
+
+## Exceptions
+
+`pandora-fits` will throw exceptions if files are not in the correct format. This includes
+
+- Files do not have the right number of extensions
+- Extensions are not the correct type
+- Header keywords have the wrong values when compared with the template
+
+## Warnings
+
+`pandora-fits` will log warnings if files are missing keyword headers, but those headers aren't valued in the excel spreadsheet.
+

pandorafits-0.1.0/README.md

@@ -0,0 +1,34 @@
+# `pandora-fits`
+
+Tools to work with fits files from Pandora.
+
+`pandora-fits` wraps `astropy.io.fits.HDUList` classes to ensure that files conform to Pandora FITS standards.
+
+The standards are defined using excel files in the `src/pandorasat/fileformats/` folder. Changing these files will change the standards that this tool checks against.
+
+## Pandora Detectors
+
+Pandora has two detectors, VISDA and NIRDA. You can read more about each of these in [pandora-sat](https://github.com/PandoraMission/pandora-sat/tree/main).
+
+## Pandora File Levels
+
+Pandora will have the following levels of files for each detector
+
+| Level | Description                                                  |
+|-------|--------------------------------------------------------------|
+| 0     | Raw data from spacecraft                                     |
+| 1     | Reorganized raw data, with potential for additional keywords |
+| 2     | Calibrated image data products                               |
+| 3     | Spectral time-series data, ready for science.                |
+
+## Exceptions
+
+`pandora-fits` will throw exceptions if files are not in the correct format. This includes
+
+- Files do not have the right number of extensions
+- Extensions are not the correct type
+- Header keywords have the wrong values when compared with the template
+
+## Warnings
+
+`pandora-fits` will log warnings if files are missing keyword headers, but those headers aren't valued in the excel spreadsheet.

pandorafits-0.1.0/pyproject.toml

@@ -0,0 +1,28 @@
+[tool.poetry]
+name = "pandorafits"
+version = "0.1.0"
+description = "Tools to work with fits files for Pandora"
+authors = ["Christina Hedges <christina.l.hedges@nasa.gov>"]
+license = "MIT"
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.9"
+pandas = "^2.2.2"
+lxml = "^5.3.0"
+astropy = ">=5"
+openpyxl = "^3.1.5"
+
+[tool.poetry.group.dev]
+optional = true
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^5.2"
+flake8 = "^4.0.1"
+black = "^22.12.0"
+isort = "^5.10.1"
+jupyterlab = "^4.2.5"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

pandorafits-0.1.0/src/pandorafits/__init__.py

@@ -0,0 +1,20 @@
+__version__ = "0.1.0"
+import logging  # noqa: E402
+import os  # noqa
+import pandas as pd  # noqa
+import numpy as np  # noqa
+
+PACKAGEDIR = os.path.abspath(os.path.dirname(__file__))
+FORMATSDIR = f"{PACKAGEDIR}/fileformats/"
+logger = logging.getLogger("pandorafits")
+
+BITPIX_DICT = {
+    8: np.uint8,
+    16: np.int16,
+    32: np.float32,
+    -32: np.float32,
+    -64: np.float64,
+}
+
+
+from .fits import NIRDALevel0HDUList, NIRDALevel2HDUList, VISDALevel0HDUList, VISDALevel2HDUList  # noqa

pandorafits-0.1.0/src/pandorafits/fileformats/README.md

@@ -0,0 +1,43 @@
+# File Formats
+
+File formats for Pandora FITS files are defined in these excel files. Excel files are used because they are highly human readable, and each sheet can be used to define each extension.
+
+There are two files with definitions:
+
+## Extension Types Files
+
+For each detector and each level of data there is a file defining the extension types, for example
+
+| Extension | Type       |
+|-----------|------------|
+| 0         | PrimaryHDU |
+| 1         | ImageHDU   |
+| 2         | ImageHDU   |
+| 3         | ImageHDU   |
+| 4         | ImageHDU   |
+| 5         | TableHDU   |
+
+This is used to check that input files have the right types of FITS extensions.
+
+## Header Formats
+
+For each detector and level the headers are defined in excel files. Each excel file has one sheet per extension. Headers are defined with the keyword name, the value expected (if any) and a comment.
+
+| Name     | Value        | Comment                       |
+|----------|--------------|-------------------------------|
+| SIMPLE   | TRUE         | conforms to FITS standard     |
+| BITPIX   | 8            | array data type               |
+| NAXIS    | 0            | number of array dimensions    |
+| EXTEND   |              |                               |
+| EXTNAME  | PRIMARY      | name of extension             |
+| NEXTEND  | 5            | number of standard extensions |
+| SIMDATA  |              | simulated data                |
+| SCIDATA  |              | science data                  |
+| TELESCOP | NASA Pandora | telescope                     |
+| INSTRMNT | NIRDA        | instrument                    |
+
+This tool will check to ensure that all the correct header keys exist in the files, and if they are valued in the excel sheet the tool will check that they have the expected value.
+
+## Dummy Files
+
+`pandora-fits` can make dummy files with the formats described in this repository which have data of the correct `dtype` in them, but the data is all values of `1`.

pandorafits-0.1.0/src/pandorafits/fits.py

@@ -0,0 +1,218 @@
+"""Class to handle Pandora fits files"""
+
+from . import logger
+from . import BITPIX_DICT, FORMATSDIR
+import numpy as np
+from astropy.io import fits
+import pandas as pd
+
+
+class FITSTemplateException(Exception):
+    """Custom exception for fits files not having the right shape."""
+
+    def __init__(self, message):
+        super().__init__(message)
+
+
+class FITSValueException(Exception):
+    """Custom exception for fits files not having the right values."""
+
+    def __init__(self, message):
+        super().__init__(message)
+
+
+class FITSHandlerMixins(object):
+    """Mixins to verify fits objects have the expected formats"""
+
+    def _get_expected_cards(self, index):
+        return [
+            fits.Card(*d.fillna("").values)
+            for _, d in self.header_formats[index].iterrows()
+        ]
+
+    def _get_dummy_hdus(self):
+        hdulist = []
+        for idx, d in self.extension_types.iterrows():
+            cards = self._get_expected_cards(idx)
+            hdr = fits.Header(cards)
+            data = None
+            if d.Type == "PrimaryHDU":
+                hdu = fits.PrimaryHDU(header=hdr)
+            elif d.Type == "ImageHDU":
+                shape = tuple(
+                    [
+                        hdr[f"NAXIS{naxis}"]
+                        for naxis in np.arange(1, hdr["NAXIS"] + 1)[::-1]
+                    ]
+                )
+                data = np.ones(shape, dtype=BITPIX_DICT[hdr["BITPIX"]])
+                hdu = fits.ImageHDU(header=hdr, data=data)
+            elif d.Type == "TableHDU":
+                ncolumns = len(
+                    [c.keyword for c in cards if c.keyword.startswith("TTYPE")]
+                )
+                coldefs = fits.ColDefs(
+                    [
+                        fits.Column(
+                            name=hdr[f"TTYPE{idx}"],
+                            format=hdr[f"TFORM{idx}"],
+                            unit=f"TUNIT{idx}",
+                        )
+                        for idx in np.arange(1, ncolumns + 1)
+                    ]
+                )
+                hdu = fits.TableHDU.from_columns(coldefs, header=hdr)
+            hdulist.append(hdu)
+        return hdulist
+
+    def _validate_ext_types(self):
+        """Validate that the extensions have the correct types, e.g. ImageHDU, TableHDU, etc"""
+        if not len(self) == self.nextension:
+            raise FITSTemplateException(
+                f"Expected {self.nextension} extensions, got {len(self)}."
+            )
+        for hdu, expected_type in zip(self, self.extension_types.Type.values):
+            if not isinstance(hdu, getattr(fits, expected_type)):
+                raise FITSTemplateException(
+                    f"Expected extension type {expected_type}, got {hdu}."
+                )
+
+    def _validate_headers(self):
+        """Validate the extensions have the right header keywords"""
+        for idx, hdu in enumerate(self):
+            hdr = hdu.header
+            expected_header = fits.Header(self._get_expected_cards(idx))
+            for key in expected_header:
+                if not (key in hdr):
+                    logger.warning(f"Key {key} expected, but not found.")
+                    continue
+                if expected_header[key] in ["", None, np.nan]:
+                    logger.warning(f"{key} header key missing from ext {idx}.")
+                else:
+                    if hdr[key] != expected_header[key]:
+                        if key not in [
+                            "NAXIS1",
+                            "NAXIS2",
+                            "NAXIS3",
+                            "NAXIS4",
+                            "NAXIS5",
+                        ]:
+                            FITSValueException(
+                                f"{key} expected to have value of {expected_header[key]}, but has value {hdr[key]}."
+                            )
+
+    def _validate_data(self):
+        """Check the data in the fits file is all the right dtype, given expected `bitpix`"""
+        for idx, hdu in enumerate(self):
+            if hdu.header["EXTNAME"] == "PRIMARY":
+                continue
+            if isinstance(hdu, fits.ImageHDU):
+                expected_header = fits.Header(self._get_expected_cards(idx))
+                expected_type = BITPIX_DICT[expected_header["bitpix"]]
+                if not hdu.data.dtype == expected_type:
+                    raise FITSTemplateException(
+                        f"Expected data of type {expected_type}, got {hdu.data.dtype}"
+                    )
+
+    def validate(self):
+        """Validate all aspects of the file"""
+        self._validate_ext_types()
+        self._validate_headers()
+        self._validate_data()
+
+
+class PandoraHDUList(fits.HDUList, FITSHandlerMixins):
+    """Base class, not designed to be used. Adds mixins to the fits.HDUList object"""
+
+    def __init__(self, file=None):
+        if file is None:
+            super().__init__(self._get_dummy_hdus())
+        elif isinstance(file, str):
+            super().__init__(fits.open(file))
+        elif isinstance(file, fits.HDUList):
+            super().__init__(file)
+        self.nextension = len(self.header_formats)
+        self.validate()
+
+    def writeto(self, *args, **kwargs):
+        """Write to file
+
+        Here we will add in some functionality to add in keywords on write that express the history somehow, and check the file names?
+        """
+        fits.HDUList(self).writeto(*args, **kwargs)
+
+
+class NIRDALevel0HDUList(PandoraHDUList):
+    """NIRDA Level 0 File Type"""
+
+    def __init__(self, file=None):
+        """
+        This class will read a file passed to it using astropy fits.
+        After loading it will validate that the file is compliant with
+        the NIRDA Level 0 file standards specified in the `fileformats`
+        folder. This object subclasses the fits.HDUList object, and
+        maintains all its class methods.
+
+        Parameters:
+        -----------
+        file: None, str, fits.HDUList
+            The file to load
+        """
+        self.header_formats = [
+            pd.read_excel(FORMATSDIR + "nirda-headers-level0.xlsx", idx)
+            for idx in range(2)
+        ]
+        self.extension_types = pd.read_excel(
+            FORMATSDIR + "nirda-extension-types-level0.xlsx"
+        )
+        super().__init__(file=file)
+
+
+class NIRDALevel2HDUList(PandoraHDUList):
+    def __init__(self, file=None):
+        self.header_formats = [
+            pd.read_excel(FORMATSDIR + "nirda-headers-level2.xlsx", idx)
+            for idx in range(6)
+        ]
+        self.extension_types = pd.read_excel(
+            FORMATSDIR + "nirda-extension-types-level2.xlsx"
+        )
+        super().__init__(file=file)
+
+
+class VISDALevel0HDUList(PandoraHDUList):
+    def __init__(self, file=None):
+        self.header_formats = [
+            pd.read_excel(FORMATSDIR + "visda-headers-level0.xlsx", idx)
+            for idx in range(3)
+        ]
+        self.extension_types = pd.read_excel(
+            FORMATSDIR + "visda-extension-types-level0.xlsx"
+        )
+        super().__init__(file=file)
+
+
+class VISDALevel2HDUList(PandoraHDUList):
+    def __init__(self, file=None, nROIs: int = 9):
+        self.header_formats = [
+            pd.read_excel(FORMATSDIR + "visda-headers-level2.xlsx", 0),
+            *[
+                pd.read_excel(FORMATSDIR + "visda-headers-level2.xlsx", 1)
+                for _ in range(1, nROIs + 1)
+            ],
+            *[
+                pd.read_excel(FORMATSDIR + "visda-headers-level2.xlsx", idx)
+                for idx in range(2, 5)
+            ],
+        ]
+        self.extension_types = pd.read_excel(
+            FORMATSDIR + "visda-extension-types-level2.xlsx"
+        )
+        self.header_formats[1].loc[
+            self.header_formats[1].Name == "EXTNAME", "Value"
+        ] = "TARGET"
+        for idx in np.arange(2, nROIs + 2):
+            self.header_formats[idx].loc[
+                self.header_formats[1].Name == "EXTNAME", "Value"
+            ] = f"STAR{idx - 1:03}"
+        super().__init__(file=file)