tglc 0.6.5__py3-none-any.whl → 0.7.0__py3-none-any.whl

tglc/ephemeris_data/README.md

@@ -0,0 +1,48 @@
+# TESS Ephemeris Files
+
+This directory contains files with TESS ephemeris data used for barycentric time correction by QLP. The files are CSV files with X,Y,Z coordinates in AU from the solar system barycenter and the reference frame is ICRF.
+
+Each file has a set of sectors it is meant to be used for:
+
+- Sectors 1-5: [20180720_tess_ephem.csv](20180720_tess_ephem.csv)
+- Sectors 6-19: [20190101_tess_ephem.csv](20190101_tess_ephem.csv)
+- Sectors 20-32: [20200101_tess_ephem.csv](20200101_tess_ephem.csv)
+- Sectors 33-45: [20210101_tess_ephem.csv](20210101_tess_ephem.csv)
+- Sectors 46-59: [20211215_tess_ephem.csv](20211215_tess_ephem.csv)
+- Sectors 60-73: [20221201_tess_ephem.csv](20221201_tess_ephem.csv)
+- Sectors 74-87: [20231201_tess_ephem.csv](20231201_tess_ephem.csv)
+- Sectors 88-101: [20241201_tess_ephem.csv](20241201_tess_ephem.csv)
+
+## Downloading the files
+
+At the start of every year, we need to add a TESS predicted ephemeris file so we can correct spacecraft time to BJD TDB.These ephemerides are downloaded from the horizons website: <https://ssd.jpl.nasa.gov/horizons.cgi>
+
+**Ephemeris Type:** Vector Table
+
+**Target Body:** TESS
+
+**Coordinate Origin:** Solar System Barycenter (SSB)
+
+**Time Span:** The required time space for the year. We want the step size to be at least one hour
+
+**Table Settings:**
+
+- Quantities code = 4 (position, LT, range, and range-rate)
+- Reference frame = ICRF
+- Reference plane = x-y axes of reference frame
+- Vector correction = geometric states
+- Output units = AU and days
+- CSV format = YES
+- Object summary = up to you. The summary will need to be commented out anyways, so I prefer to not include it.
+
+Make sure you comment out the header and footer information in the csv file. Also, when last generated (Jan 15 2023) I needed to add the following line before the values:
+
+```
+JDTDB,Calendar Date (TDB),X,Y,Z,LT,RG,RR,
+```
+
+(see the format of the other .csv files in the directory for guidance)
+
+After adding the file to this directory, plot all the ephemerides by running [plot_ephemerides.py](plot_ephemerides.py). This ensures the files can all be read successfully. Examine the plot it produces and check that the transition between years is smooth, and that the new file continues the existing sine wave patterns.
+
+If you have a function that specifies where the ephemeris file is for each sector (like in the [notebook in this repository](/TESS%20Time%20Correction.ipynb)), make sure to update that as well.

tglc/ephemeris_data/plot_ephemerides.py

@@ -0,0 +1,54 @@
+"""
+Plot X,Y,Z coordinates from JPL horizons TESS ephemeris files to ensure the
+transitions are smooth.
+"""
+
+from pathlib import Path
+from typing import Optional
+
+import matplotlib.pyplot as plt
+import pandas as pd
+
+
+def plot_ephemerides(
+    ephemeris_directory: Optional[Path] = None, output_file: Optional[Path] = None
+) -> None:
+    """
+    Plot ephemerides from all files in given directory.
+
+    Parameters
+    ----------
+    ephemeris_directory : Path, optional
+        Directory containing files with ephemeris data to plot
+    output_file : Path, optional
+        File to save plot to. If omitted, calls plt.show()
+    """
+    if ephemeris_directory is None:
+        ephemeris_directory = Path(__file__).parent
+    ephemeris_files = sorted(list(ephemeris_directory.iterdir()))
+    ephemerides = [
+        pd.read_csv(ephemeris_file, comment="#")
+        for ephemeris_file in ephemeris_files
+        if ephemeris_file.suffix == ".csv"
+    ]
+
+    fig, (ax_x, ax_y, ax_z) = plt.subplots(3, 1, figsize=(9, 6), layout="constrained")
+
+    for ephemeris, path in zip(ephemerides, ephemeris_files):
+        ax_x.plot(ephemeris["JDTDB"], ephemeris["X"], label=path.stem, alpha=0.8)
+        ax_y.plot(ephemeris["JDTDB"], ephemeris["Y"], label=path.stem, alpha=0.8)
+        ax_z.plot(ephemeris["JDTDB"], ephemeris["Z"], label=path.stem, alpha=0.8)
+
+    ax_x.set(xlabel="JD TDB", ylabel="TESS X coordinate [AU]")
+    ax_y.set(xlabel="JD TDB", ylabel="TESS Y coordinate [AU]")
+    ax_z.set(xlabel="JD TDB", ylabel="TESS Z coordinate [AU]")
+    fig.suptitle("JPL TESS Ephemerides for QLP")
+
+    if output_file is None:
+        plt.show()
+    else:
+        fig.savefig(output_file)
+
+
+if __name__ == "__main__":
+    plot_ephemerides(Path(__file__).parent)

tglc/ffi.py

@@ -4,7 +4,7 @@ import pickle
 import sys
 import astropy.units as u
 import numpy as np
-import pkg_resources
+import importlib_resources
 import requests
 import time
 
@@ -86,7 +86,7 @@ def convert_gaia_id(catalogdata_tic):
             FROM gaiadr3.dr2_neighbourhood
             WHERE dr2_source_id IN {gaia_ids}
             """
-    gaia_array = np.array(catalogdata_tic['GAIA'])
+    gaia_array = np.array([str(item) for item in catalogdata_tic['GAIA']], dtype=object)
     gaia_array = gaia_array[gaia_array != 'None']
     # np.save('gaia_array.npy', gaia_array)
     segment = (len(gaia_array) - 1) // 10000
@@ -327,7 +327,7 @@ def ffi(ccd=1, camera=1, sector=1, size=150, local_directory='', producing_mask=
         np.save(f'{local_directory}mask/mask_sector{sector:04d}_cam{camera}_ccd{ccd}.npy', mask)
         return
     # load mask
-    mask = pkg_resources.resource_stream(__name__, f'background_mask/median_mask.fits')
+    mask = importlib_resources.files(__package__).joinpath("background_mask/median_mask.fits").open("rb")
     mask = fits.open(mask)[0].data[(camera - 1) * 4 + (ccd - 1), :]
     mask = np.repeat(mask.reshape(1, 2048), repeats=2048, axis=0)
     bad_pixels = np.zeros(np.shape(flux[0]))

tglc/ffi_cut.py

@@ -4,6 +4,10 @@ import sys
 import pickle
 import numpy as np
 import astropy.units as u
+import requests
+import time
+import contextlib
+import threading
 
 from os.path import exists
 from astroquery.gaia import Gaia
@@ -22,8 +26,34 @@ Gaia.ROW_LIMIT = -1
 Gaia.MAIN_GAIA_TABLE = "gaiadr3.gaia_source"
 
 
+@contextlib.contextmanager
+def _dot_wait(message, interval=0.6, done_format=None):
+    stop_event = threading.Event()
+    start = time.time()
+
+    def _run():
+        dots = 0
+        while not stop_event.is_set():
+            dots = (dots % 3) + 1
+            print(f"\r{message} {'.' * dots}", end="", flush=True)
+            time.sleep(interval)
+
+    thread = threading.Thread(target=_run, daemon=True)
+    thread.start()
+    try:
+        yield
+    finally:
+        stop_event.set()
+        thread.join()
+        if done_format is None:
+            done_format = "{message} ... done in {elapsed:.1f}s."
+        done_line = done_format.format(message=message, elapsed=time.time() - start)
+        print(f"\r{done_line}   ")
+
+
 class Source_cut(object):
-    def __init__(self, name, size=50, sector=None, cadence=None, limit_mag=None, transient=None):
+    def __init__(self, name, size=50, sector=None, cadence=None, limit_mag=None, transient=None, ffi='TICA',
+                 mast_timeout=3600):
         """
         Source_cut object that includes all data from TESS and Gaia DR3
         :param name: str, required
@@ -51,35 +81,100 @@ class Source_cut(object):
         self.quality = []
         self.mask = []
         self.transient = transient
+        self.ffi = ffi
+        Tesscut._service_api_connection.TIMEOUT = mast_timeout
+        print(f'MAST Tesscut timeout set to {mast_timeout}s.')
+
+        def _parse_tic_id(t):
+            if not isinstance(t, str):
+                return None
+            s = t.strip()
+            if s.upper().startswith('TIC'):
+                parts = s.split()
+                if len(parts) > 1 and parts[1].isdigit():
+                    return int(parts[1])
+                s = s[3:].strip()
+            return int(s) if s.isdigit() else None
+
+        def _is_tic_id(t):
+            if not isinstance(t, str):
+                return False
+            s = t.strip()
+            return s.upper().startswith('TIC') or s.isdigit()
+
+        target = None
+        is_tic = _is_tic_id(self.name) and _parse_tic_id(self.name) is not None
+        try:
+            target = Catalogs.query_object(self.name, radius=21 * 0.707 / 3600, catalog="Gaia", version=2)
+        except requests.exceptions.RequestException as e:
+            warnings.warn(f'MAST name lookup failed for "{self.name}": {e}')
+
+        if target is None or len(target) == 0:
+            try:
+                target = Catalogs.query_object(self.name, radius=5 * 21 * 0.707 / 3600, catalog="Gaia", version=2)
+            except requests.exceptions.RequestException as e:
+                warnings.warn(f'MAST name lookup failed for "{self.name}": {e}')
+
+        if target is None or len(target) == 0:
+            if is_tic:
+                raise RuntimeError(
+                    f'MAST name lookup failed for TIC target "{self.name}". Please retry when MAST is available.'
+                )
+            raise RuntimeError(
+                f'Unable to resolve target "{self.name}". MAST name lookup appears unavailable.'
+            )
 
-        target = Catalogs.query_object(self.name, radius=21 * 0.707 / 3600, catalog="Gaia", version=2)
-        if len(target) == 0:
-            target = Catalogs.query_object(self.name, radius=5 * 21 * 0.707 / 3600, catalog="Gaia", version=2)
         ra = target[0]['ra']
         dec = target[0]['dec']
+        target_designation = target[0].get('designation', None)
         coord = SkyCoord(ra=ra, dec=dec, unit=(u.degree, u.degree), frame='icrs')
         radius = u.Quantity((self.size + 6) * 21 * 0.707 / 3600, u.deg)
-        print(f'Target Gaia: {target[0]["designation"]}')
-        catalogdata = Gaia.cone_search_async(coord, radius=radius,
-                                             columns=['DESIGNATION', 'phot_g_mean_mag', 'phot_bp_mean_mag',
-                                                      'phot_rp_mean_mag', 'ra', 'dec', 'pmra', 'pmdec']).get_results()
+        if target_designation:
+            print(f'Target Gaia: {target_designation}')
+        with _dot_wait('Querying Gaia DR3 cone search'):
+            catalogdata = Gaia.cone_search_async(
+                coord,
+                radius=radius,
+                columns=['DESIGNATION', 'phot_g_mean_mag', 'phot_bp_mean_mag',
+                         'phot_rp_mean_mag', 'ra', 'dec', 'pmra', 'pmdec']
+            ).get_results()
         print(f'Found {len(catalogdata)} Gaia DR3 objects.')
-        catalogdata_tic = tic_advanced_search_position_rows(ra=ra, dec=dec, radius=(self.size + 2) * 21 * 0.707 / 3600, limit_mag=limit_mag)
+        with _dot_wait('Querying TIC around target'):
+            catalogdata_tic = tic_advanced_search_position_rows(
+                ra=ra,
+                dec=dec,
+                radius=(self.size + 2) * 21 * 0.707 / 3600,
+                limit_mag=limit_mag
+            )
         print(f'Found {len(catalogdata_tic)} TIC objects.')
-        self.tic = convert_gaia_id(catalogdata_tic)
+        with _dot_wait('Crossmatching TIC -> Gaia DR3 (this may take a while)'):
+            self.tic = convert_gaia_id(catalogdata_tic)
         sector_table = Tesscut.get_sectors(coordinates=coord)
         if len(sector_table) == 0:
             warnings.warn('TESS has not observed this position yet :(')
+        wait_note = 'later sectors with 200s cadence can take ~20 minutes'
         if sector is None:
-            hdulist = Tesscut.get_cutouts(coordinates=coord, size=self.size)
+            wait_message = f'Requesting Tesscut cutouts (all sectors). Waiting on MAST response ({wait_note})'
+            with _dot_wait(wait_message):
+                hdulist = Tesscut.get_cutouts(coordinates=coord, size=self.size, product=ffi)
         elif sector == 'first':
-            hdulist = Tesscut.get_cutouts(coordinates=coord, size=self.size, sector=sector_table['sector'][0])
+            wait_message = f'Requesting Tesscut cutouts (first sector). Waiting on MAST response ({wait_note})'
+            with _dot_wait(wait_message):
+                hdulist = Tesscut.get_cutouts(
+                    coordinates=coord, size=self.size, product=ffi, sector=sector_table['sector'][0]
+                )
             sector = sector_table['sector'][0]
         elif sector == 'last':
-            hdulist = Tesscut.get_cutouts(coordinates=coord, size=self.size, sector=sector_table['sector'][-1])
+            wait_message = f'Requesting Tesscut cutouts (last sector). Waiting on MAST response ({wait_note})'
+            with _dot_wait(wait_message):
+                hdulist = Tesscut.get_cutouts(
+                    coordinates=coord, size=self.size, product=ffi, sector=sector_table['sector'][-1]
+                )
             sector = sector_table['sector'][-1]
         else:
-            hdulist = Tesscut.get_cutouts(coordinates=coord, size=self.size, sector=sector)
+            wait_message = f'Requesting Tesscut cutouts (sector {sector}). Waiting on MAST response ({wait_note})'
+            with _dot_wait(wait_message):
+                hdulist = Tesscut.get_cutouts(coordinates=coord, size=self.size, product=ffi, sector=sector)
         self.catalogdata = catalogdata
         self.sector_table = sector_table
         self.camera = int(sector_table[0]['camera'])
@@ -114,8 +209,14 @@ class Source_cut(object):
         self.ccd = int(hdu[0].header['CCD'])
         wcs = WCS(hdu[2].header)
         data_time = hdu[1].data['TIME']
-        data_flux = hdu[1].data['FLUX']
-        data_flux_err = hdu[1].data['FLUX_ERR']
+        if self.ffi == 'SPOC':
+            data_flux = hdu[1].data['FLUX']
+            data_flux_err = hdu[1].data['FLUX_ERR']
+        elif self.ffi == 'TICA':
+            data_flux = hdu[1].data['FLUX'] / (200 * 0.8 * 0.99)
+            data_flux_err = hdu[1].data['FLUX_ERR'] / (200 * 0.8 * 0.99)
+        else:
+            raise Exception(f'FFI type {self.ffi} not supported')
         data_quality = hdu[1].data['QUALITY']
         # data_time = data_time[np.where(data_quality == 0)]
         # data_flux = data_flux[np.where(data_quality == 0), :, :][0]
@@ -245,7 +346,8 @@ class Source_cut_pseudo(object):
         self.gaia = gaia_targets
 
 
-def ffi_cut(target='', local_directory='', size=90, sector=None, limit_mag=None, transient=None):
+def ffi_cut(target='', local_directory='', size=90, sector=None, limit_mag=None, transient=None, ffi='TICA',
+            mast_timeout=3600):
     """
     Function to generate Source_cut objects
     :param target: string, required
@@ -259,13 +361,13 @@ def ffi_cut(target='', local_directory='', size=90, sector=None, limit_mag=None,
     :return: tglc.ffi_cut.Source_cut
     """
     if sector is None:
-        source_name = f'source_{target}'
+        source_name = f'source_{ffi}_{target}'
     elif sector == 'first':
-        source_name = f'source_{target}_earliest_sector'
+        source_name = f'source_{ffi}_{target}_earliest_sector'
     elif sector == 'last':
-        source_name = f'source_{target}_last_sector'
+        source_name = f'source_{ffi}_{target}_last_sector'
     else:
-        source_name = f'source_{target}_sector_{sector}'
+        source_name = f'source_{ffi}_{target}_sector_{sector}'
     source_exists = exists(f'{local_directory}source/{source_name}.pkl')
     if source_exists and os.path.getsize(f'{local_directory}source/{source_name}.pkl') > 0:
         with open(f'{local_directory}source/{source_name}.pkl', 'rb') as input_:
@@ -274,6 +376,7 @@ def ffi_cut(target='', local_directory='', size=90, sector=None, limit_mag=None,
         print('Loaded ffi_cut from directory. ')
     else:
         with open(f'{local_directory}source/{source_name}.pkl', 'wb') as output:
-            source = Source_cut(target, size=size, sector=sector, limit_mag=limit_mag, transient=transient)
+            source = Source_cut(target, size=size, sector=sector, limit_mag=limit_mag, transient=transient, ffi=ffi,
+                                mast_timeout=mast_timeout)
             pickle.dump(source, output, pickle.HIGHEST_PROTOCOL)
     return source