arvi 0.1.12__py3-none-any.whl → 0.1.14__py3-none-any.whl

arvi/dace_wrapper.py

@@ -1,6 +1,7 @@
 import os
 import tarfile
 import collections
+from functools import lru_cache
 import numpy as np
 from dace_query import DaceClass
 from dace_query.spectroscopy import SpectroscopyClass, Spectroscopy as default_Spectroscopy
@@ -20,6 +21,17 @@ def load_spectroscopy() -> SpectroscopyClass:
     # elif os.path.exists(os.path.expanduser('~/.dacerc')):
     return default_Spectroscopy
 
+@lru_cache()
+def get_dace_id(star):
+    filters = {"obj_id_catname": {"contains": [star]}}
+    try:
+        with stdout_disabled(), all_logging_disabled():
+            r = load_spectroscopy().query_database(filters=filters, limit=1)
+        return r['obj_id_daceid'][0]
+    except KeyError:
+        logger.error(f"Could not find DACE ID for {star}")
+        raise ValueError from None
+
 def get_arrays(result, latest_pipeline=True, ESPRESSO_mode='HR11', NIRPS_mode='HE', verbose=True):
     arrays = []
     instruments = list(result.keys())
@@ -39,7 +51,7 @@ def get_arrays(result, latest_pipeline=True, ESPRESSO_mode='HR11', NIRPS_mode='H
                 i = [i for i, pipe in enumerate(pipelines) if ESPRESSO_mode in pipe][0]
                 pipelines = [pipelines[i]]
             else:
-                if verbose:
+                if len(pipelines) > 1 and verbose:
                     logger.warning(f'no observations for requested ESPRESSO mode ({ESPRESSO_mode})')
 
         if latest_pipeline:
@@ -73,19 +85,118 @@ def get_arrays(result, latest_pipeline=True, ESPRESSO_mode='HR11', NIRPS_mode='H
 
     return arrays
 
-def get_observations(star, instrument=None, verbose=True):
-    Spectroscopy = load_spectroscopy()
+def get_observations_from_instrument(star, instrument, main_id=None):
+    """ Query DACE for all observations of a given star and instrument
+
+    Args:
+        star (str): name of the star
+        instrument (str): instrument name
+        main_id (str, optional): Simbad main id of target to query DACE id. Defaults to None.
+
+    Raises:
+        ValueError: If query for DACE id fails
+
+    Returns:
+        dict: dictionary with data from DACE
+    """
     try:
-        with stdout_disabled(), all_logging_disabled():
-            result = Spectroscopy.get_timeseries(target=star,
-                                                sorted_by_instrument=True,
-                                                output_format='numpy')
-    except TypeError:
-        if instrument is None:
-            msg = f'no observations for {star}'
+        dace_id = get_dace_id(star)
+    except ValueError as e:
+        if main_id is not None:
+            dace_id = get_dace_id(main_id)
         else:
+            raise e
+
+    Spectroscopy = load_spectroscopy()
+    filters = {
+        "ins_name": {"contains": [instrument]}, 
+        "obj_id_daceid": {"contains": [dace_id]}
+    }
+    with stdout_disabled(), all_logging_disabled():
+        result = Spectroscopy.query_database(filters=filters)
+    
+    if len(result) == 0:
+        raise ValueError
+
+    r = {}
+    for inst in np.unique(result['ins_name']):
+        mask1 = result['ins_name'] == inst
+        r[inst] = {}
+        for pipe in np.unique(result['ins_drs_version'][mask1]):
+            mask2 = mask1 & (result['ins_drs_version'] == pipe)
+            ins_mode = np.unique(result['ins_mode'][mask2])[0]
+            _nan = np.full(mask2.sum(), np.nan)
+            r[inst][pipe] = {
+                ins_mode: {
+                    'texp': result['texp'][mask2],
+                    'bispan': result['spectro_ccf_bispan'][mask2],
+                    'bispan_err': result['spectro_ccf_bispan_err'][mask2],
+                    'drift_noise': result['spectro_cal_drift_noise'][mask2],
+                    'rjd': result['obj_date_bjd'][mask2],
+                    'cal_therror': _nan,
+                    'fwhm': result['spectro_ccf_fwhm'][mask2],
+                    'fwhm_err': result['spectro_ccf_fwhm_err'][mask2],
+                    'rv': result['spectro_ccf_rv'][mask2],
+                    'rv_err': result['spectro_ccf_rv_err'][mask2],
+                    'berv': result['spectro_cal_berv'][mask2],
+                    'ccf_noise': _nan,
+                    'rhk': result['spectro_analysis_rhk'][mask2],
+                    'rhk_err': result['spectro_analysis_rhk_err'][mask2],
+                    'contrast': result['spectro_ccf_contrast'][mask2],
+                    'contrast_err': result['spectro_ccf_contrast_err'][mask2],
+                    'cal_thfile': result['spectro_cal_thfile'][mask2],
+                    'spectroFluxSn50': result['spectro_flux_sn50'][mask2],
+                    'protm08': result['spectro_analysis_protm08'][mask2],
+                    'protm08_err': result['spectro_analysis_protm08_err'][mask2],
+                    'caindex': result['spectro_analysis_ca'][mask2],
+                    'caindex_err': result['spectro_analysis_ca_err'][mask2],
+                    'pub_reference': result['pub_ref'][mask2],
+                    'drs_qc': result['spectro_drs_qc'][mask2],
+                    'haindex': result['spectro_analysis_halpha'][mask2],
+                    'haindex_err': result['spectro_analysis_halpha_err'][mask2],
+                    'protn84': result['spectro_analysis_protn84'][mask2],
+                    'protn84_err': result['spectro_analysis_protn84_err'][mask2],
+                    'naindex': result['spectro_analysis_na'][mask2],
+                    'naindex_err': result['spectro_analysis_na_err'][mask2],
+                    'snca2': _nan,
+                    'mask': result['spectro_ccf_mask'][mask2],
+                    'public': result['public'][mask2],
+                    'spectroFluxSn20': result['spectro_flux_sn20'][mask2],
+                    'sindex': result['spectro_analysis_smw'][mask2],
+                    'sindex_err': result['spectro_analysis_smw_err'][mask2],
+                    'drift_used': _nan,
+                    'ccf_asym': result['spectro_ccf_asym'][mask2],
+                    'ccf_asym_err': result['spectro_ccf_asym_err'][mask2],
+                    'date_night': result['date_night'][mask2],
+                    'raw_file': result['file_rootpath'][mask2],
+                    'prog_id': result['prog_id'][mask2],
+                    'th_ar': result['th_ar'][mask2],
+                    'th_ar1': result['th_ar1'][mask2],
+                    'th_ar2': result['th_ar2'][mask2],
+                }
+            }
+    return r
+
+def get_observations(star, instrument=None, main_id=None, verbose=True):
+    if instrument is None:
+        Spectroscopy = load_spectroscopy()
+        try:
+            with stdout_disabled(), all_logging_disabled():
+                result = Spectroscopy.get_timeseries(target=star,
+                                                     sorted_by_instrument=True,
+                                                     output_format='numpy')
+        except TypeError:
+            if instrument is None:
+                msg = f'no observations for {star}'
+            else:
+                msg = f'no {instrument} observations for {star}'
+            raise ValueError(msg) from None
+    else:
+        try:
+            result = get_observations_from_instrument(star, instrument, main_id)
+        except ValueError:
             msg = f'no {instrument} observations for {star}'
-        raise ValueError(msg) from None
+            raise ValueError(msg) from None
 
     # defaultdict --> dict
     if isinstance(result, collections.defaultdict):
@@ -206,6 +317,49 @@ def extract_fits(output_directory):
     return files
 
 
+def do_symlink_filetype(type, raw_files, output_directory, clobber=False, top_level=None, verbose=True):
+    terminations = {
+        'CCF': '_CCF_A.fits',
+        'S1D': '_S1D_A.fits',
+        'S2D': '_S2D_A.fits',
+    }
+
+    create_directory(output_directory)
+
+    raw_files = np.atleast_1d(raw_files)
+
+    # check existing files
+    if not clobber:
+        raw_files = check_existing(output_directory, raw_files, type)
+
+    n = raw_files.size
+
+    # any file left?
+    if n == 0:
+        if verbose:
+            logger.info('no files to symlink')
+        return
+
+    if verbose:
+        msg = f"symlinking {n} {type}s into '{output_directory}'..."
+        logger.info(msg)
+
+    for file in tqdm(raw_files):
+        if top_level is not None:
+            top = file.split('/')[0] + '/'
+            if not top_level.endswith('/'):
+                top_level = top_level + '/'
+            file = file.replace(top, top_level)
+
+        file = file.replace('.fits', terminations[type])
+
+        if os.path.exists(file):
+            os.symlink(file, os.path.join(output_directory, os.path.basename(file)))
+            # print(file, os.path.join(output_directory, os.path.basename(file)))
+        else:
+            logger.warning(f'file not found: {file}')
+
+
 def do_download_filetype(type, raw_files, output_directory, clobber=False,
                          verbose=True, chunk_size=20):
     """ Download CCFs / S1Ds / S2Ds from DACE """

arvi/data/obs_affected_ADC_issues.dat

@@ -155,4 +155,7 @@
 2021-01-28      r.ESPRE.2021-01-29T01:46:34.785   106.21M2.002    2897966
 2021-01-28      r.ESPRE.2021-01-29T02:06:35.596   106.21M2.002    2899965
 2021-01-31      r.ESPRE.2021-02-01T01:34:18.507   106.21M2.001    2899996
-2019-12-22      r.ESPRE.2019-12-23T00:30:55.824   1104.C-0350(A)  2496531
+2019-12-22      r.ESPRE.2019-12-23T00:30:55.824   1104.C-0350(A)  2496531
+# additional frames with ADC2 absent or set to zero
+2021-04-06      r.ESPRE.2021-04-06T23:30:53.085   1104.C-0350     3026604
+2021-04-09      r.ESPRE.2021-04-10T00:40:18.132   1104.C-0350     3027219

arvi/gaia_wrapper.py

@@ -1,11 +1,9 @@
 import os
 from io import StringIO
 from csv import DictReader
-from dataclasses import dataclass, field
 import requests
 
 from astropy.coordinates import SkyCoord
-import pysweetcat
 
 DATA_PATH = os.path.dirname(__file__)
 DATA_PATH = os.path.join(DATA_PATH, 'data')
@@ -77,8 +75,11 @@ class gaia:
             results = parse_csv(table1)[0]
         except IndexError:
             raise ValueError(f'Gaia query for {star} failed')
-
-        self.dr3_id = int(results['source_id'])
+        
+        try:
+            self.dr3_id = int(results['source_id'])
+        except KeyError:
+            raise ValueError(f'Gaia query for {star} failed')
 
         self.ra = float(results['ra'])
         self.dec = float(results['dec'])
@@ -86,7 +87,10 @@ class gaia:
         self.pmdec = float(results['pmdec'])
         self.coords = SkyCoord(self.ra, self.dec, unit='deg')
         self.plx = float(results['parallax'])
-        self.radial_velocity = float(results['radial_velocity'])
+        try:
+            self.radial_velocity = float(results['radial_velocity'])
+        except ValueError:
+            self.radial_velocity = None
 
         return
 

arvi/headers.py

@@ -0,0 +1,47 @@
+from tqdm import tqdm
+from astropy.io import fits
+import iCCF
+
+from . import config
+
+def get_headers(self, check_lesta=False, lesta_username=config.username,
+                check_exo2=False, instrument=None):
+    try:
+        import paramiko
+    except ImportError:
+        raise ImportError("paramiko is not installed. Please install it with 'pip install paramiko'")
+
+    H = []
+
+    if check_lesta:
+        with paramiko.SSHClient() as ssh:
+            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+            ssh.connect("lesta02.astro.unige.ch", username=lesta_username)
+            sftp = ssh.open_sftp()
+
+            pbar = tqdm(self.raw_file, total=len(self.raw_file), unit='file', desc='Reading headers')
+            for f in pbar:
+                f = f.replace('espresso/', '/projects/astro/ESPRESSODRS/')
+                f = f.replace('nirps/', '/projects/astro/NIRPSDRS/')
+                f = f.replace('.fits', '_CCF_A.fits')#.replace(':', r'\:')
+                with sftp.open(f) as fp:
+                    header = fits.getheader(fp)
+                    H.append(header)
+
+    if len(H) == 0 and check_exo2:
+        raise NotImplementedError('getting headers from exo2 not yet implemented')
+
+    if len(H) == 0:
+        self.download_ccf()
+        if instrument is None:
+            I = iCCF.from_file(f'{self.star}_downloads/*CCF_A.fits')
+        else:
+            I = iCCF.from_file(f'{self.star}_downloads/r.{instrument}.*CCF_A.fits',
+                               guess_instrument='HARPS' not in instrument)
+        H = [i.HDU[0].header for i in I]
+
+    # sort by BJD
+    H = sorted(H, key=lambda x: x['HIERARCH ESO QC BJD'])
+
+    return H
+

arvi/instrument_specific.py

@@ -1,17 +1,19 @@
 import os, sys
 import numpy as np
-import matplotlib.pyplot as plt
 
 from .setup_logger import logger
 
 # ESPRESSO ADC issues
 from .utils import ESPRESSO_ADC_issues
 
-def ADC_issues(self, plot=True):
+def ADC_issues(self, plot=True, check_headers=False):
     """ Identify and mask points affected by ADC issues (ESPRESSO).
 
     Args:
-        plot (bool, optional): Whether to plot the masked points.
+        plot (bool, optional):
+            Whether to plot the masked points.
+        check_headers (bool, optional):
+            Whether to (double-)check the headers for missing/zero keywords.
     """
     instruments = self._check_instrument('ESPRESSO')
     
@@ -25,6 +27,14 @@ def ADC_issues(self, plot=True):
     file_roots = [os.path.basename(f).replace('.fits', '') for f in self.raw_file]
     intersect = np.in1d(file_roots, affected_file_roots)
 
+    if check_headers:
+        from .headers import get_headers
+        H = get_headers(self, check_lesta=False, check_exo2=False, instrument='ESPRE')
+        badACD2 = np.array([h['*ADC2 RA'][0] for h in H]) == 0
+        badACD2 |= np.array([h['*ADC2 SENS1'][0] for h in H]) == 0
+        badACD2 |= np.array([h['*ADC2 TEMP'][0] for h in H]) == 0
+        intersect = np.logical_or(intersect, badACD2)
+
     total_affected = intersect.sum()
 
     if self.verbose:
@@ -76,6 +86,52 @@ def blue_cryostat_issues(self, plot=True):
     return intersect
 
 
+def qc_scired_issues(self, plot=False, **kwargs):
+    """ Identify and mask points with failed SCIRED QC
+
+    Args:
+        plot (bool, optional): Whether to plot the masked points.
+    """
+    from .headers import get_headers
+
+    instruments = self._check_instrument('ESPRESSO')
+    
+    if instruments is None:
+        if self.verbose:
+            logger.error(f"no data from ESPRESSO")
+            logger.info(f'available: {self.instruments}')
+        return
+
+    H = kwargs.get('H', None)
+    if H is None:
+        H = get_headers(self, check_lesta=False, check_exo2=False, instrument='ESPRE')
+    if len(H) == 0:
+        if self.verbose:
+            logger.warning('this function requires access to headers, but none found')
+            logger.warning('trying to download')
+        self.download_ccf()
+        H = get_headers(self, check_lesta=False, check_exo2=False, instrument='ESPRE')
+    
+    scired_check = np.array([h['HIERARCH ESO QC SCIRED CHECK'] for h in H])
+    affected = scired_check == 0
+    n = affected.sum()
+
+    if self.verbose:
+        logger.info(f"there {'are'[:n^1]}{'is'[n^1:]} {n} frame{'s'[:n^1]} "
+                     "where QC SCIRED CHECK is 0")
+
+    if n == 0:
+        return
+
+    self.mask[affected] = False
+    self._propagate_mask_changes()
+
+    if plot:
+        self.plot(show_masked=True)
+
+    return affected
+
+
 def known_issues(self, plot=False, **kwargs):
     """ Identify and mask known instrumental issues (ADC and blue cryostat for ESPRESSO)
 

arvi/nasaexo_wrapper.py

@@ -125,7 +125,7 @@ class Planets:
             self.model.keplerian.uid = 0
 
         for i in range(self.np):
-            if self.K[i] == False or np.isnan(self.K[i]):
+            if not self.K[i] or np.isnan(self.K[i]):
                 self.model.add_keplerian_from_period(self.P[i], fit=True)
             else:
                 self.model.add_keplerian([self.P[i], self.K[i], self.e[i], 0.0, self.w[i]],