arvi 0.2.4__tar.gz → 0.2.11__tar.gz

{arvi-0.2.4 → arvi-0.2.11}/.github/workflows/docs-gh-pages.yml

@@ -26,13 +26,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         
       - name: Setup Pages
         uses: actions/configure-pages@v5
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.10"
   

{arvi-0.2.4 → arvi-0.2.11}/.github/workflows/install.yml

@@ -16,9 +16,9 @@ jobs:
         python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: ${{ matrix.python-version }}
 

{arvi-0.2.4 → arvi-0.2.11}/.github/workflows/python-publish.yml

@@ -22,9 +22,9 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
     - name: Set up Python
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: '3.x'
     - name: Install dependencies
@@ -34,7 +34,7 @@ jobs:
     - name: Build package
       run: python -m build
     - name: Publish package
-      uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
+      uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e
       with:
         user: __token__
         password: ${{ secrets.PYPI_API_TOKEN }}

{arvi-0.2.4/arvi.egg-info → arvi-0.2.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: arvi
-Version: 0.2.4
+Version: 0.2.11
 Summary: The Automated RV Inspector
 Author-email: João Faria <joao.faria@unige.ch>
 License: MIT

{arvi-0.2.4 → arvi-0.2.11}/arvi/dace_wrapper.py

@@ -2,15 +2,15 @@ import os
 import sys
 import tarfile
 import collections
-from functools import lru_cache
+from functools import lru_cache, partial
 from itertools import islice
 import numpy as np
 
 from .setup_logger import setup_logger
-from .utils import create_directory, all_logging_disabled, stdout_disabled, tqdm
+from .utils import create_directory, all_logging_disabled, stdout_disabled, timer, tqdm
 
 
-def load_spectroscopy(user=None):
+def load_spectroscopy(user=None, verbose=True):
     logger = setup_logger()
     with all_logging_disabled():
         from dace_query.spectroscopy import SpectroscopyClass, Spectroscopy as default_Spectroscopy
@@ -19,7 +19,8 @@ def load_spectroscopy(user=None):
     from .config import config
     # requesting as public
     if config.request_as_public:
-        logger.warning('requesting DACE data as public')
+        if verbose:
+            logger.warning('requesting DACE data as public')
         with all_logging_disabled():
             dace = DaceClass(dace_rc_config_path='none')
         return SpectroscopyClass(dace_instance=dace)
@@ -86,11 +87,16 @@ def get_arrays(result, latest_pipeline=True, ESPRESSO_mode='HR11', NIRPS_mode='H
                 i = [i for i, pipe in enumerate(pipelines) if ESPRESSO_mode in pipe][0]
                 pipelines = [pipelines[i]]
 
+        # select NIRPS mode
+        if 'NIRPS' in inst:
+            if any(this_mode := [p for p in pipelines if NIRPS_mode in p]):
+                pipelines = this_mode
+
         if latest_pipeline:
             npipe = len(pipelines)
             if 'NIRPS' in inst and any(['LBL' in p for p in pipelines]):
                 # TODO: correctly load both CCF and LBL
-                pipelines = [pipelines[1]]
+                pipelines = [pipelines[0]]
             if 'HARPS' in inst and npipe > 1 and pipelines[1] == pipelines[0] + '-EGGS':
                 pipelines = pipelines[:2]
             else:
@@ -163,20 +169,20 @@ def get_observations_from_instrument(star, instrument, user=None, main_id=None,
         dict:
             dictionary with data from DACE
     """
-    Spectroscopy = load_spectroscopy(user)
-
+    Spectroscopy = load_spectroscopy(user, verbose)
     found_dace_id = False
-    try:
-        dace_id = get_dace_id(star, verbose=verbose, raise_error=True)
-        found_dace_id = True
-    except ValueError as e:
-        if main_id is not None:
-            try:
-                dace_id = get_dace_id(main_id, verbose=verbose, raise_error=True)
-                found_dace_id = True
-            except ValueError:
-                pass
-    
+    with timer('dace_id query'):
+        try:
+            dace_id = get_dace_id(star, verbose=verbose, raise_error=True)
+            found_dace_id = True
+        except ValueError as e:
+            if main_id is not None:
+                try:
+                    dace_id = get_dace_id(main_id, verbose=verbose, raise_error=True)
+                    found_dace_id = True
+                except ValueError:
+                    pass
+
     if not found_dace_id:
         try:
             with all_logging_disabled():
@@ -188,10 +194,16 @@ def get_observations_from_instrument(star, instrument, user=None, main_id=None,
             msg = f'no {instrument} observations for {star}'
             raise ValueError(msg) from None
 
-    filters = {
-        "ins_name": {"contains": [instrument]},
-        "obj_id_daceid": {"contains": [dace_id]}
-    }
+    if (isinstance(instrument, str)):
+        filters = {
+            "ins_name": {"contains": [instrument]},
+            "obj_id_daceid": {"contains": [dace_id]}
+        }
+    elif (isinstance(instrument, (list, tuple, np.ndarray))):
+        filters = {
+            "ins_name": {"contains": instrument},
+            "obj_id_daceid": {"contains": [dace_id]}
+        }
     with all_logging_disabled():
         result = Spectroscopy.query_database(filters=filters)
     
@@ -202,17 +214,22 @@ def get_observations_from_instrument(star, instrument, user=None, main_id=None,
 
     for inst in np.unique(result['ins_name']):
         mask1 = result['ins_name'] == inst
-        r[inst] = {}
+        r[str(inst)] = {}
+
+        key2 = 'ins_drs_version'
+        n_key2 = len(np.unique(result[key2][mask1]))
+        if len(np.unique(result['pub_bibcode'][mask1])) >= n_key2:
+            key2 = 'pub_bibcode'
 
-        for pipe in np.unique(result['ins_drs_version'][mask1]):
-            mask2 = mask1 & (result['ins_drs_version'] == pipe)
-            r[inst][pipe] = {}
+        for pipe in np.unique(result[key2][mask1]):
+            mask2 = mask1 & (result[key2] == pipe)
+            r[str(inst)][str(pipe)] = {}
 
             for ins_mode in np.unique(result['ins_mode'][mask2]):
                 mask3 = mask2 & (result['ins_mode'] == ins_mode)
                 _nan = np.full(mask3.sum(), np.nan)
 
-                r[inst][pipe][ins_mode] = {
+                r[str(inst)][str(pipe)][str(ins_mode)] = {
                     'texp': result['texp'][mask3],
                     'bispan': result['spectro_ccf_bispan'][mask3],
                     'bispan_err': result['spectro_ccf_bispan_err'][mask3],
@@ -224,7 +241,9 @@ def get_observations_from_instrument(star, instrument, user=None, main_id=None,
                     'rv': result['spectro_ccf_rv'][mask3],
                     'rv_err': result['spectro_ccf_rv_err'][mask3],
                     'berv': result['spectro_cal_berv'][mask3],
-                    'ccf_noise': _nan,
+                    'ccf_noise': np.sqrt(
+                        np.square(result['spectro_ccf_rv_err'][mask3]) - np.square(result['spectro_cal_drift_noise'][mask3])
+                    ),
                     'rhk': result['spectro_analysis_rhk'][mask3],
                     'rhk_err': result['spectro_analysis_rhk_err'][mask3],
                     'contrast': result['spectro_ccf_contrast'][mask3],
@@ -268,7 +287,7 @@ def get_observations_from_instrument(star, instrument, user=None, main_id=None,
 def get_observations(star, instrument=None, user=None, main_id=None, verbose=True):
     logger = setup_logger()
     if instrument is None:
-        Spectroscopy = load_spectroscopy(user)
+        Spectroscopy = load_spectroscopy(user, verbose)
 
         try:
             with stdout_disabled(), all_logging_disabled():
@@ -302,12 +321,14 @@ def get_observations(star, instrument=None, user=None, main_id=None, verbose=Tru
             result[inst] = dict(result[inst])
     #
 
-    instruments = list(result.keys())
+    instruments = list(map(str, result.keys()))
 
     if instrument is not None:
         # select only the provided instrument (if it's there)
-        instruments = [inst for inst in instruments if instrument in inst]
-
+        if (isinstance(instrument, str)):
+            instruments = [inst for inst in instruments if instrument in inst]
+        elif (isinstance(instrument, list)):
+            instruments = [inst for inst in instruments if any(i in inst for i in instrument)]
     if len(instruments) == 0:
         if instrument is None:
             msg = f'no observations for {star}'
@@ -331,30 +352,57 @@ def get_observations(star, instrument=None, user=None, main_id=None, verbose=Tru
     #     else:
     #         return -1
 
-    # sort pipelines, must be extra careful with HARPS/HARPN pipeline version numbers
-    # got here with the help of DeepSeek
-    from re import match
-    def custom_sort_key(s):
-        s = s[0]
-        # Check for version number pattern (e.g., 3.2.5 or 3.2.5-EGGS)
-        version_match = match(r'^(\d+(?:\.\d+)*)(?:[-\s](.*))?$', s)
-        if version_match:
-            version_parts = list(map(int, version_match.group(1).split('.')))
-            if len(version_parts) == 2:
-                version_parts.insert(1, -1)
-            return (0, 1, version_parts)
-        # Check for scientific reference pattern (e.g., 2004A&A...)
-        year_match = match(r'^(\d{4})', s)
-        if year_match:
-            year = int(year_match.group(1))
-            return (1, year)
-        # For all other strings, sort alphabetically
-        return (2, s)
-
-    # from functools import cmp_to_key
+    # # sort pipelines, must be extra careful with HARPS/HARPN pipeline version numbers
+    # # got here with the help of DeepSeek
+    # # from functools import cmp_to_key
+    # from re import match
+    # def custom_sort_key(s):
+    #     s = s[0]
+    #     # Check for version number pattern (e.g., 3.2.5 or 3.2.5-EGGS)
+    #     version_match = match(r'^(\d+(?:\.\d+)*)(?:[-\s](.*))?$', s)
+    #     if version_match:
+    #         version_parts = list(map(int, version_match.group(1).split('.')))
+    #         if len(version_parts) == 2:
+    #             version_parts.insert(1, -1)
+    #         # if version_match.group(2) and 'LBL' in version_match.group(2):
+    #         #     version_parts.append(-1)
+    #         # else:
+    #         #     version_parts.append(0)
+    #         if version_match.group(2) is None:
+    #             version_parts.append('')
+    #         else:
+    #             version_parts.append(version_match.group(2))
+    #         return (0, 1, version_parts)
+    #     # Check for scientific reference pattern (e.g., 2004A&A...)
+    #     year_match = match(r'^(\d{4})', s)
+    #     if year_match:
+    #         year = int(year_match.group(1))
+    #         return (1, year)
+    #     # For all other strings, sort alphabetically
+    #     return (2, s)
+
+    def custom_key(val, strip_EGGS=False):
+        if strip_EGGS:
+            val = val.replace('-EGGS', '').replace(' EGGS', '')
+        key = 0
+        key -= 1 if '3.5' in val else 0
+        key -= 1 if 'EGGS' in val else 0
+        key -= 1 if ('UHR' in val or 'MR' in val) else 0
+        key -= 1 if 'LBL' in val else 0
+        return str(key) if key != 0 else val
+
     new_result = {}
     for inst in instruments:
-        new_result[inst] = dict(sorted(result[inst].items(), key=custom_sort_key, reverse=True))
+        # new_result[inst] = dict(
+        #     sorted(result[inst].items(), key=custom_sort_key, reverse=True)
+        # )
+        if all(['EGGS' in k for k in result[inst].keys()]):
+            custom_key = partial(custom_key, strip_EGGS=True)
+        # WARNING: not the same as reverse=True (not sure why)
+        sorted_keys = sorted(result[inst].keys(), key=custom_key)[::-1]
+        new_result[inst] = {}
+        for key in sorted_keys:
+            new_result[inst][key] = result[inst][key]
 
     if verbose:
         logger.info('RVs available from')
@@ -388,7 +436,7 @@ def check_existing(output_directory, files, type):
     ]
 
     if type == 'S2D':
-        existing = [
+        existing += [
             f.partition('.fits')[0] for f in os.listdir(output_directory)
             if 'e2ds' in f
         ]   
@@ -500,20 +548,21 @@ def do_download_filetype(type, raw_files, output_directory, clobber=False, user=
     """ Download CCFs / S1Ds / S2Ds from DACE """
     logger = setup_logger()
     raw_files = np.atleast_1d(raw_files)
+    raw_files_original = raw_files.copy()
 
     create_directory(output_directory)
 
     # check existing files to avoid re-downloading
     if not clobber:
         raw_files = check_existing(output_directory, raw_files, type)
-
+    
     n = raw_files.size
 
     # any file left to download?
     if n == 0:
         if verbose:
             logger.info('no files to download')
-        return
+        return list(map(os.path.basename, raw_files_original))
 
     # avoid an empty chunk
     if chunk_size > n:
@@ -531,7 +580,9 @@ def do_download_filetype(type, raw_files, output_directory, clobber=False, user=
 
     if n < parallel_limit:
         iterator = [raw_files[i:i + chunk_size] for i in range(0, n, chunk_size)]
-        for files in tqdm(iterator, total=len(iterator)):
+        if len(iterator) > 1:
+            iterator = tqdm(iterator, total=len(iterator))
+        for files in iterator:
             download(files, type, output_directory, quiet=False, user=user)
             extract_fits(output_directory)
 
@@ -560,6 +611,7 @@ def do_download_filetype(type, raw_files, output_directory, clobber=False, user=
 
     sys.stdout.flush()
     logger.info('extracted .fits files')
+    return list(map(os.path.basename, raw_files_original))
 
 
 # def do_download_s1d(raw_files, output_directory, clobber=False, verbose=True):

{arvi-0.2.4 → arvi-0.2.11}/arvi/gaia_wrapper.py

@@ -3,8 +3,6 @@ from io import StringIO
 from csv import DictReader
 import requests
 
-from astropy.coordinates import SkyCoord
-
 DATA_PATH = os.path.dirname(__file__)
 DATA_PATH = os.path.join(DATA_PATH, 'data')
 
@@ -78,6 +76,8 @@ class gaia:
         Args:
             star (str): The name of the star to query simbad
         """
+        from astropy.coordinates import SkyCoord
+
         self.star = star
 
         if simbad is None:

{arvi-0.2.4 → arvi-0.2.11}/arvi/instrument_specific.py

@@ -5,7 +5,7 @@ from .setup_logger import setup_logger
 from .utils import ESPRESSO_ADC_issues, ESPRESSO_cryostat_issues
 
 
-# HARPS started operations in October 1st, 2003
+# HARPS started operations on October 1st, 2003
 # https://www.eso.org/sci/facilities/lasilla/instruments/harps/news.html
 HARPS_start = 52913
 
@@ -21,6 +21,11 @@ HARPS_technical_intervention = 57170
 # when the instrument was handed back to Science Operations.
 HARPS_technical_intervention_range = (57161, 57176)
 
+
+# ESPRESSO started operations on October 1st, 2018
+# see Pepe et al. (2021, A&A 645, A96)
+ESPRESSO_start = 58392
+
 # ESPRESSO fiber link upgrade (1 July 2019)
 ESPRESSO_technical_intervention = 58665
 
@@ -102,18 +107,11 @@ def divide_HARPS(self):
     
 
 def check(self, instrument):
-    logger = setup_logger()
-    instruments = self._check_instrument(instrument)
-    if instruments is None:
-        if self.verbose:
-            logger.error(f"HARPS_fiber_commissioning: no data from {instrument}")
-        return None
-    return instruments
-
+    return self._check_instrument(instrument)
 
 # HARPS commissioning
 def HARPS_commissioning(self, mask=True, plot=True):
-    """ Identify and optionally mask points during HARPS commissioning (HARPS).
+    """ Identify and optionally mask points during HARPS commissioning.
 
     Args:
         mask (bool, optional):
@@ -125,17 +123,20 @@ def HARPS_commissioning(self, mask=True, plot=True):
     if check(self, 'HARPS') is None:
         return
 
-    affected = self.time < HARPS_start
+    affected = np.logical_and(
+        self.instrument_array == 'HARPS03', 
+        self.time < HARPS_start
+    )
     total_affected = affected.sum()
 
     if self.verbose:
-        n = total_affected
-        logger.info(f"there {'are'[:n^1]}{'is'[n^1:]} {n} frame{'s'[:n^1]} "
-                     "during HARPS commissioning")
+        n, i = total_affected, int(total_affected != 1)
+        logger.info(f"there {['is', 'are'][i]} {n} frame{['', 's'][i]} "
+                    "during HARPS commissioning")
 
     if mask:
         self.mask[affected] = False
-        self._propagate_mask_changes()
+        self._propagate_mask_changes(_remove_instrument=False)
 
         if plot:
             self.plot(show_masked=True)
@@ -157,17 +158,24 @@ def HARPS_fiber_commissioning(self, mask=True, plot=True):
     if check(self, 'HARPS') is None:
         return
 
-    affected = (self.time >= HARPS_technical_intervention_range[0]) & (self.time <= HARPS_technical_intervention_range[1])
+    affected = np.logical_and(
+        self.time >= HARPS_technical_intervention_range[0],
+        self.time <= HARPS_technical_intervention_range[1]
+    )
+    affected = np.logical_and(
+        affected,
+        np.char.find(self.instrument_array, 'HARPS') == 0
+    )
     total_affected = affected.sum()
 
     if self.verbose:
-        n = total_affected
-        logger.info(f"there {'are'[:n^1]}{'is'[n^1:]} {n} frame{'s'[:n^1]} "
-                     "during the HARPS fiber commissioning period")
+        n, i = total_affected, int(total_affected != 1)
+        logger.info(f"there {['is', 'are'][i]} {n} frame{['', 's'][i]} "
+                    "during the HARPS fiber commissioning period")
 
     if mask:
         self.mask[affected] = False
-        self._propagate_mask_changes()
+        self._propagate_mask_changes(_remove_instrument=False)
 
         if plot:
             self.plot(show_masked=True)
@@ -175,6 +183,41 @@ def HARPS_fiber_commissioning(self, mask=True, plot=True):
     return affected
 
 
+# ESPRESSO commissioning
+def ESPRESSO_commissioning(self, mask=True, plot=True):
+    """ Identify and optionally mask points during ESPRESSO commissioning.
+
+    Args:
+        mask (bool, optional):
+            Whether to mask out the points.
+        plot (bool, optional):
+            Whether to plot the masked points.
+    """
+    logger = setup_logger()
+    if check(self, 'ESPRESSO') is None:
+        return
+
+    affected = np.logical_and(
+        self.instrument_array == 'ESPRESSO18',
+        self.time < ESPRESSO_start
+    )
+    total_affected = affected.sum()
+
+    if self.verbose:
+        n, i = total_affected, int(total_affected != 1)
+        logger.info(f"there {['is', 'are'][i]} {n} frame{['', 's'][i]} "
+                    "during ESPRESSO commissioning")
+
+    if mask:
+        self.mask[affected] = False
+        self._propagate_mask_changes(_remove_instrument=False)
+
+        if plot and total_affected > 0:
+            self.plot(show_masked=True)
+
+    return affected
+
+
 # ESPRESSO ADC issues
 def ADC_issues(self, mask=True, plot=True, check_headers=False):
     """ Identify and optionally mask points affected by ADC issues (ESPRESSO).
@@ -210,13 +253,13 @@ def ADC_issues(self, mask=True, plot=True, check_headers=False):
     total_affected = intersect.sum()
 
     if self.verbose:
-        n = total_affected
-        logger.info(f"there {'are'[:n^1]}{'is'[n^1:]} {n} frame{'s'[:n^1]} "
-                     "affected by ADC issues")
+        n, i = total_affected, int(total_affected != 1)
+        logger.info(f"there {['is', 'are'][i]} {n} frame{['', 's'][i]} "
+                    "affected by ADC issues")
 
     if mask:
         self.mask[intersect] = False
-        self._propagate_mask_changes()
+        self._propagate_mask_changes(_remove_instrument=False)
 
         if plot:
             self.plot(show_masked=True)
@@ -246,13 +289,13 @@ def blue_cryostat_issues(self, mask=True, plot=True):
     total_affected = intersect.sum()
 
     if self.verbose:
-        n = total_affected
-        logger.info(f"there {'are'[:n^1]}{'is'[n^1:]} {n} frame{'s'[:n^1]} "
-                     "affected by blue cryostat issues")
+        n, i = total_affected, int(total_affected != 1)
+        logger.info(f"there {['is', 'are'][i]} {n} frame{['', 's'][i]} "
+                    "affected by blue cryostat issues")
 
     if mask:
         self.mask[intersect] = False
-        self._propagate_mask_changes()
+        self._propagate_mask_changes(_remove_instrument=False)
 
         if plot:
             self.plot(show_masked=True)
@@ -292,14 +335,15 @@ def qc_scired_issues(self, plot=False, **kwargs):
     n = affected.sum()
 
     if self.verbose:
-        logger.info(f"there {'are'[:n^1]}{'is'[n^1:]} {n} frame{'s'[:n^1]} "
-                     "where QC SCIRED CHECK is 0")
+        i = int(n != 1)
+        logger.info(f"there {['is', 'are'][i]} {n} frame{['', 's'][i]} "
+                    "where QC SCIRED CHECK is 0")
 
     if n == 0:
         return
 
     self.mask[affected] = False
-    self._propagate_mask_changes()
+    self._propagate_mask_changes(_remove_instrument=False)
 
     if plot:
         self.plot(show_masked=True)
@@ -316,31 +360,26 @@ class ISSUES:
             plot (bool, optional): Whether to plot the masked points.
         """
         logger = setup_logger()
-        try:
-            adc = ADC_issues(self, mask, plot, **kwargs)
-        except IndexError:
-            logger.error('are the data binned? cannot proceed to mask these points...')
-
-        try:
-            cryostat = blue_cryostat_issues(self, mask, plot)
-        except IndexError:
-            logger.error('are the data binned? cannot proceed to mask these points...')
 
-        try:
-            harps_comm = HARPS_commissioning(self, mask, plot)
-        except IndexError:
-            logger.error('are the data binned? cannot proceed to mask these points...')
-
-        try:
-            harps_fibers = HARPS_fiber_commissioning(self, mask, plot)
-        except IndexError:
-            logger.error('are the data binned? cannot proceed to mask these points...')
-
-        # if None in (adc, cryostat, harps_comm, harps_fibers):
-        #     return
+        functions = (
+            ESPRESSO_commissioning,
+            ADC_issues,
+            blue_cryostat_issues,
+            HARPS_commissioning,
+            HARPS_fiber_commissioning
+        )
+        results = []
+
+        for fun in functions:
+            try:
+                results.append(fun(self, mask, plot, **kwargs))
+            except IndexError:
+                logger.error('are the data binned? cannot proceed to mask these points...')
+        
+        results = list(filter(lambda x: x is not None, results))
+        self._propagate_mask_changes()
 
         try:
-            # return adc | cryostat
-            return np.logical_or.reduce((adc, cryostat, harps_comm, harps_fibers))
+            return np.logical_or.reduce(results)
         except UnboundLocalError:
             return