arvi 0.2.8__py3-none-any.whl → 0.2.11__py3-none-any.whl

arvi/timeseries.py

@@ -4,8 +4,9 @@ from typing import Union
 from functools import partial, partialmethod
 from glob import glob
 import warnings
-from copy import deepcopy
+from copy import copy, deepcopy
 from datetime import datetime, timezone
+
 import numpy as np
 
 from .setup_logger import setup_logger
@@ -24,10 +25,12 @@ from .HZ import getHZ_period
 from .instrument_specific import ISSUES
 from .reports import REPORTS
 from .utils import sanitize_path, strtobool, there_is_internet, timer, chdir
-from .utils import lazy_import
+from .setup_logger import setup_logger
+logger = setup_logger()
 
-units = lazy_import('astropy.units')
-# from astropy import units
+# units = lazy_import('astropy.units')
+# units = lazy.load('astropy.units')
+from astropy import units
 
 class ExtraFields:
     @property
@@ -408,35 +411,71 @@ class RV(ISSUES, REPORTS):
         self._did_correct_berv = False
         self.__post_init__()
 
-    def snapshot(self, directory=None, delete_others=False):
-        import pickle
+    def snapshot(self, directory=None, delete_others=False, compress=False):
+        if compress:
+            try:
+                import compress_pickle as pickle
+            except ImportError:
+                logger.warning('compress_pickle not installed, not compressing')
+                import pickle
+                compress = False
+        else:
+            import pickle
+        import re
         from datetime import datetime
+
         ts = datetime.now().timestamp()
         star_name = self.star.replace(' ', '')
         file = f'{star_name}_{ts}.pkl'
 
+        server = None
         if directory is None:
             directory = '.'
         else:
-            os.makedirs(directory, exist_ok=True)
-
-        file = os.path.join(directory, file)
-
-        if delete_others:
-            import re
-            other_pkls = [
-                f for f in os.listdir(directory)
-                if re.search(fr'{star_name}_\d+.\d+.pkl', f)
-            ]
-            for pkl in other_pkls:
-                os.remove(os.path.join(directory, pkl))
+            if ':' in directory:
+                server, directory = directory.split(':')
+                delete_others = False
+            else:
+                os.makedirs(directory, exist_ok=True)
 
         metadata = {
             'star': self.star,
             'timestamp': ts,
             'description': 'arvi snapshot'
         }
-        pickle.dump((self, metadata), open(file, 'wb'), protocol=0)
+
+
+        if server:
+            import posixpath
+            from .utils import server_sftp, server_file
+            with server_sftp(server=server) as sftp:
+                try:
+                    sftp.chdir(directory)
+                except FileNotFoundError:
+                    sftp.mkdir(directory)
+                finally:
+                    sftp.chdir(directory)
+                with sftp.open(file, 'wb') as f:
+                    print('saving snapshot to server...', end='', flush=True)
+                    pickle.dump((self, metadata), f, protocol=0)
+                    print('done')
+            file = posixpath.join(directory, file)
+        else:
+            if delete_others:
+                other_pkls = [
+                    f for f in os.listdir(directory)
+                    if re.search(fr'{star_name}_\d+.\d+.pkl', f)
+                ]
+                for pkl in other_pkls:
+                    os.remove(os.path.join(directory, pkl))
+
+            file = os.path.join(directory, file)
+
+            if compress:
+                file += '.gz'
+
+            with open(file, 'wb') as f:
+                pickle.dump((self, metadata), f)
 
         if self.verbose:
             logger.info(f'saved snapshot to {file}')
@@ -511,6 +550,15 @@ class RV(ISSUES, REPORTS):
     def instrument_array(self):
         return np.concatenate([[i] * n for i, n in self.NN.items()])
 
+    def _instrument_mask(self, instrument):
+        if isinstance(instrument, str):
+            return np.char.find(self.instrument_array, instrument) == 0
+        elif isinstance(instrument, (list, tuple, np.ndarray)):
+            m = np.full_like(self.time, False, dtype=bool)
+            for i in instrument:
+                m |= np.char.find(self.instrument_array, i) == 0
+            return m
+
     @property
     def rms(self) -> float:
         """ Weighted rms of the (masked) radial velocities """
@@ -537,6 +585,11 @@ class RV(ISSUES, REPORTS):
     def _mtime_sorter(self):
         return np.argsort(self.mtime)
 
+    @property
+    def timespan(self):
+        """ Total time span of the (masked) observations """
+        return np.ptp(self.mtime)
+
     def _index_from_instrument_index(self, index, instrument):
         ind = np.where(self.instrument_array == instrument)[0]
         return ind[getattr(self, instrument).mask][index]
@@ -577,7 +630,8 @@ class RV(ISSUES, REPORTS):
                     # --> not just in rhk and rhk_err...
                     if data[arr].dtype == float and (bad := data[arr] == -99999).any():
                         data[arr][bad] = np.nan
-
+                    if data[arr].dtype == float and (bad := data[arr] == -99).any():
+                        data[arr][bad] = np.nan
                     setattr(s, arr, data[arr][ind])
                     s._quantities.append(arr)
 
@@ -629,22 +683,28 @@ class RV(ISSUES, REPORTS):
         import pickle
         from datetime import datetime
         if star is None:
-            assert file.endswith('.pkl'), 'expected a .pkl file'
-            star, timestamp = file.replace('.pkl', '').split('_')
+            assert file.endswith(('.pkl', '.pkl.gz')), 'expected a .pkl file'
+            basefile = os.path.basename(file)
+            star, timestamp = basefile.replace('.pkl.gz', '').replace('.pkl', '').split('_')
         else:
             try:
-                file = sorted(glob(f'{star}_*.*.pkl'))[-1]
+                file = sorted(glob(f'{star}_*.*.pkl*'))[-1]
             except IndexError:
                 raise ValueError(f'cannot find any file matching {star}_*.pkl')
-            star, timestamp = file.replace('.pkl', '').split('_')
+            star, timestamp = file.replace('.pkl.gz', '').replace('.pkl', '').split('_')
 
         dt = datetime.fromtimestamp(float(timestamp))
         if verbose:
             logger.info(f'reading snapshot of {star} from {dt}')
 
-        s = pickle.load(open(file, 'rb'))
+        with open(file, 'rb') as f:
+            if file.endswith('.gz'):
+                import compress_pickle as pickle
+            s = pickle.load(f)
+
         if isinstance(s, tuple) and len(s) == 2:
             s, _metadata = s
+
         s._snapshot = file
         return s
 
@@ -1504,7 +1564,7 @@ class RV(ISSUES, REPORTS):
         """ Remove all observations that satisfy a condition
 
         Args:
-            condition (np.ndarray):
+            condition (ndarray):
                 Boolean array of the same length as the observations
         """
         if self.verbose:
@@ -1664,16 +1724,18 @@ class RV(ISSUES, REPORTS):
         self._propagate_mask_changes()
 
 
-    def _propagate_mask_changes(self):
+    def _propagate_mask_changes(self, _remove_instrument=True):
         """ link self.mask with each self.`instrument`.mask """
         masked = np.where(~self.mask)[0]
         for m in masked:
             inst = self.instruments[self.obs[m] - 1]
             n_before = (self.obs < self.obs[m]).sum()
             getattr(self, inst).mask[m - n_before] = False
-        for inst in self.instruments:
-            if getattr(self, inst).mtime.size == 0:
-                self.remove_instrument(inst, strict=True)
+        if _remove_instrument:
+            instruments = copy(self.instruments)
+            for inst in instruments:
+                if getattr(self, inst).mtime.size == 0:
+                    self.remove_instrument(inst, strict=True)
 
     def secular_acceleration(self, epoch=None, just_compute=False, force_simbad=False):
         """ 
@@ -1691,9 +1753,12 @@ class RV(ISSUES, REPORTS):
             force_simbad (bool, optional):
                 Use Simbad proper motions even if Gaia is available
         """
-        if self._did_secular_acceleration and not just_compute:  # don't do it twice
+        # don't do it twice
+        if self._did_secular_acceleration and not just_compute:
             return
 
+        from astropy import units
+
         #as_yr = units.arcsec / units.year
         mas_yr = units.milliarcsecond / units.year
         mas = units.milliarcsecond
@@ -1825,15 +1890,21 @@ class RV(ISSUES, REPORTS):
 
             self._did_secular_acceleration = False
 
-    def sigmaclip(self, sigma=5, instrument=None, strict=True):
+    def sigmaclip(self, sigma=5, quantity='vrad', instrument=None,
+                  strict=True):
         """
-        Sigma-clip RVs (per instrument!), by MAD away from the median.
+        Sigma-clip RVs or other quantities (per instrument!), by MAD away from
+        the median.
 
         Args:
             sigma (float):
-                Number of MADs to clip
+                Number of MADs away from the median
+            quantity (str):
+                Quantity to sigma-clip (by default the RVs)
             instrument (str, list):
                 Instrument(s) to sigma-clip
+            strict (bool):
+                Passed directly to self._check_instrument
         """
         #from scipy.stats import sigmaclip as dosigmaclip
         from .stats import sigmaclip_median as dosigmaclip
@@ -1842,20 +1913,26 @@ class RV(ISSUES, REPORTS):
             return
 
         instruments = self._check_instrument(instrument, strict)
+        if instruments is None:
+            return
         changed_instruments = []
 
         for inst in instruments:
             m = self.instrument_array == inst
-            result = dosigmaclip(self.vrad[m], low=sigma, high=sigma)
+            d = getattr(self, quantity)
+
+            if np.isnan(d[m]).all():
+                continue
+
+            result = dosigmaclip(d[m], low=sigma, high=sigma)
             # n = self.vrad[m].size - result.clipped.size
 
-            ind = m & self.mask & \
-                  ((self.vrad < result.lower) | (self.vrad > result.upper))
+            ind = m & self.mask & ((d < result.lower) | (d > result.upper))
             n = ind.sum()
 
             if self.verbose and n > 0:
                 s = 's' if (n == 0 or n > 1) else ''
-                logger.warning(f'sigma-clip RVs will remove {n} point{s} for {inst}')
+                logger.warning(f'sigma-clip {quantity} will remove {n} point{s} for {inst}')
 
             if n > 0:
                 self.mask[ind] = False
@@ -1880,21 +1957,32 @@ class RV(ISSUES, REPORTS):
         if config.return_self:
             return self
 
-    def clip_maxerror(self, maxerror:float):
-        """ Mask out points with RV error larger than a given value
+    def clip_maxerror(self, maxerror:float, instrument=None):
+        """
+        Mask out points with RV error larger than a given value. If `instrument`
+        is given, mask only observations from that instrument.
 
         Args:
             maxerror (float): Maximum error to keep.
+            instrument (str, list, tuple, ndarray): Instrument(s) to clip
         """
         if self._child:
             return
 
         self.maxerror = maxerror
+
+        if instrument is None:
+            inst_mask = np.ones_like(self.svrad, dtype=bool)
+        else:
+            inst_mask = self._instrument_mask(instrument)
+        
         above = self.svrad > maxerror
-        n = above.sum()
-        self.mask[above] = False
+        old_mask = self.mask.copy()
+
+        self.mask[inst_mask & above] = False
 
         if self.verbose and above.sum() > 0:
+            n = (above[inst_mask] & old_mask[inst_mask]).sum()
             s = 's' if (n == 0 or n > 1) else ''
             logger.warning(f'clip_maxerror ({maxerror} {self.units}) removed {n} point' + s)
 
@@ -1902,6 +1990,36 @@ class RV(ISSUES, REPORTS):
         if config.return_self:
             return self
 
+    def sigmaclip_ew(self, sigma=5):
+        """ Sigma-clip EW (FWHM x contrast), by MAD away from the median """
+        from .stats import sigmaclip_median as dosigmaclip, weighted_median
+
+        S = deepcopy(self)
+        for _s in S:
+            m = _s.mask
+            _s.fwhm -= weighted_median(_s.fwhm[m], 1 / _s.fwhm_err[m])
+            _s.contrast -= weighted_median(_s.contrast[m], 1 / _s.contrast_err[m])
+        S._build_arrays()
+        ew = S.fwhm * S.contrast
+        ew_err = np.hypot(S.fwhm_err * S.contrast, S.fwhm * S.contrast_err)
+
+        wmed = weighted_median(ew[S.mask], 1 / ew_err[S.mask])
+        data = (ew - wmed) / ew_err
+        result = dosigmaclip(data, low=sigma, high=sigma)
+        ind = (data < result.lower) | (data > result.upper)
+        self.mask[ind] = False
+
+        if self.verbose and ind.sum() > 0:
+            n = ind.sum()
+            s = 's' if (n == 0 or n > 1) else ''
+            logger.warning(f'sigmaclip_ew removed {n} point' + s)
+
+        self._propagate_mask_changes()
+        if config.return_self:
+            return self
+
+
+
     def bin(self):
         """
         Nightly bin the observations.
@@ -1912,6 +2030,8 @@ class RV(ISSUES, REPORTS):
 
         # create copy of self to be returned
         snew = deepcopy(self)
+        # store original object
+        snew._unbinned = deepcopy(self)
 
         all_bad_quantities = []
 
@@ -1943,7 +2063,8 @@ class RV(ISSUES, REPORTS):
 
                 # treat ccf_mask specially, doing a 'unique' bin
                 if q == 'ccf_mask':
-                    setattr(s, q, bin_ccf_mask(s.mtime, getattr(s, q)))
+                    ccf_mask = getattr(s, q)[s.mask]
+                    setattr(s, q, bin_ccf_mask(s.mtime, ccf_mask))
                     continue
 
                 if Q.dtype != np.float64:
@@ -2101,23 +2222,32 @@ class RV(ISSUES, REPORTS):
         if config.return_self:
             return self
 
-    def detrend(self, degree=1):
-        """ Detrend the RVs of all instruments """
+    def detrend(self, degree: int=1):
+        """
+        Detrend the RVs of all instruments using a polynomial of degree `degree`
+        """
         instrument_indices = np.unique_inverse(self.instrument_array).inverse_indices
-        def fun(p, t, degree, ninstruments, just_model=False, index=None):
+        instrument_indices_masked = np.unique_inverse(self.instrument_array[self.mask]).inverse_indices
+
+        def fun(p, t, degree, ninstruments, just_model=False, index=None, masked=True):
             polyp, offsets = p[:degree], p[-ninstruments:]
             polyp = np.r_[polyp, 0.0]
             if index is None:
-                model = offsets[instrument_indices] + np.polyval(polyp, t)
+                if masked:
+                    model = offsets[instrument_indices_masked] + np.polyval(polyp, t)
+                else:
+                    model = offsets[instrument_indices] + np.polyval(polyp, t)
             else:
                 model = offsets[index] + np.polyval(polyp, t)
             if just_model:
                 return model
             return self.mvrad - model
+
         coef = np.polyfit(self.mtime, self.mvrad, degree)
         x0 = np.append(coef, [0.0] * (len(self.instruments) - 1))
-        print(x0)
+        # print(x0)
         fun(x0, self.mtime, degree, len(self.instruments))
+
         from scipy.optimize import leastsq
         xbest, _ = leastsq(fun, x0, args=(self.mtime, degree, len(self.instruments)))
 
@@ -2127,12 +2257,13 @@ class RV(ISSUES, REPORTS):
         self.plot(ax=ax)
         for i, inst in enumerate(self.instruments):
             s = getattr(self, inst)
-            ax.plot(s.time, fun(xbest, s.time, degree, len(self.instruments), just_model=True, index=i),
+            ax.plot(s.time,
+                    fun(xbest, s.time, degree, len(self.instruments), just_model=True, index=i, masked=False),
                     color=f'C{i}')
         ax.set_title('original', loc='left', fontsize=10)
         ax.set_title(f'coefficients: {xbest[:degree]}', loc='right', fontsize=10)
 
-        self.add_to_vrad(-fun(xbest, self.time, degree, len(self.instruments), just_model=True))
+        self.add_to_vrad(-fun(xbest, self.time, degree, len(self.instruments), just_model=True, masked=False))
         ax = fig.add_subplot(2, 1, 2)
         self.plot(ax=ax)
         ax.set_title('detrended', loc='left', fontsize=10)
@@ -2141,7 +2272,7 @@ class RV(ISSUES, REPORTS):
         # axs[1].errorbar(self.mtime, fun(xbest, self.mtime, degree, len(self.instruments)), self.msvrad, fmt='o')
 
         return
-        
+
 
 
 
@@ -2301,24 +2432,31 @@ class RV(ISSUES, REPORTS):
         self.units = new_units
 
 
-    def put_at_systemic_velocity(self):
+    def put_at_systemic_velocity(self, factor=1.0, ignore=None):
         """
-        For instruments in which mean(RV) < ptp(RV), "move" RVs to the systemic
-        velocity from simbad. This is useful if some instruments are centered
-        at zero while others are not, and instead of calling `.adjust_means()`,
-        but it only works when the systemic velocity is smaller than ptp(RV).
+        For instruments in which mean(RV) < `factor` * ptp(RV), "move" RVs to
+        the systemic velocity from simbad. This is useful if some instruments
+        are centered at zero while others are not, and instead of calling
+        `.adjust_means()`, but it only works when the systemic velocity is
+        smaller than `factor` * ptp(RV).
         """
         changed = False
         for inst in self.instruments:
+            if ignore is not None:
+                if inst in ignore or any([i in inst for i in ignore]):
+                    continue
+            changed_inst = False
             s = getattr(self, inst)
             if s.mask.any():
-                if np.abs(s.mvrad.mean()) < np.ptp(s.mvrad):
+                if np.abs(s.mvrad.mean()) < factor * np.ptp(s.mvrad):
                     s.vrad += self.simbad.rvz_radvel * 1e3
-                    changed = True
+                    changed = changed_inst = True
             else:  # all observations are masked, use non-masked arrays
-                if np.abs(s.vrad.mean()) < np.ptp(s.vrad):
+                if np.abs(s.vrad.mean()) < factor * np.ptp(s.vrad):
                     s.vrad += self.simbad.rvz_radvel * 1e3
-                    changed = True
+                    changed = changed_inst = True
+            if changed_inst and self.verbose:
+                logger.info(f"putting {inst} RVs at systemic velocity")
         if changed:
             self._build_arrays()
 
@@ -2340,34 +2478,72 @@ class RV(ISSUES, REPORTS):
             self.instruments = sorted(self.instruments, key=lambda i: getattr(self, i).time.max())
             self._build_arrays()
 
+    def put_instrument_last(self, instrument):
+        if not self._check_instrument(instrument, strict=True, log=True):
+            return
+        self.instruments = [i for i in self.instruments if i != instrument] + [instrument]
+        self._build_arrays()
 
-    def save(self, directory=None, instrument=None, full=False, postfix=None,
-             save_masked=False, save_nans=True):
-        """ Save the observations in .rdb files.
+    def save(self, directory=None, instrument=None, format='rdb',
+             indicators=False, join_instruments=False, postfix=None,
+             save_masked=False, save_nans=True, **kwargs):
+        """ Save the observations in .rdb or .csv files.
 
         Args:
             directory (str, optional):
                 Directory where to save the .rdb files.
             instrument (str, optional):
                 Instrument for which to save observations.
-            full (bool, optional):
-                Save just RVs and errors (False) or more indicators (True).
+            format (str, optional):
+                Format to use ('rdb' or 'csv').
+            indicators (bool, str, list[str], optional):
+                Save only RVs and errors (False) or more indicators. If True,
+                use a default list, if `str`, use an existing list, if list[str]
+                provide a sequence of specific indicators.
+            join_instruments (bool, optional):
+                Join all instruments in a single file.
             postfix (str, optional):
                 Postfix to add to the filenames ([star]_[instrument]_[postfix].rdb).
+            save_masked (bool, optional)
+                If True, also save masked observations (those for which
+                self.mask == False)
             save_nans (bool, optional)
                 Whether to save NaN values in the indicators, if they exist. If
                 False, the full observation which contains NaN values is not saved.
         """
+        if format not in ('rdb', 'csv'):
+            logger.error(f"format must be 'rdb' or 'csv', got '{format}'")
+            return
+
         star_name = self.star.replace(' ', '')
 
-        if directory is None:
-            directory = '.'
-        else:
+        if directory is not None:
             os.makedirs(directory, exist_ok=True)
 
+        indicator_sets = {
+            "default": [
+                "fwhm", "fwhm_err",
+                "bispan", "bispan_err",
+                "contrast", "contrast_err",
+                "rhk", "rhk_err",
+                "berv",
+            ],
+            "CORALIE": [
+                "fwhm", "fwhm_err",
+                "bispan", "bispan_err",
+                "contrast", "contrast_err",
+                "haindex", "haindex_err",
+                "berv",
+            ],
+        } 
+
+        if 'full' in kwargs:
+            logger.warning('argument `full` is deprecated, use `indicators` instead')
+            indicators = kwargs['full']
+
         files = []
 
-        for inst in self.instruments:
+        for _i, inst in enumerate(self.instruments):
             if instrument is not None:
                 if instrument not in inst:
                     continue
@@ -2377,75 +2553,95 @@ class RV(ISSUES, REPORTS):
             if not _s.mask.any():  # all observations are masked, don't save
                 continue
 
-            if full:
-                if save_masked:
-                    arrays = [
-                        _s.time, _s.vrad, _s.svrad,
-                        _s.fwhm, _s.fwhm_err,
-                        _s.bispan, _s.bispan_err,
-                        _s.contrast, _s.contrast_err,
-                        _s.rhk, _s.rhk_err,
-                        _s.berv,
-                    ]
-                else:
-                    arrays = [
-                        _s.mtime, _s.mvrad, _s.msvrad,
-                        _s.fwhm[_s.mask], _s.fwhm_err[_s.mask],
-                        _s.bispan[_s.mask], _s.bispan_err[_s.mask],
-                        _s.contrast[_s.mask], _s.contrast_err[_s.mask],
-                        _s.rhk[_s.mask], _s.rhk_err[_s.mask],
-                        _s.berv[_s.mask],
-                    ]
-                if not save_nans:
-                    raise NotImplementedError
-                    # if np.isnan(d).any():
-                    #     # remove observations where any of the indicators are # NaN
-                    #     nan_mask = np.isnan(d[:, 3:]).any(axis=1)
-                    #     d = d[~nan_mask]
-                    #     if self.verbose:
-                    #         logger.warning(f'masking {nan_mask.sum()} observations with NaN in indicators')
-
-                header = '\t'.join(['rjd', 'vrad', 'svrad',
-                                    'fwhm', 'sig_fwhm',
-                                    'bispan', 'sig_bispan',
-                                    'contrast', 'sig_contrast',
-                                    'rhk', 'sig_rhk',
-                                    'berv',
-                                    ])
-                header += '\n'
-                header += '\t'.join(['-' * len(c) for c in header.strip().split('\t')])
+            if save_masked:
+                arrays = [_s.time, _s.vrad, _s.svrad]
+                if join_instruments:
+                    arrays += [_s.instrument_array]
+            else:
+                arrays = [_s.mtime, _s.mvrad, _s.msvrad]
+                if join_instruments:
+                    arrays += [_s.instrument_array[_s.mask]]
+            
+            if indicators in (False, None):
+                indicator_names = []
+            else:
+                if indicators is True:
+                    indicator_names = indicator_sets["default"]
+                elif isinstance(indicators, str):
+                    try:
+                        indicator_names = indicator_sets[indicators]
+                    except KeyError:
+                        logger.error(f"unknown indicator set '{indicators}'")
+                        logger.error(f"available: {list(indicator_sets.keys())}")
+                        return
+                elif isinstance(indicators, list) and all(isinstance(i, str) for i in indicators):
+                    indicator_names = indicators
 
+            if save_masked:
+                arrays += [getattr(_s, ind) for ind in indicator_names]
             else:
-                if save_masked:
-                    arrays = [_s.time, _s.vrad, _s.svrad]
-                else:
-                    arrays = [_s.mtime, _s.mvrad, _s.msvrad]
+                arrays += [getattr(_s, ind)[_s.mask] for ind in indicator_names]
+
+            d = np.stack(arrays, axis=1)
+            if not save_nans:
+                # raise NotImplementedError
+                if np.isnan(d).any():
+                    # remove observations where any of the indicators are # NaN
+                    nan_mask = np.isnan(d[:, 3:]).any(axis=1)
+                    d = d[~nan_mask]
+                    if self.verbose:
+                        msg = f'{inst}: masking {nan_mask.sum()} observations with NaN in indicators'
+                        logger.warning(msg)
+
+            cols = ['rjd', 'vrad', 'svrad']
+            cols += ['inst'] if join_instruments else []
+            cols += indicator_names
 
-                # d = np.stack(arrays, axis=1)
-                header = 'rjd\tvrad\tsvrad\n---\t----\t-----'
+            if format == 'rdb':
+                header = '\t'.join(cols)
+                header += '\n'
+                header += '\t'.join(['-' * len(c) for c in header.strip().split('\t')])
+            else:
+                header = ','.join(cols)
 
-            file = f'{star_name}_{inst}.rdb'
-            if postfix is not None:
-                file = f'{star_name}_{inst}_{postfix}.rdb'
+            if join_instruments:
+                file = f'{star_name}.{format}'
+                if postfix is not None:
+                    file = f'{star_name}_{postfix}.{format}'
+            else:
+                file = f'{star_name}_{inst}.{format}'
+                if postfix is not None:
+                    file = f'{star_name}_{inst}_{postfix}.{format}'
 
+            if directory is not None:
+                file = os.path.join(directory, file)
             files.append(file)
-            file = os.path.join(directory, file)
 
             N = len(arrays[0])
-            with open(file, 'w') as f:
-                f.write(header + '\n')
+            with open(file, 'a' if join_instruments and _i != 0 else 'w') as f:
+                if join_instruments and _i != 0:
+                    pass
+                else:
+                    f.write(header + '\n')
+
                 for i in range(N):
                     for j, a in enumerate(arrays):
                         f.write(str(a[i]))
                         if j < len(arrays) - 1:
-                            f.write('\t')
+                            f.write('\t' if format == 'rdb' else ',')
                     f.write('\n')
 
             # np.savetxt(file, d, header=header, delimiter='\t', comments='', fmt='%f')
 
-            if self.verbose:
+            if self.verbose and not join_instruments:
                 logger.info(f'saving to {file}')
 
+        if self.verbose and join_instruments:
+            logger.info(f'saving to {files[0]}')
+
+        if join_instruments:
+            files = [files[0]]
+
         return files
 
     def checksum(self, write_to=None):