geospacelab 0.11.4__py3-none-any.whl → 0.12.0__py3-none-any.whl

geospacelab/datahub/sources/tud/goce/wnd_acc/__init__.py

@@ -0,0 +1,339 @@
+# Licensed under the BSD 3-Clause License
+# Copyright (C) 2021 GeospaceLab (geospacelab)
+# Author: Lei Cai, Space Physics and Astronomy, University of Oulu
+
+import numpy as np
+import datetime
+
+import geospacelab.datahub as datahub
+from geospacelab.datahub import DatabaseModel, FacilityModel, InstrumentModel, ProductModel
+from geospacelab.datahub.sources.tud import tud_database
+from geospacelab.datahub.sources.tud.grace import grace_facility
+from geospacelab.config import prf
+import geospacelab.toolbox.utilities.pybasic as basic
+import geospacelab.toolbox.utilities.pylogging as mylog
+import geospacelab.toolbox.utilities.pydatetime as dttool
+from geospacelab.datahub.sources.tud.goce.wnd_acc.loader import Loader as default_Loader
+from geospacelab.datahub.sources.tud.goce.wnd_acc.downloader import (Downloader as default_Downloader)
+import geospacelab.datahub.sources.tud.goce.wnd_acc.variable_config as var_config
+
+
+default_dataset_attrs = {
+    'database': tud_database,
+    'facility': grace_facility,
+    'instrument': 'ACC',
+    'product': 'WND-ACC',
+    'data_file_ext': 'txt',
+    'product_version': 'v02',
+    'data_root_dir': prf.datahub_data_root_dir / 'TUD' / 'GOCE',
+    'allow_load': True,
+    'allow_download': True,
+    'force_download': False,
+    'data_search_recursive': False,
+    'add_AACGM': False,
+    'add_APEX': False,
+    'label_fields': ['database', 'facility', 'instrument', 'product', 'product_version'],
+    'load_mode': 'AUTO',
+    'time_clip': True,
+}
+
+default_variable_names_v01 = []
+default_variable_names_v02 = [
+    'SC_DATETIME',
+    'SC_GEO_LAT',
+    'SC_GEO_LON',
+    'SC_GEO_ALT',
+    'SC_ARG_LAT',
+    'SC_GEO_LST',
+    'u_CROSS',
+    'UNIT_VECTOR_N',
+    'UNIT_VECTOR_E',
+    'UNIT_VECTOR_D',
+    'u_CROSS_N',
+    'u_CROSS_E',
+    'u_CROSS_D',
+    'FLAG',
+    ]
+
+# default_data_search_recursive = True
+
+default_attrs_required = ['sat_id']
+
+
+class Dataset(datahub.DatasetSourced):
+    def __init__(self, **kwargs):
+        kwargs = basic.dict_set_default(kwargs, **default_dataset_attrs)
+
+        super().__init__(**kwargs)
+
+        self.database = kwargs.pop('database', 'TUD')
+        self.facility = kwargs.pop('facility', 'GOCE')
+        self.instrument = kwargs.pop('instrument', 'ACC')
+        self.product = kwargs.pop('product', 'WND-ACC')
+        self.product_version = kwargs.pop('product_version', 'v02')
+        self.local_latest_version = ''
+        self.allow_download = kwargs.pop('allow_download', False)
+        self.force_download = kwargs.pop('force_download', False)
+        self.download_dry_run = kwargs.pop('download_dry_run', False)
+        
+        self.add_AACGM = kwargs.pop('add_AACGM', False) 
+        self.add_APEX = kwargs.pop('add_APEX', False)
+        self._data_root_dir = self.data_root_dir    # Record the initial root dir
+
+        self.sat_id = kwargs.pop('sat_id', '')
+
+        self.metadata = {}
+
+        allow_load = kwargs.pop('allow_load', False)
+
+        # self.config(**kwargs)
+
+        if self.loader is None:
+            self.loader = default_Loader
+
+        if self.downloader is None:
+            self.downloader = default_Downloader
+
+        self._validate_attrs()
+
+        if allow_load:
+            self.load_data()
+
+    def _validate_attrs(self):
+        for attr_name in default_attrs_required:
+            attr = getattr(self, attr_name)
+            if not attr:
+                mylog.StreamLogger.warning("The parameter {} is required before loading data!".format(attr_name))
+
+        self.data_root_dir = self.data_root_dir / self.product.upper() / self.product_version
+
+    def label(self, **kwargs):
+        label = super().label()
+        return label
+
+    def load_data(self, **kwargs):
+        self.check_data_files(**kwargs)
+        if self.product_version == 'v01':
+            default_variable_names = default_variable_names_v01
+        else:
+            default_variable_names = default_variable_names_v02 
+        self._set_default_variables(
+            default_variable_names,
+            configured_variables=var_config.configured_variables
+        )
+        for file_path in self.data_file_paths:
+            load_obj = self.loader(file_path, file_type='txt', version=self.product_version)
+
+            for var_name in self._variables.keys():
+                value = load_obj.variables[var_name]
+                self._variables[var_name].join(value)
+
+            # self.select_beams(field_aligned=True)
+        if self.time_clip:
+            self.time_filter_by_range(var_datetime_name='SC_DATETIME')
+
+        if self.add_AACGM:
+            self.convert_to_AACGM()
+
+        if self.add_APEX:
+            self.convert_to_APEX()
+
+        self._add_u_CT()
+
+    def _add_u_CT(self):
+        from geospacelab.observatory.orbit.utilities import LEOToolbox
+        ds_leo = LEOToolbox(self.dt_fr, self.dt_to)
+        ds_leo.clone_variables(self)
+
+        wind_unit_vector = np.concatenate(
+            (
+                self['UNIT_VECTOR_N'].value,
+                self['UNIT_VECTOR_E'].value,
+                self['UNIT_VECTOR_D'].value
+            ),
+        axis=1
+        )
+        orbit_unit_vector = ds_leo.trajectory_local_unit_vector()
+        cp = np.cross(orbit_unit_vector, wind_unit_vector)
+        u_CT = -np.sign(cp[:, 2]) * (
+            (self['u_CROSS'].value.flatten() * self['UNIT_VECTOR_N'].flatten())**2 + 
+            (self['u_CROSS'].value.flatten() * self['UNIT_VECTOR_E'].flatten())**2)**0.5
+        u_VCT = self['u_CROSS'].value.flatten() * self['UNIT_VECTOR_D'].flatten()
+
+        var = self['u_CROSS'].clone()
+        var.name = 'u_CT'
+        var.label = r'$u_{CT}$'
+        var.visual.axis[1].lim = [None, None]
+        var.value = u_CT[:, np.newaxis]
+        self['u_CT'] = var
+        
+        var = self['u_CROSS'].clone()
+        var.name = 'u_VCT'
+        var.label = r'$u_{VCT}$'
+        var.visual.axis[1].lim = [None, None]
+        var.value = u_VCT[:, np.newaxis]
+        self['u_VCT'] = var
+
+
+    def add_GEO_LST(self):
+        lons = self['SC_GEO_LON'].flatten()
+        uts = self['SC_DATETIME'].flatten()
+        lsts = [ut + datetime.timedelta(seconds=int(lon/15.*3600)) for ut, lon in zip(uts, lons)]
+        lsts = [lst.hour + lst.minute/60. + lst.second/3600. for lst in lsts]
+        var = self.add_variable(var_name='SC_GEO_LST')
+        var.value = np.array(lsts)[:, np.newaxis]
+        var.label = 'LST'
+        var.unit = 'h'
+        var.depends = self['SC_GEO_LON'].depends
+        return var
+
+
+    def convert_to_APEX(self):
+        import geospacelab.cs as gsl_cs
+
+        coords_in = {
+            'lat': self['SC_GEO_LAT'].value.flatten(),
+            'lon': self['SC_GEO_LON'].value.flatten(),
+            'height': self['SC_GEO_ALT'].value.flatten()
+        }
+        dts = self['SC_DATETIME'].value.flatten()
+        cs_sph = gsl_cs.GEOCSpherical(coords=coords_in, ut=dts)
+        cs_apex = cs_sph.to_APEX(append_mlt=True)
+        self.add_variable('SC_APEX_LAT')
+        self.add_variable('SC_APEX_LON')
+        self.add_variable('SC_APEX_MLT')
+        self['SC_APEX_LAT'].value = cs_apex['lat'].reshape(self['SC_DATETIME'].value.shape)
+        self['SC_APEX_LON'].value = cs_apex['lon'].reshape(self['SC_DATETIME'].value.shape)
+        self['SC_APEX_MLT'].value = cs_apex['mlt'].reshape(self['SC_DATETIME'].value.shape)
+
+    def convert_to_AACGM(self):
+        import geospacelab.cs as gsl_cs
+
+        coords_in = {
+            'lat': self['SC_GEO_LAT'].value.flatten(),
+            'lon': self['SC_GEO_LON'].value.flatten(),
+            'height': self['SC_GEO_ALT'].value.flatten()
+        }
+        dts = self['SC_DATETIME'].value.flatten()
+        cs_sph = gsl_cs.GEOCSpherical(coords=coords_in, ut=dts)
+        cs_aacgm = cs_sph.to_AACGM(append_mlt=True)
+        self.add_variable('SC_AACGM_LAT')
+        self.add_variable('SC_AACGM_LON')
+        self.add_variable('SC_AACGM_MLT')
+        self['SC_AACGM_LAT'].value = cs_aacgm['lat'].reshape(self['SC_DATETIME'].value.shape)
+        self['SC_AACGM_LON'].value = cs_aacgm['lon'].reshape(self['SC_DATETIME'].value.shape)
+        self['SC_AACGM_MLT'].value = cs_aacgm['mlt'].reshape(self['SC_DATETIME'].value.shape)
+
+    def search_data_files(self, **kwargs):
+
+        dt_fr = self.dt_fr
+        dt_to = self.dt_to
+
+        diff_months = dttool.get_diff_months(dt_fr, dt_to)
+
+        dt0 = dttool.get_first_day_of_month(self.dt_fr)
+
+        for i in range(diff_months + 1):
+            this_day = dttool.get_next_n_months(dt0, i)
+
+            initial_file_dir = kwargs.pop(
+                'initial_file_dir', self.data_root_dir
+            )
+
+            file_patterns = [
+                'GO',
+                self.product.upper().replace('-', '_'),
+                this_day.strftime('%Y_%m'), 
+                self.product_version + '.txt'
+            ]
+            # remove empty str
+            file_patterns = [pattern for pattern in file_patterns if str(pattern)]
+            search_pattern = '*' + '*'.join(file_patterns) + '*'
+
+            done = super().search_data_files(
+                initial_file_dir=initial_file_dir,
+                search_pattern=search_pattern,
+                allow_multiple_files=False,
+                include_extension=False,
+            )
+            # Validate file paths
+
+            if (not done and self.allow_download) or self.force_download:
+                done = self.download_data()
+                if done:
+                    initial_file_dir = self.data_root_dir
+                    done = super().search_data_files(
+                        initial_file_dir=initial_file_dir,
+                        search_pattern=search_pattern,
+                        allow_multiple_files=False
+                    )
+        self.data_file_paths = np.unique(self.data_file_paths)
+        return done
+
+    def download_data(self, dt_fr=None, dt_to=None):
+        if dt_fr is None:
+            dt_fr = self.dt_fr
+        if dt_to is None:
+            dt_to = self.dt_to
+        download_obj = self.downloader(
+            dt_fr, dt_to,
+            sat_id=self.sat_id,
+            product=self.product,
+            version=self.product_version,
+            force_download=self.force_download,
+            dry_run=self.download_dry_run,
+        )
+
+        return any(download_obj.done)
+
+    @property
+    def database(self):
+        return self._database
+
+    @database.setter
+    def database(self, value):
+        if isinstance(value, str):
+            self._database = DatabaseModel(value)
+        elif issubclass(value.__class__, DatabaseModel):
+            self._database = value
+        else:
+            raise TypeError
+
+    @property
+    def product(self):
+        return self._product
+
+    @product.setter
+    def product(self, value):
+        if isinstance(value, str):
+            self._product = ProductModel(value)
+        elif issubclass(value.__class__, ProductModel):
+            self._product = value
+        else:
+            raise TypeError
+
+    @property
+    def facility(self):
+        return self._facility
+
+    @facility.setter
+    def facility(self, value):
+        if isinstance(value, str):
+            self._facility = FacilityModel(value)
+        elif issubclass(value.__class__, FacilityModel):
+            self._facility = value
+        else:
+            raise TypeError
+
+    @property
+    def instrument(self):
+        return self._instrument
+
+    @instrument.setter
+    def instrument(self, value):
+        if isinstance(value, str):
+            self._instrument = InstrumentModel(value)
+        elif issubclass(value.__class__, InstrumentModel):
+            self._instrument = value
+        else:
+            raise TypeError

geospacelab/datahub/sources/tud/goce/wnd_acc/downloader.py

@@ -0,0 +1,42 @@
+# Licensed under the BSD 3-Clause License
+# Copyright (C) 2021 GeospaceLab (geospacelab)
+# Author: Lei Cai, Space Physics and Astronomy, University of Oulu
+
+__author__ = "Lei Cai"
+__copyright__ = "Copyright 2021, GeospaceLab"
+__license__ = "BSD-3-Clause License"
+__email__ = "lei.cai@oulu.fi"
+__docformat__ = "reStructureText"
+
+import datetime
+from geospacelab.config import prf
+
+from geospacelab.datahub.sources.tud.downloader import TUDownloader as DownloaderBase   
+
+
+class Downloader(DownloaderBase):
+    
+    def __init__(self, 
+        dt_fr, dt_to, 
+        mission='GOCE', 
+        sat_id='',
+        version='v02',  
+        product='WND-ACC',
+        direct_download=True,
+        force_download=False,
+        dry_run=False,
+        ):
+        self.mission = mission
+        self.version = version
+        
+        super().__init__(
+            dt_fr, dt_to, 
+            mission=mission,
+            sat_id=sat_id, 
+            version=version,
+            product=product,
+            direct_download=direct_download,
+            force_download=force_download,
+            dry_run=dry_run
+        )
+        

geospacelab/datahub/sources/tud/goce/wnd_acc/loader.py

@@ -0,0 +1,65 @@
+# Licensed under the BSD 3-Clause License
+# Copyright (C) 2021 GeospaceLab (geospacelab)
+# Author: Lei Cai, Space Physics and Astronomy, University of Oulu
+
+__author__ = "Lei Cai"
+__copyright__ = "Copyright 2021, GeospaceLab"
+__license__ = "BSD-3-Clause License"
+__email__ = "lei.cai@oulu.fi"
+__docformat__ = "reStructureText"
+
+import re
+import numpy as np
+import datetime
+import geospacelab.toolbox.utilities.pydatetime as dttool
+
+
+class Loader(object):
+
+    def __init__(self, file_path, file_type='txt', version='v01', direct_load=True):
+        self.file_path = file_path
+        self.file_type = file_type
+        self.version = version
+        self.variables = {}
+        self.done = False
+        if direct_load:
+            self.load()
+
+    def load(self):
+
+        if self.version == 'v01':
+            raise ValueError 
+        elif 'v02' in self.version:
+            self.load_v02()
+        else:
+            raise NotImplementedError
+
+    def load_v02(self):
+        with open(self.file_path, 'r') as f:
+            text = f.read()
+            results = re.findall(
+                r"^(\d{4}-\d{2}-\d{2}\s*\d{2}\:\d{2}\:\d{2}\.\d{3})\s*(\w{3})\s*"
+                + r"([\-\d.]+)\s*([\-\d.]+)\s*([\-\d.]+)\s*([\-\d.]+)\s*([\-\d.]+)\s*"
+                + r"([\-\d.]+)\s*([\-\d.]+)\s*([\-\d.]+)\s*([\-\d.]+)\s*([\-\d.]+)",
+                text,
+                re.M)
+            results = list(zip(*results))
+            dts = [datetime.datetime.strptime(dtstr + '000', "%Y-%m-%d %H:%M:%S.%f") for dtstr in results[0]]
+            if results[1][0] == 'GPS':
+                t_gps = [(dt - dttool._GPS_DATETIME_0).total_seconds() for dt in dts]
+                dts = dttool.convert_gps_time_to_datetime(t_gps, weeks=None)
+            num_rec = len(dts)
+            self.variables['SC_DATETIME'] = np.array(dts).reshape(num_rec, 1)
+            self.variables['SC_GEO_ALT'] = np.array(results[2]).astype(np.float32).reshape(num_rec, 1) * 1e-3   # in km
+            self.variables['SC_GEO_LON'] = np.array(results[3]).astype(np.float32).reshape(num_rec, 1)
+            self.variables['SC_GEO_LAT'] = np.array(results[4]).astype(np.float32).reshape(num_rec, 1)
+            self.variables['SC_GEO_LST'] = np.array(results[5]).astype(np.float32).reshape(num_rec, 1)
+            self.variables['SC_ARG_LAT'] = np.array(results[6]).astype(np.float32).reshape(num_rec, 1)
+            self.variables['u_CROSS'] = np.array(results[7]).astype(np.float32).reshape(num_rec, 1)
+            self.variables['UNIT_VECTOR_N'] = np.array(results[8]).astype(np.float32).reshape(num_rec, 1)
+            self.variables['UNIT_VECTOR_E'] = np.array(results[9]).astype(np.float32).reshape(num_rec, 1)
+            self.variables['UNIT_VECTOR_D'] = np.array(results[10]).astype(np.float32).reshape(num_rec, 1)
+            self.variables['FLAG'] = np.array(results[6]).astype(np.float32).reshape(num_rec, 1)
+            self.variables['u_CROSS_N'] = self.variables['u_CROSS'] * self.variables['UNIT_VECTOR_N']
+            self.variables['u_CROSS_E'] = self.variables['u_CROSS'] * self.variables['UNIT_VECTOR_E']
+            self.variables['u_CROSS_D'] = self.variables['u_CROSS'] * self.variables['UNIT_VECTOR_D']

geospacelab/datahub/sources/tud/goce/wnd_acc/variable_config.py

@@ -0,0 +1,188 @@
+# Licensed under the BSD 3-Clause License
+# Copyright (C) 2021 GeospaceLab (geospacelab)
+# Author: Lei Cai, Space Physics and Astronomy, University of Oulu
+
+__author__ = "Lei Cai"
+__copyright__ = "Copyright 2021, GeospaceLab"
+__license__ = "BSD-3-Clause License"
+__email__ = "lei.cai@oulu.fi"
+__docformat__ = "reStructureText"
+
+from geospacelab.datahub import VariableModel as Var
+import geospacelab.visualization.mpl.colormaps as cm
+
+import numpy as np
+
+database = 'ESA/EarthOnline'
+
+timestamps = {
+    'SC_DATETIME': 'SC_DATETIME',
+}
+
+
+default_colormap = "gist_ncar"
+
+default_plot_config = {
+    'line':         {
+        'linestyle':        '-',
+        'linewidth':        1.5,
+        'marker':           '',
+        'markersize':       3,
+    },
+    'pcolormesh':   {
+        'cmap':            default_colormap,
+    }
+}
+
+configured_variables = {}
+visual = 'on'
+
+depend_0 = {'UT': 'SC_DATETIME',
+            'GEO_LAT': 'SC_GEO_LAT', 'GEO_LON': 'SC_GEO_LON',
+            'AACGM_LAT': 'SC_AACGM_LAT', 'AACGM_LON': 'SC_AACGM_LON', 'AACGM_MLT': 'SC_AACGM_MLT'}
+# depend_c = {'SPECTRA': 'EMISSION_SPECTRA'}
+
+####################################################################################################################
+var_name = 'u_CROSS'
+var = Var(name=var_name, ndim=1, variable_type='scalar', visual=visual)
+# set variable attrs
+var.fullname = 'Cross neutral wind'
+var.label = r'$u_{cross}$'
+var.unit = 'm/s'
+var.unit_label = r'm/s'
+var.group = r'$u$'
+# var.error = var_name + '_err'
+var.depends = {0: depend_0}
+# set plot attrs
+plot_config = var.visual.plot_config
+plot_config.config(**default_plot_config)
+plot_config.style = '1noE'
+# set axis attrs
+axis = var.visual.axis
+axis[1].data = "@v.value"
+# axis[1].lim = [np.nan, np.nan]
+axis[2].label = '@v.label'
+axis[1].label = '@v.label'
+axis[1].unit = '@v.unit_label'
+
+configured_variables[var_name] = var
+
+####################################################################################################################
+var_name = 'u_CROSS_E'
+var = Var(name=var_name, ndim=1, variable_type='scalar', visual=visual)
+# set variable attrs
+var.fullname = 'Cross neutral wind (Eastward)'
+var.label = r'$u_{E}$'
+var.unit = 'm/s'
+var.unit_label = r'm/s'
+var.group = r'$u$'
+# var.error = var_name + '_err'
+var.depends = {0: depend_0}
+# set plot attrs
+plot_config = var.visual.plot_config
+plot_config.config(**default_plot_config)
+plot_config.style = '1noE'
+# set axis attrs
+axis = var.visual.axis
+axis[1].data = "@v.value"
+# axis[1].lim = [np.nan, np.nan]
+axis[2].label = '@v.label'
+axis[1].unit = '@v.unit_label'
+
+configured_variables[var_name] = var
+
+####################################################################################################################
+var_name = 'u_CROSS_N'
+var = Var(name=var_name, ndim=1, variable_type='scalar', visual=visual)
+# set variable attrs
+var.fullname = 'Cross neutral wind (Northward)'
+var.label = r'$u_{N}$'
+var.unit = 'm/s'
+var.unit_label = r'm/s'
+var.group = r'$u$'
+# var.error = var_name + '_err'
+var.depends = {0: depend_0}
+# set plot attrs
+plot_config = var.visual.plot_config
+plot_config.config(**default_plot_config)
+plot_config.style = '1noE'
+# set axis attrs
+axis = var.visual.axis
+axis[1].data = "@v.value"
+# axis[1].lim = [np.nan, np.nan]
+axis[2].label = '@v.label'
+axis[1].unit = '@v.unit_label'
+
+configured_variables[var_name] = var
+
+####################################################################################################################
+var_name = 'u_CROSS_D'
+var = Var(name=var_name, ndim=1, variable_type='scalar', visual=visual)
+# set variable attrs
+var.fullname = 'Cross neutral wind (Downward)'
+var.label = r'$u_{Down}$'
+var.unit = 'm/s'
+var.unit_label = r'm/s'
+var.group = r'$u$'
+# var.error = var_name + '_err'
+var.depends = {0: depend_0}
+# set plot attrs
+plot_config = var.visual.plot_config
+plot_config.config(**default_plot_config)
+plot_config.style = '1noE'
+# set axis attrs
+axis = var.visual.axis
+axis[1].data = "@v.value"
+# axis[1].lim = [np.nan, np.nan]
+axis[2].label = '@v.label'
+axis[1].unit = '@v.unit_label'
+
+configured_variables[var_name] = var
+
+####################################################################################################################
+var_name = 'SC_GEO_LAT'
+var = Var(name=var_name, ndim=1, variable_type='scalar', visual=visual)
+# set variable attrs
+var.fullname = 'S/C geographic latitude'
+var.label = r'GLAT'
+var.unit = 'degree'
+var.unit_label = r'$^\circ$'
+var.group = r'GEO'
+# var.error = var_name + '_err'
+var.depends = {0: depend_0}
+# set plot attrs
+plot_config = var.visual.plot_config
+plot_config.config(**default_plot_config)
+plot_config.style = '1noE'
+# set axis attrs
+axis = var.visual.axis
+axis[1].data = "@v.value"
+# axis[1].lim = [-2000, 2000]
+axis[1].label = '@v.label'
+axis[1].unit = '@v.unit_label'
+
+configured_variables[var_name] = var
+
+####################################################################################################################
+var_name = 'SC_GEO_LON'
+var = Var(name=var_name, ndim=1, variable_type='scalar', visual=visual)
+# set variable attrs
+var.fullname = 'S/C geographic latitude'
+var.label = r'GLON'
+var.unit = 'degree'
+var.unit_label = r'$^\circ$'
+var.group = r'GEO'
+# var.error = var_name + '_err'
+var.depends = {0: depend_0}
+# set plot attrs
+plot_config = var.visual.plot_config
+plot_config.config(**default_plot_config)
+plot_config.style = '1noE'
+# set axis attrs
+axis = var.visual.axis
+axis[1].data = "@v.value"
+# axis[1].lim = [-2000, 2000]
+axis[1].label = '@v.label'
+axis[1].unit = '@v.unit_label'
+
+configured_variables[var_name] = var

geospacelab/datahub/sources/tud/grace/dns_acc/__init__.py

@@ -78,6 +78,8 @@ class Dataset(datahub.DatasetSourced):
         self.local_latest_version = ''
         self.allow_download = kwargs.pop('allow_download', False)
         self.force_download = kwargs.pop('force_download', False)
+        self.download_dry_run = kwargs.pop('download_dry_run', False)
+        
         self.add_AACGM = kwargs.pop('add_AACGM', False) 
         self.add_APEX = kwargs.pop('add_APEX', False)
         self._data_root_dir = self.data_root_dir    # Record the initial root dir
@@ -289,7 +291,7 @@ class Dataset(datahub.DatasetSourced):
                         search_pattern=search_pattern,
                         allow_multiple_files=False
                     )
-
+        self.data_file_paths = np.unique(self.data_file_paths)
         return done
 
     def download_data(self, dt_fr=None, dt_to=None):
@@ -302,10 +304,11 @@ class Dataset(datahub.DatasetSourced):
             sat_id=self.sat_id,
             product=self.product,
             version=self.product_version,
-            force=self.force_download
+            force_download=self.force_download,
+            dry_run=self.download_dry_run
         )
 
-        return download_obj.done
+        return any(download_obj.done)
 
     @property
     def database(self):