geospacelab 0.11.4__py3-none-any.whl → 0.12.0__py3-none-any.whl

geospacelab/datahub/sources/tud/goce/dns_acc/__init__.py

@@ -0,0 +1,354 @@
+# Licensed under the BSD 3-Clause License
+# Copyright (C) 2021 GeospaceLab (geospacelab)
+# Author: Lei Cai, Space Physics and Astronomy, University of Oulu
+
+import numpy as np
+import datetime
+
+import geospacelab.datahub as datahub
+from geospacelab.datahub import DatabaseModel, FacilityModel, InstrumentModel, ProductModel
+from geospacelab.datahub.sources.tud import tud_database
+from geospacelab.datahub.sources.tud.grace import grace_facility
+from geospacelab.config import prf
+import geospacelab.toolbox.utilities.pybasic as basic
+import geospacelab.toolbox.utilities.pylogging as mylog
+import geospacelab.toolbox.utilities.pydatetime as dttool
+from geospacelab.datahub.sources.tud.goce.dns_acc.loader import Loader as default_Loader
+from geospacelab.datahub.sources.tud.goce.dns_acc.downloader import Downloader as default_Downloader
+import geospacelab.datahub.sources.tud.goce.dns_acc.variable_config as var_config
+
+
+default_dataset_attrs = {
+    'database': tud_database,
+    'facility': grace_facility,
+    'instrument': 'ACC',
+    'product': 'DNS-ACC',
+    'data_file_ext': 'txt',
+    'product_version': 'v02',
+    'data_root_dir': prf.datahub_data_root_dir / 'TUD' / 'GOCE',
+    'allow_load': True,
+    'allow_download': True,
+    'force_download': False,
+    'data_search_recursive': False,
+    'add_AACGM': False,
+    'add_APEX': False,
+    'label_fields': ['database', 'facility', 'instrument', 'product', 'product_version'],
+    'load_mode': 'AUTO',
+    'time_clip': True,
+}
+
+default_variable_names_v01 = []
+
+default_variable_names_v02 = [
+    'SC_DATETIME',
+    'SC_GEO_LAT',
+    'SC_GEO_LON',
+    'SC_GEO_ALT',
+    'SC_ARG_LAT',
+    'SC_GEO_LST',
+    'rho_n',
+    'rho_n_MEAN',
+    'FLAG',
+    'FLAG_MEAN',
+    ]
+
+# default_data_search_recursive = True
+
+default_attrs_required = []
+
+
+class Dataset(datahub.DatasetSourced):
+    def __init__(self, **kwargs):
+        kwargs = basic.dict_set_default(kwargs, **default_dataset_attrs)
+
+        super().__init__(**kwargs)
+
+        self.database = kwargs.pop('database', 'TUD')
+        self.facility = kwargs.pop('facility', 'GOCE')
+        self.instrument = kwargs.pop('instrument', 'ACC')
+        self.product = kwargs.pop('product', 'DNS-ACC')
+        self.product_version = kwargs.pop('product_version', 'v02')
+        self.local_latest_version = ''
+        self.allow_download = kwargs.pop('allow_download', False)
+        self.force_download = kwargs.pop('force_download', False)
+        self.download_dry_run = kwargs.pop('download_dry_run', False)
+        
+        self.add_AACGM = kwargs.pop('add_AACGM', False) 
+        self.add_APEX = kwargs.pop('add_APEX', False)
+        self._data_root_dir = self.data_root_dir    # Record the initial root dir
+
+        self.sat_id = kwargs.pop('sat_id', '')
+
+        self.metadata = {}
+
+        allow_load = kwargs.pop('allow_load', False)
+
+        # self.config(**kwargs)
+
+        if self.loader is None:
+            self.loader = default_Loader
+
+        if self.downloader is None:
+            self.downloader = default_Downloader
+
+        self._validate_attrs()
+
+        if allow_load:
+            self.load_data()
+
+    def _validate_attrs(self):
+        for attr_name in default_attrs_required:
+            attr = getattr(self, attr_name)
+            if not attr:
+                mylog.StreamLogger.warning("The parameter {} is required before loading data!".format(attr_name))
+
+        self.data_root_dir = self.data_root_dir / self.product.upper() / self.product_version
+
+    def label(self, **kwargs):
+        label = super().label()
+        return label
+
+    def load_data(self, **kwargs):
+        self.check_data_files(**kwargs)
+        if self.product_version == 'v01':
+            default_variable_names = default_variable_names_v01
+        else:
+            default_variable_names = default_variable_names_v02 
+        self._set_default_variables(
+            default_variable_names,
+            configured_variables=var_config.configured_variables
+        )
+        
+        if self.product_version == 'v01':
+            raise NotImplementedError
+                
+        for file_path in self.data_file_paths:
+            load_obj = self.loader(file_path, file_type='txt', version=self.product_version)
+
+            for var_name in self._variables.keys():
+                value = load_obj.variables[var_name]
+                self._variables[var_name].join(value)
+
+            # self.select_beams(field_aligned=True)
+        if self.time_clip:
+            self.time_filter_by_range(var_datetime_name='SC_DATETIME')
+
+        if self.add_AACGM:
+            self.convert_to_AACGM()
+
+        if self.add_APEX:
+            self.convert_to_APEX()
+
+    def add_GEO_LST(self):
+        lons = self['SC_GEO_LON'].flatten()
+        uts = self['SC_DATETIME'].flatten()
+        lsts = [ut + datetime.timedelta(seconds=int(lon/15.*3600)) for ut, lon in zip(uts, lons)]
+        lsts = [lst.hour + lst.minute/60. + lst.second/3600. for lst in lsts]
+        var = self.add_variable(var_name='SC_GEO_LST')
+        var.value = np.array(lsts)[:, np.newaxis]
+        var.label = 'LST'
+        var.unit = 'h'
+        var.depends = self['SC_GEO_LON'].depends
+        return var
+
+    def convert_to_APEX(self):
+        import geospacelab.cs as gsl_cs
+
+        coords_in = {
+            'lat': self['SC_GEO_LAT'].value.flatten(),
+            'lon': self['SC_GEO_LON'].value.flatten(),
+            'height': self['SC_GEO_ALT'].value.flatten()
+        }
+        dts = self['SC_DATETIME'].value.flatten()
+        cs_sph = gsl_cs.GEOCSpherical(coords=coords_in, ut=dts)
+        cs_apex = cs_sph.to_APEX(append_mlt=True)
+        self.add_variable('SC_APEX_LAT')
+        self.add_variable('SC_APEX_LON')
+        self.add_variable('SC_APEX_MLT')
+        self['SC_APEX_LAT'].value = cs_apex['lat'].reshape(self['SC_DATETIME'].value.shape)
+        self['SC_APEX_LON'].value = cs_apex['lon'].reshape(self['SC_DATETIME'].value.shape)
+        self['SC_APEX_MLT'].value = cs_apex['mlt'].reshape(self['SC_DATETIME'].value.shape)
+
+    def convert_to_AACGM(self):
+        import geospacelab.cs as gsl_cs
+
+        coords_in = {
+            'lat': self['SC_GEO_LAT'].value.flatten(),
+            'lon': self['SC_GEO_LON'].value.flatten(),
+            'height': self['SC_GEO_ALT'].value.flatten()
+        }
+        dts = self['SC_DATETIME'].value.flatten()
+        cs_sph = gsl_cs.GEOCSpherical(coords=coords_in, ut=dts)
+        cs_aacgm = cs_sph.to_AACGM(append_mlt=True)
+        self.add_variable('SC_AACGM_LAT')
+        self.add_variable('SC_AACGM_LON')
+        self.add_variable('SC_AACGM_MLT')
+        self['SC_AACGM_LAT'].value = cs_aacgm['lat'].reshape(self['SC_DATETIME'].value.shape)
+        self['SC_AACGM_LON'].value = cs_aacgm['lon'].reshape(self['SC_DATETIME'].value.shape)
+        self['SC_AACGM_MLT'].value = cs_aacgm['mlt'].reshape(self['SC_DATETIME'].value.shape)
+
+    def interp_evenly(self, time_res=None, time_res_o=10, dt_fr=None, dt_to=None, masked=False):
+        from scipy.interpolate import interp1d
+        import geospacelab.toolbox.utilities.numpymath as nm
+        if time_res is None:
+            time_res = time_res_o
+        ds_new = datahub.DatasetUser(dt_fr=self.dt_fr, dt_to=self.dt_to, visual=self.visual)
+        ds_new.clone_variables(self)
+        dts = ds_new['SC_DATETIME'].value.flatten()
+        dt0 = dttool.get_start_of_the_day(dts[0])
+        x_0 = np.array([(dt - dt0).total_seconds() for dt in dts])
+        if dt_fr is None:
+            dt_fr = dts[0] - datetime.timedelta(
+                seconds=np.floor(((dts[0] - ds_new.dt_fr).total_seconds() / time_res)) * time_res
+            )
+        if dt_to is None:
+            dt_to = dts[-1] + datetime.timedelta(
+                seconds=np.floor(((ds_new.dt_to - dts[-1]).total_seconds() / time_res)) * time_res
+            )
+        sec_fr = (dt_fr - dt0).total_seconds()
+        sec_to = (dt_to - dt0).total_seconds()
+        x_1 = np.arange(sec_fr, sec_to + time_res / 2, time_res)
+
+        f = interp1d(x_0, x_0, kind='nearest', bounds_error=False, fill_value=(x_0[0], x_0[-1]))
+        pseudo_x = f(x_1)
+        mask = np.abs(pseudo_x-x_1) > time_res_o/1.5
+
+        dts_new = np.array([dt0 + datetime.timedelta(seconds=sec) for sec in x_1])
+        ds_new['SC_DATETIME'].value = dts_new.reshape((dts_new.size, 1))
+
+        period_var_dict = {'SC_GEO_LON': 360.,
+                           'SC_AACGM_LON': 360.,
+                           'SC_APEX_LON': 360.,
+                           'SC_GEO_LST': 24.,
+                           'SC_AACGM_MLT': 24.,
+                           'SC_APEX_MLT': 24}
+
+        for var_name in ds_new.keys():
+            if var_name in ['SC_DATETIME']:
+                continue
+            if var_name in period_var_dict.keys():
+                var = ds_new[var_name].value.flatten()
+                var_new = nm.interp_period_data(x_0, var, x_1, period=period_var_dict[var_name], method='linear', bounds_error=False)
+            else:
+                method = 'linear' if 'FLAG' not in var_name else 'nearest'
+                var = ds_new[var_name].value.flatten()
+                f = interp1d(x_0, var, kind=method, bounds_error=False)
+                var_new = f(x_1)
+            if masked:
+                var_new = np.ma.array(var_new, mask=mask, fill_value=np.nan)
+            else:
+                var_new[mask] = np.nan
+            ds_new[var_name].value = var_new.reshape((dts_new.size, 1))
+        return ds_new
+
+    def search_data_files(self, **kwargs):
+
+        dt_fr = self.dt_fr
+        dt_to = self.dt_to
+
+        diff_months = dttool.get_diff_months(dt_fr, dt_to)
+
+        dt0 = dttool.get_first_day_of_month(self.dt_fr)
+
+        for i in range(diff_months + 1):
+            this_day = dttool.get_next_n_months(dt0, i)
+
+            initial_file_dir = kwargs.pop(
+                'initial_file_dir', self.data_root_dir
+            )
+
+            file_patterns = [
+                'GO',
+                self.product.upper().replace('-', '_'),
+                this_day.strftime('%Y_%m'),
+            ]
+            # remove empty str
+            file_patterns = [pattern for pattern in file_patterns if str(pattern)]
+            search_pattern = '*' + '*'.join(file_patterns) + '*'
+
+            done = super().search_data_files(
+                initial_file_dir=initial_file_dir,
+                search_pattern=search_pattern,
+                allow_multiple_files=False,
+            )
+            # Validate file paths
+
+            if (not done and self.allow_download) or self.force_download:
+                done = self.download_data()
+                if done:
+                    initial_file_dir = self.data_root_dir
+                    done = super().search_data_files(
+                        initial_file_dir=initial_file_dir,
+                        search_pattern=search_pattern,
+                        allow_multiple_files=False
+                    )
+        self.data_file_paths = np.unique(self.data_file_paths)
+        return done
+
+    def download_data(self, dt_fr=None, dt_to=None):
+        if dt_fr is None:
+            dt_fr = self.dt_fr
+        if dt_to is None:
+            dt_to = self.dt_to
+        download_obj = self.downloader(
+            dt_fr, dt_to,
+            sat_id=self.sat_id,
+            product=self.product,
+            version=self.product_version,
+            force_download=self.force_download,
+            dry_run=self.download_dry_run
+        )
+
+        return any(download_obj.done)
+
+    @property
+    def database(self):
+        return self._database
+
+    @database.setter
+    def database(self, value):
+        if isinstance(value, str):
+            self._database = DatabaseModel(value)
+        elif issubclass(value.__class__, DatabaseModel):
+            self._database = value
+        else:
+            raise TypeError
+
+    @property
+    def product(self):
+        return self._product
+
+    @product.setter
+    def product(self, value):
+        if isinstance(value, str):
+            self._product = ProductModel(value)
+        elif issubclass(value.__class__, ProductModel):
+            self._product = value
+        else:
+            raise TypeError
+
+    @property
+    def facility(self):
+        return self._facility
+
+    @facility.setter
+    def facility(self, value):
+        if isinstance(value, str):
+            self._facility = FacilityModel(value)
+        elif issubclass(value.__class__, FacilityModel):
+            self._facility = value
+        else:
+            raise TypeError
+
+    @property
+    def instrument(self):
+        return self._instrument
+
+    @instrument.setter
+    def instrument(self, value):
+        if isinstance(value, str):
+            self._instrument = InstrumentModel(value)
+        elif issubclass(value.__class__, InstrumentModel):
+            self._instrument = value
+        else:
+            raise TypeError

geospacelab/datahub/sources/tud/goce/dns_acc/downloader.py

@@ -0,0 +1,42 @@
+# Licensed under the BSD 3-Clause License
+# Copyright (C) 2021 GeospaceLab (geospacelab)
+# Author: Lei Cai, Space Physics and Astronomy, University of Oulu
+
+__author__ = "Lei Cai"
+__copyright__ = "Copyright 2021, GeospaceLab"
+__license__ = "BSD-3-Clause License"
+__email__ = "lei.cai@oulu.fi"
+__docformat__ = "reStructureText"
+
+import datetime
+from geospacelab.config import prf
+
+from geospacelab.datahub.sources.tud.downloader import TUDownloader as DownloaderBase   
+
+
+class Downloader(DownloaderBase):
+    
+    def __init__(self, 
+        dt_fr, dt_to, 
+        mission='GOCE', 
+        sat_id='',
+        version='v02',  
+        product='DNS-ACC',
+        direct_download=True,
+        force_download=False,
+        dry_run=False,
+        ):
+        self.mission = mission
+        self.version = version
+        
+        super().__init__(
+            dt_fr, dt_to, 
+            mission=mission,
+            sat_id=sat_id, 
+            version=version,
+            product=product,
+            direct_download=direct_download,
+            force_download=force_download,
+            dry_run=dry_run
+        )
+        

geospacelab/datahub/sources/tud/goce/dns_acc/loader.py

@@ -0,0 +1,66 @@
+# Licensed under the BSD 3-Clause License
+# Copyright (C) 2021 GeospaceLab (geospacelab)
+# Author: Lei Cai, Space Physics and Astronomy, University of Oulu
+
+__author__ = "Lei Cai"
+__copyright__ = "Copyright 2021, GeospaceLab"
+__license__ = "BSD-3-Clause License"
+__email__ = "lei.cai@oulu.fi"
+__docformat__ = "reStructureText"
+
+import re
+import numpy as np
+import datetime
+import geospacelab.toolbox.utilities.pydatetime as dttool
+
+
+class Loader(object):
+
+    def __init__(self, file_path, file_type='txt', version='v01', direct_load=True):
+        self.file_path = file_path
+        self.file_type = file_type
+        self.version = version
+        self.variables = {}
+        self.done = False
+        self.t_res = 10
+        if direct_load:
+            self.load()
+
+    def load(self):
+
+        if self.version == 'v01':
+            self.load_v01()
+        elif self.version == 'v02':
+            self.load_v02()
+        else:
+            raise NotImplementedError
+
+    def load_v01(self):
+        raise NotImplementedError("GOCE DNS-ACC v01 data loading is not implemented.")
+    
+    def load_v02(self):
+        with open(self.file_path, 'r') as f:
+            text = f.read()
+            results = re.findall(
+            r"^(\d{4}-\d{2}-\d{2}\s*\d{2}\:\d{2}\:\d{2}\.\d{3})\s*(\w{3})\s*" 
+            + r"([\-\d.]+)\s*([\-\d.]+)\s*([\-\d.]+)\s*([\-\d.]+)\s*([\-\d.]+)\s*([+\-\d.Ee]+)\s*([+\-\d.Ee]+)\s*([\-\d.]+)\s*([\-\d.]+)",
+            text,
+            re.M)
+            results = list(zip(*results))
+            dts = [datetime.datetime.strptime(dtstr + '000', "%Y-%m-%d %H:%M:%S.%f") for dtstr in results[0]]
+            num_rec = len(dts)
+            if results[1][0] == 'GPS':
+                self.variables['SC_GPSTIME'] = np.array(dts).reshape(num_rec, 1)
+                t_gps = [(dt - dttool._GPS_DATETIME_0).total_seconds() for dt in dts]
+                dts = dttool.convert_gps_time_to_datetime(t_gps, weeks=None)
+
+            self.variables['SC_DATETIME'] = np.array(dts).reshape(num_rec, 1)
+            self.variables['SC_GEO_ALT'] = np.array(results[2]).astype(np.float32).reshape(num_rec, 1) * 1e-3   # in km
+            self.variables['SC_GEO_LON'] = np.array(results[3]).astype(np.float32).reshape(num_rec, 1)
+            self.variables['SC_GEO_LAT'] = np.array(results[4]).astype(np.float32).reshape(num_rec, 1)
+            self.variables['SC_GEO_LST'] = np.array(results[5]).astype(np.float32).reshape(num_rec, 1)
+            self.variables['SC_ARG_LAT'] = np.array(results[6]).astype(np.float32).reshape(num_rec, 1)
+            self.variables['rho_n'] = np.array(results[7]).astype(np.float32).reshape(num_rec, 1)
+            self.variables['rho_n_MEAN'] = np.array(results[8]).astype(np.float32).reshape(num_rec, 1)
+            self.variables['FLAG'] = np.array(results[9]).astype(np.float32).reshape(num_rec, 1)
+            self.variables['FLAG_MEAN'] = np.array(results[10]).astype(np.float32).reshape(num_rec, 1)

geospacelab/datahub/sources/tud/goce/dns_acc/variable_config.py

@@ -0,0 +1,139 @@
+# Licensed under the BSD 3-Clause License
+# Copyright (C) 2021 GeospaceLab (geospacelab)
+# Author: Lei Cai, Space Physics and Astronomy, University of Oulu
+
+__author__ = "Lei Cai"
+__copyright__ = "Copyright 2021, GeospaceLab"
+__license__ = "BSD-3-Clause License"
+__email__ = "lei.cai@oulu.fi"
+__docformat__ = "reStructureText"
+
+from geospacelab.datahub import VariableModel as Var
+import geospacelab.visualization.mpl.colormaps as cm
+
+import numpy as np
+
+database = 'ESA/EarthOnline'
+
+timestamps = {
+    'SC_DATETIME': 'SC_DATETIME',
+}
+
+
+default_colormap = "gist_ncar"
+
+default_plot_config = {
+    'line':         {
+        'linestyle':        '-',
+        'linewidth':        1.5,
+        'marker':           '',
+        'markersize':       3,
+    },
+    'pcolormesh':   {
+        'cmap':            default_colormap,
+    }
+}
+
+configured_variables = {}
+visual = 'on'
+
+depend_0 = {'UT': 'SC_DATETIME',
+            'GEO_LAT': 'SC_GEO_LAT', 'GEO_LON': 'SC_GEO_LON',
+            'AACGM_LAT': 'SC_AACGM_LAT', 'AACGM_LON': 'SC_AACGM_LON', 'AACGM_MLT': 'SC_AACGM_MLT'}
+# depend_c = {'SPECTRA': 'EMISSION_SPECTRA'}
+
+####################################################################################################################
+var_name = 'rho_n'
+var = Var(name=var_name, ndim=1, variable_type='scalar', visual=visual)
+# set variable attrs
+var.fullname = 'Neutral mass density'
+var.label = r'$\rho_n$'
+var.unit = 'kg/m-3'
+var.unit_label = r'kg$\cdot$m$^{-3}$'
+var.group = r'$\rho$'
+# var.error = var_name + '_err'
+var.depends = {0: depend_0}
+# set plot attrs
+plot_config = var.visual.plot_config
+plot_config.config(**default_plot_config)
+plot_config.style = '1noE'
+# set axis attrs
+axis = var.visual.axis
+axis[1].data = "@v.value"
+# axis[1].lim = [np.nan, np.nan]
+axis[2].label = '@v.label'
+axis[1].unit = '@v.unit_label'
+
+configured_variables[var_name] = var
+
+####################################################################################################################
+var_name = 'rho_n_MEAN'
+var = Var(name=var_name, ndim=1, variable_type='scalar', visual=visual)
+# set variable attrs
+var.fullname = 'Neutral mass density averaged per orbit'
+var.label = r'$\bar{\rho}_n$'
+var.unit = 'kg/m-3'
+var.unit_label = r'kg$\cdot$m$^{-3}$'
+var.group = r'$\rho$'
+# var.error = var_name + '_err'
+var.depends = {0: depend_0}
+# set plot attrs
+plot_config = var.visual.plot_config
+plot_config.config(**default_plot_config)
+plot_config.style = '1noE'
+# set axis attrs
+axis = var.visual.axis
+axis[1].data = "@v.value"
+# axis[1].lim = [np.nan, np.nan]
+axis[2].label = '@v.label'
+axis[1].unit = '@v.unit_label'
+
+configured_variables[var_name] = var
+
+####################################################################################################################
+var_name = 'SC_GEO_LAT'
+var = Var(name=var_name, ndim=1, variable_type='scalar', visual=visual)
+# set variable attrs
+var.fullname = 'S/C geographic latitude'
+var.label = r'GLAT'
+var.unit = 'degree'
+var.unit_label = r'$^\circ$'
+var.group = r'GEO'
+# var.error = var_name + '_err'
+var.depends = {0: depend_0}
+# set plot attrs
+plot_config = var.visual.plot_config
+plot_config.config(**default_plot_config)
+plot_config.style = '1noE'
+# set axis attrs
+axis = var.visual.axis
+axis[1].data = "@v.value"
+# axis[1].lim = [-2000, 2000]
+axis[1].label = '@v.label'
+axis[1].unit = '@v.unit_label'
+
+configured_variables[var_name] = var
+
+####################################################################################################################
+var_name = 'SC_GEO_LON'
+var = Var(name=var_name, ndim=1, variable_type='scalar', visual=visual)
+# set variable attrs
+var.fullname = 'S/C geographic latitude'
+var.label = r'GLON'
+var.unit = 'degree'
+var.unit_label = r'$^\circ$'
+var.group = r'GEO'
+# var.error = var_name + '_err'
+var.depends = {0: depend_0}
+# set plot attrs
+plot_config = var.visual.plot_config
+plot_config.config(**default_plot_config)
+plot_config.style = '1noE'
+# set axis attrs
+axis = var.visual.axis
+axis[1].data = "@v.value"
+# axis[1].lim = [-2000, 2000]
+axis[1].label = '@v.label'
+axis[1].unit = '@v.unit_label'
+
+configured_variables[var_name] = var

geospacelab/datahub/sources/tud/goce/dns_wnd_acc/__init__.py

@@ -14,9 +14,9 @@ import geospacelab.toolbox.utilities.pybasic as basic
 import geospacelab.toolbox.utilities.pylogging as mylog
 import geospacelab.toolbox.utilities.pydatetime as dttool
 
-from geospacelab.datahub.sources.tud.goce.dns_wnd_acc.loader import Loader as default_Loader
-from geospacelab.datahub.sources.tud.goce.dns_wnd_acc.downloader import Downloader as default_Downloader
-import geospacelab.datahub.sources.tud.goce.dns_wnd_acc.variable_config as var_config
+from geospacelab.datahub.sources.tud.goce.wnd_acc.loader import Loader as default_Loader
+from geospacelab.datahub.sources.tud.goce.wnd_acc.downloader import Downloader as default_Downloader
+import geospacelab.datahub.sources.tud.goce.wnd_acc.variable_config as var_config
 
 
 default_dataset_attrs = {