geospacelab 0.10.0__tar.gz → 0.10.1__tar.gz

{geospacelab-0.10.0/geospacelab.egg-info → geospacelab-0.10.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: geospacelab
-Version: 0.10.0
+Version: 0.10.1
 Summary: Collect, manage, and visualize geospace data.
 Home-page: https://github.com/JouleCai/geospacelab
 Author: Lei Cai

{geospacelab-0.10.0 → geospacelab-0.10.1}/geospacelab/__init__.py

@@ -6,7 +6,7 @@ __author__ = "Lei Cai"
 __copyright__ = "Copyright 2021, GeospaceLAB"
 __credits__ = ["Lei Cai"]
 __license__ = "BSD-3-Clause License"
-__version__ = "0.10.0"
+__version__ = "0.10.1"
 __maintainer__ = "Lei Cai"
 __email__ = "lei.cai@oulu.fi"
 __status__ = "Developing"

{geospacelab-0.10.0 → geospacelab-0.10.1}/geospacelab/datahub/sources/madrigal/isr/eiscat/__init__.py

@@ -72,7 +72,8 @@ class Dataset(datahub.DatasetSourced):
         self.data_file_type = kwargs.pop('data_file_type', '')
         self.affiliation = kwargs.pop('affiliation', '')
         self.allow_download = kwargs.pop('allow_download', True)
-        self.metadata = None
+        self.gate_num = kwargs.pop('gate_num', None)
+        self.metadata = {}
         self.add_AACGM = kwargs.pop('add_AACGM', True)
         self.add_APEX = kwargs.pop('add_APEX', False)
 
@@ -116,7 +117,9 @@ class Dataset(datahub.DatasetSourced):
         self.check_data_files(**kwargs)
 
         for file_path in self.data_file_paths:
-            load_obj = self.loader(file_path, file_type=self.data_file_type)
+            load_obj = self.loader(file_path, file_type=self.data_file_type, gate_num=self.gate_num)
+            if self.gate_num is None:
+                self.gate_num = load_obj.gate_num
 
             for var_name in self._variables.keys():
                 self._variables[var_name].join(load_obj.variables[var_name])
@@ -130,6 +133,12 @@ class Dataset(datahub.DatasetSourced):
             self.experiment = rawdata_path.split('/')[-1].split('@')[0]
             self.affiliation = load_obj.metadata['affiliation']
             self.metadata = load_obj.metadata
+
+        inds_cmb = np.argsort(self['DATETIME'].flatten())
+        if any(np.diff(np.array(inds_cmb))<0):
+            for var_name in self.keys():
+                self[var_name].value = self[var_name].value[inds_cmb, :]
+
         if self.add_AACGM or self.add_APEX:
             self.calc_lat_lon()
             # self.select_beams(field_aligned=True)
@@ -357,21 +366,23 @@ class Dataset(datahub.DatasetSourced):
             fps_sub = []
             for ii in inds_id:
                 fp = file_paths[ii]
-                rc = re.compile(r".*_([\d]{8}T[\d]{6}).*_([\d]{8}T[\d]{6}).*[\d]{4}\-[\d]{2}\-[\d]{2}_([\w]+)@.*")
+                rc = re.compile(r".*_([\d]{8}T[\d]{6}).*_([\d]{8}T[\d]{6}).*[\d]{4}\-[\d]{2}\-[\d]{2}_([\w.]+)@.*")
                 res = rc.search(str(fp))
                 dt_0 = datetime.datetime.strptime(res.groups()[0], '%Y%m%dT%H%M%S')
                 dt_1 = datetime.datetime.strptime(res.groups()[1], '%Y%m%dT%H%M%S')
                 if (dt_0 >= self.dt_to) or (dt_1<=self.dt_fr):
                     continue
                 if str(self.pulse_code):
-                    if self.pulse_code not in res.groups()[2]:
+                    if self.pulse_code not in res.groups()[2].lower():
                         continue
                 if str(self.modulation):
-                    if self.modulation not in res.groups()[2]:
+                    if self.modulation not in res.groups()[2].lower():
                         continue
+                if '_v' in res.groups()[2].lower():
+                    continue
                 fps_sub.extend([fp])
             if len(fps_sub) > 1:
-                mylog.StreamLogger.warning("Multiple data files for a single experiment detected! Only the first one is selected.")
+                mylog.StreamLogger.warning("Multiple data files for a single experiment detected!")
                 # for fp in fps_sub:
                 #     mylog.simpleinfo.info(str(fp))
                 # fps_sub = fps_sub[0]

{geospacelab-0.10.0 → geospacelab-0.10.1}/geospacelab/datahub/sources/madrigal/isr/eiscat/downloader.py

@@ -207,8 +207,8 @@ class Downloader(DownloaderBase):
             dt_to_exp = datetime.datetime(
                 exp.endyear, exp.endmonth, exp.endday, exp.endhour, exp.endmin, exp.endsec
             )
-            if (dt_fr_exp >= self.dt_to) or (dt_to_exp <= self.dt_fr):
-                continue
+            # if (dt_fr_exp >= self.dt_to) or (dt_to_exp <= self.dt_fr):
+            #     continue
             for file in list(exp.files):
 
                 file_path_remote = pathlib.Path(file.name)

{geospacelab-0.10.0 → geospacelab-0.10.1}/geospacelab/datahub/sources/madrigal/isr/eiscat/loader.py

@@ -13,6 +13,7 @@ import h5py
 import numpy as np
 import pathlib
 import re
+import scipy.interpolate as si
 
 from geospacelab.config import pref as prf
 
@@ -32,12 +33,12 @@ default_variable_names = [
 
 
 class Loader:
-    def __init__(self, file_path, file_type="eiscat-hdf5"):
+    def __init__(self, file_path, file_type="eiscat-hdf5", gate_num=None):
         self.variables = {}
         self.metadata = {}
         self.file_path = file_path
         self.file_type = file_type
-
+        self.gate_num = gate_num
         self.load_data()
 
     def load_data(self):
@@ -131,8 +132,19 @@ class Loader:
                         if num_col != nrec[0]:
                             print("Note: the number of range gates doesn't match nrec!")
                         var = var.reshape(num_row, num_col)
+                        if self.gate_num is None:
+                            self.gate_num = num_col
+                        else:
+                            var_array = np.empty((num_row, num_gates))
+                            var_array[::] = np.nan
+                            for i in range(num_row):
+                                var_array[i, 0:num_col] = var[i, :]
+                            var = var_array
                     elif nrec_group == 'par1d':
-                        num_gates = int(np.max(nrec))
+                        if self.gate_num is None:
+                            self.gate_num = num_gates = int(np.max(nrec))
+                        else:
+                            num_gates = self.gate_num
                         var_array = np.empty((num_row, num_gates))
                         var_array[:, :] = np.nan
                         rec_ind_1 = 0
@@ -201,21 +213,30 @@ class Loader:
         # check height and range
         vars['HEIGHT'] = vars['HEIGHT'] / 1000.
         vars['RANGE'] = vars['RANGE'] / 1000.
-        inds = np.where(np.isnan(vars['HEIGHT']))
-        for i in range(len(inds[0])):
-            ind_0 = inds[0][i]
-            ind_1 = inds[1][i]
-            x0 = np.arange(vars['HEIGHT'].shape[0])
-            y0 = vars['HEIGHT'][:, ind_1]
-            xp = x0[np.where(np.isfinite(y0))[0]]
-            yp = y0[np.isfinite(y0)]
-            vars['HEIGHT'][ind_0, ind_1] = np.interp(x0[ind_0], xp, yp)
-
-            x0 = np.arange(vars['RANGE'].shape[0])
-            y0 = vars['RANGE'][:, ind_1]
-            xp = x0[np.isfinite(y0)]
-            yp = y0[np.isfinite(y0)]
-            vars['RANGE'][ind_0, ind_1] = np.interp(x0[ind_0], xp, yp)
+
+        inds_nan = np.where(np.isnan(vars['HEIGHT']))
+        if list(inds_nan):
+            m, n = vars['HEIGHT'].shape
+            for i in range(m):
+                yy = vars['HEIGHT'][i, :].flatten()
+                iii = np.where(~np.isfinite(yy))
+                if not list(iii):
+                    continue
+                iii = np.where(np.isfinite(yy))[0]
+                xx = np.arange(0, n)
+                f = si.interp1d(xx[iii], yy[iii], kind='linear', bounds_error=False, fill_value='extrapolate')
+                yy_new = f(xx)
+                vars['HEIGHT'][i, :] = yy_new
+
+                yy = vars['RANGE'][i, :].flatten()
+                iii = np.where(~np.isfinite(yy))
+                if not list(iii):
+                    continue
+                iii = np.where(np.isfinite(yy))[0]
+                xx = np.arange(0, n)
+                f = si.interp1d(xx[iii], yy[iii], kind='linear', bounds_error=False, fill_value='extrapolate')
+                yy_new = f(xx)
+                vars['RANGE'][i, :] = yy_new
 
         if np.isscalar(vars['AZ']):
             az = np.empty((vars['DATETIME_1'].shape[0], 1))
@@ -336,8 +357,12 @@ class Loader:
             inds_ran_max = np.append(inds_ran_max, len(ran)-1)
             inds_ran_max.sort()
 
-            ngates_max = np.max(np.diff(inds_ran_max))
-            data_array = np.empty((nvar_h5, inds_ran_min.shape[0], ngates_max))
+            if self.gate_num is None:
+                self.gate_num = ngates_max = np.max(np.diff(inds_ran_max))
+            else:
+                ngates_max = self.gate_num
+            num_row = inds_ran_min.shape[0]
+            data_array = np.empty((nvar_h5, num_row, ngates_max))
             data_array[::] = np.nan
             for ip in range(nvar_h5):
                 var_tmp = np.array(data[ip])
@@ -345,7 +370,7 @@ class Loader:
                     ind1 = inds_ran_min[i]
                     ind2 = inds_ran_max[i]
                     data_array[ip, i, 0: ind2-ind1+1] = var_tmp[ind1: ind2+1]
-            num_row = inds_ran_min.shape[0]
+
             vars_h5 = {}
             for ip in range(nvar_h5):
 
@@ -382,21 +407,31 @@ class Loader:
             vars['T_e_err'] = vars['T_e'] * np.sqrt((vars['T_i_err'] / vars['T_i']) ** 2
                                                     + (vars['T_r_err'] / vars['T_r']) ** 2)
             vars['AZ'] = vars['AZ'] % 360.
-            inds = np.where(np.isnan(vars['HEIGHT']))
-            for i in range(len(inds[0])):
-                ind_0 = inds[0][i]
-                ind_1 = inds[1][i]
-                x0 = np.arange(vars['HEIGHT'].shape[0])
-                y0 = vars['HEIGHT'][:, ind_1]
-                xp = x0[np.where(np.isfinite(y0))[0]]
-                yp = y0[np.isfinite(y0)]
-                vars['HEIGHT'][ind_0, ind_1] = np.interp(x0[ind_0], xp, yp)
-
-                x0 = np.arange(vars['RANGE'].shape[0])
-                y0 = vars['RANGE'][:, ind_1]
-                xp = x0[np.isfinite(y0)]
-                yp = y0[np.isfinite(y0)]
-                vars['RANGE'][ind_0, ind_1] = np.interp(x0[ind_0], xp, yp)
+
+            inds_nan = np.where(np.isnan(vars['HEIGHT']))
+            if list(inds_nan):
+                m, n = vars['HEIGHT'].shape
+                for i in range(m):
+                    yy = vars['HEIGHT'][i, :].flatten()
+                    iii = np.where(~np.isfinite(yy))
+                    if not list(iii):
+                        continue
+                    iii = np.where(np.isfinite(yy))[0]
+                    xx = np.arange(0, n)
+                    f = si.interp1d(xx[iii], yy[iii], kind='linear', bounds_error=False, fill_value='extrapolate')
+                    yy_new = f(xx)
+                    vars['HEIGHT'][i, :] = yy_new
+
+                    yy = vars['RANGE'][i, :].flatten()
+                    iii = np.where(~np.isfinite(yy))
+                    if not list(iii):
+                        continue
+                    iii = np.where(np.isfinite(yy))[0]
+                    xx = np.arange(0, n)
+                    f = si.interp1d(xx[iii], yy[iii], kind='linear', bounds_error=False, fill_value='extrapolate')
+                    yy_new = f(xx)
+                    vars['RANGE'][i, :] = yy_new
+
 
             self.variables = vars
             self.metadata = metadata

{geospacelab-0.10.0 → geospacelab-0.10.1}/geospacelab/toolbox/utilities/numpyarray.py

@@ -10,9 +10,11 @@ __docformat__ = "reStructureText"
 
 
 import numpy
-from scipy.interpolate import interp1d
+import numpy as np
+from scipy.interpolate import interp1d, griddata
 import geospacelab.toolbox.utilities.pydatetime as dttool
 import geospacelab.toolbox.utilities.pybasic as basic
+import geospacelab.toolbox.utilities.pylogging as mylog
 import datetime
 
 
@@ -133,6 +135,140 @@ def data_resample(
 
     return xnew, ynew
 
+# def resample_2d(
+#         x_data, y_data, z_data,
+#         x_data_type=None,
+#         x_data_res=None,
+#         x_data_res_scale=1.,
+#         x_grid_res=None,
+#         x_grid_res_scale=1,
+#         x_grid_min=None,
+#         x_grid_max=None,
+#         along_x_interp=True,
+#         along_x_interp_method='nearest',
+#         along_x_binning=False,
+#         y_data_res=None,
+#         y_data_res_scale=1.,
+#         y_grid_res=None,
+#         y_grid_res_scale=1.,
+#         y_grid_min=None,
+#         y_grid_max=None,
+#         along_y_interp=True,
+#         along_y_interp_method='nearest',
+#         along_y_binning=False,
+# ):
+#     if along_x_interp==True and along_x_binning==True:
+#         mylog.StreamLogger.error('The keywords "along_x_interp" and "along_x_binning" cannot be True at the same time!')
+#         raise ValueError
+#     if along_y_interp==True and along_y_binning==True:
+#         mylog.StreamLogger.error('The keywords "along_y_interp" and "along_y_binning" cannot be True at the same time!')
+#         raise ValueError
+#
+#     num_x, num_y = z_data.shape
+#     if x_data_type == 'datetime':
+#         dt0 = dttool.get_start_of_the_day(numpy.nanmin(x_data.flatten()))
+#         sectime, dt0 = dttool.convert_datetime_to_sectime(x_data, dt0=dt0)
+#         x_data = sectime
+#
+#     # check x dim:
+#     if len(x_data.shape) == 1:
+#         x_dim = 1
+#     elif len(x_data.shape) == 2:
+#         if x_data.shape[1] == num_y:
+#             x_dim = 2
+#         else:
+#             x_dim = 1
+#     else:
+#         raise ValueError
+#     if x_dim == 1:
+#         xd = np.tile(x_data.flatten(), (1, num_y))
+#     else:
+#         xd = x_data
+#     min_x = np.nanmin(xd.flatten())
+#     max_x = np.nanmax(xd.flatten())
+#     # check y dim:
+#     if len(y_data.shape) == 1:
+#         y_dim = 1
+#     elif len(y_data.shape) == 2:
+#         if y_data.shape[1] == num_y:
+#             y_dim = 2
+#         else:
+#             y_dim = 1
+#     else:
+#         raise ValueError
+#     if y_dim == 1:
+#         yd = np.tile(x_data.flatten(), (num_x, 1))
+#     else:
+#         yd = y_data
+#     min_y = np.nanmin(yd.flatten())
+#     max_y = np.nanmax(yd.flatten())
+#
+#     if x_data_res is None:
+#         x_data_res_ = np.median(np.diff(xd[:, 0].flatten()))
+#     else:
+#         x_data_res_ = x_data_res
+#     if y_data_res is None:
+#         y_data_res_ = np.median(np.diff(yd[0, :].flatten()))
+#     else:
+#         y_data_res_ = y_data_res
+#
+#     if x_grid_res is None:
+#         xx = xd[:, 0].flatten()
+#         # along_x_interp=False
+#         # along_x_binning=False
+#         # x_grid_res=x_data_res
+#     else:
+#         if x_grid_min is None:
+#             x_grid_min = np.floor((min_x / x_grid_res)) * x_grid_res
+#         if x_grid_max is None:
+#             x_grid_max = np.ceil((max_x / x_grid_res)) * x_grid_res
+#         xx = np.arange(x_grid_min, x_grid_max+x_grid_res, x_grid_res)
+#     if y_grid_res is None:
+#         yy = yd[0, :].flatten()
+#         # along_y_interp=False
+#         # along_y_binning=False
+#         # y_grid_res=y_data_res
+#     else:
+#         if y_grid_min is None:
+#             y_grid_min = np.floor((min_y / y_grid_res)) * y_grid_res
+#         if y_grid_max is None:
+#             y_grid_max = np.ceil((max_y / y_grid_res)) * y_grid_res
+#         yy = np.arange(y_grid_min, y_grid_max+y_grid_res, y_grid_res)
+#
+#     grid_x, grid_y = numpy.meshgrid(xx,yy)
+#     grid_z = np.empty_like(grid_x)
+#     grid_z[::] = np.nan
+#
+#     if along_y_interp:
+#         grid_x_1 = np.empty_like((num_x, grid_z.shape[1])) * np.nan
+#         grid_y_1 = np.empty_like((num_x, grid_z.shape[1])) * np.nan
+#         grid_z_1 = np.empty_like((num_x, grid_z.shape[1])) * np.nan
+#         for i in range(num_x):
+#             x1 = xd[i, :].flatten()
+#             y1 = yd[i, :].flatten()
+#             z1 = z_data[i, :].flatten()
+#
+#             inds_finite = np.where(np.isfinite(x1) & np.isfinite(y1) & np.isfinite(z1))[0]
+#             if not list(inds_finite):
+#                 continue
+#
+#             f = interp1d(y1[inds_finite], x1[inds_finite],
+#                 kind='nearest', bounds_error=False, fill_value=np.nan)
+#             x_i = f(grid_y[i, :].flatten())
+#             grid_x_1[i, :] = x_i
+#
+#             f = interp1d(y1[inds_finite], y1[inds_finite],
+#                 kind='nearest', bounds_error=False, fill_value=np.nan)
+#             y_i = f(grid_y[i, :].flatten())
+#             grid_y_1[i, :] = y_i
+#
+#             f = interp1d(
+#                 y1[inds_finite], z1[inds_finite],
+#                 kind=along_y_interp_method, bounds_error=False, fill_value=np.nan)
+#             z_i = f(grid_y[i, :].flatten())
+#
+#             grid_z_1[i, :] = z_i
+
 
 def regridding_2d_xgaps(
     x, y, z,

{geospacelab-0.10.0 → geospacelab-0.10.1}/geospacelab/visualization/mpl/panels.py

@@ -736,6 +736,7 @@ class TSPanel(Panel):
         time_gap = var.visual.axis[0].mask_gap
         if time_gap is None:
             time_gap = self.time_gap
+        y_data = self._check_ydata(y_data)
         if time_gap:
             # x, y, z = arraytool.data_resample_2d(
             #     x=x_data, y=y_data, z=z_data, xtype='datetime', xres=x_data_res, method='Null', axis=0)
@@ -748,6 +749,21 @@ class TSPanel(Panel):
         data = {'x': x, 'y': y, 'z': z}
         return data
 
+    def _check_ydata(self, ydata):
+        inds_infinite = np.where(~np.isfinite(ydata))
+        if list(inds_infinite):
+            m, n = ydata.shape
+            for i in range(m):
+                yy = ydata[i, :].flatten()
+                iii = np.where(~np.isfinite(yy))
+                if not list(iii):
+                    continue
+                iii = np.where(np.isfinite(yy))[0]
+                xx = np.arange(0, n)
+                f = interp1d(xx[iii], yy[iii], kind='linear', bounds_error=False, fill_value='extrapolate')
+                yy_new = f(xx)
+                ydata[i, :] = yy_new
+        return ydata
     @staticmethod
     def generate_label(label: str, unit: str='', style: str='double'):
         label = label

{geospacelab-0.10.0 → geospacelab-0.10.1/geospacelab.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: geospacelab
-Version: 0.10.0
+Version: 0.10.1
 Summary: Collect, manage, and visualize geospace data.
 Home-page: https://github.com/JouleCai/geospacelab
 Author: Lei Cai