geospacelab 0.10.2__py3-none-any.whl → 0.10.3__py3-none-any.whl

geospacelab/__init__.py

@@ -6,7 +6,7 @@ __author__ = "Lei Cai"
 __copyright__ = "Copyright 2021, GeospaceLAB"
 __credits__ = ["Lei Cai"]
 __license__ = "BSD-3-Clause License"
-__version__ = "0.10.2"
+__version__ = "0.10.3"
 __maintainer__ = "Lei Cai"
 __email__ = "lei.cai@oulu.fi"
 __status__ = "Developing"

geospacelab/datahub/sources/madrigal/isr/millstonehill/basic/__init__.py

@@ -43,7 +43,7 @@ default_dataset_attrs = {
 }
 
 default_variable_names = [
-    'DATETIME', 'AZ', 'AZ1', 'AZ2', 'EL', 'EL1', 'EL2', 'PULSE_LENGTH',
+    'DATETIME', 'DATETIME_1', 'DATETIME_2', 'AZ', 'AZ1', 'AZ2', 'EL', 'EL1', 'EL2', 'PULSE_LENGTH',
     'T_SYS', 'POWER_NORM', 'P_Tx', 'MODE_TYPE', 'POWER_LENGTH_F',
     'LAG_SPACING', 'IPP', 'f_Tx', 'v_PHASE_Tx', 'v_PHASE_Tx_err',
     'SCAN_TYPE', 'CYCN', 'POSN', 'RANGE_RES', 'RANGE',
@@ -52,7 +52,7 @@ default_variable_names = [
     'n_e_err', 'T_i', 'T_i_err', 'T_r', 'T_r_err', 'T_e', 'T_e_err',
     'nu_i', 'nu_i_err', 'v_i_los', 'v_i_los_err', 'comp_H_p',
     'comp_H_p_err', 'comp_mix', 'comp_mix_err', 'v_DOP_los', 'v_DOP_los_err',
-    'HEIGHT'
+    'HEIGHT', 
 ]
 
 # default_data_search_recursive = True

geospacelab/datahub/sources/madrigal/isr/millstonehill/basic/loader.py

@@ -29,6 +29,9 @@ antenna_code_dict = {
     'misa': 31,
 }
 
+antenna_code_dict_r = {v: k for k, v in antenna_code_dict.items()}
+pulse_code_dict_r = {v: k for k, v in pulse_code_dict.items()}
+
 var_name_dict = {
     'AZ1': 'az1',
     'AZ2': 'az2',
@@ -98,7 +101,200 @@ class Loader:
 
         self.done = False
         if load_data:
-            self.load()
+            self.load_from_table_layout()
+
+
+    def load_from_table_layout(self):
+        variables = {}
+
+        with h5py.File(self.file_path, 'r') as fh5:
+            table = fh5['Data']['Table Layout'][::]
+            table_dtype = table.dtype
+
+            # Get variable names
+            table_var_names = list(table_dtype.fields.keys())
+            # Get table data
+            table_data = list(zip(*table))
+            table_vars = {k: np.array(table_data[table_var_names.index(k)]) for k in table_var_names}
+            # Check modes
+            antenna_codes = table_vars['kinst'] = table_vars['kinst'].astype(np.int32)
+            pulse_lengths = table_vars['pl'] = table_vars['pl'] * 1e6
+            pulse_codes = table_vars['mdtyp'] = table_vars['mdtyp'].astype(np.int32)
+            modes_ = list(zip(antenna_codes, pulse_codes, pulse_lengths))
+            modes = np.empty(len(modes_), dtype=object)
+            modes[:] = modes_
+            modes_unique = self.get_modes(modes)
+
+            mode = self.validate_mode(modes_unique)
+
+            inds_mode = np.where((antenna_codes==mode[0]) & (pulse_codes==mode[1]) & (pulse_lengths==mode[2]) )[0]
+
+            self.metadata['antenna'] = antenna_code_dict_r[mode[0]]
+            self.metadata['pulse_code'] = pulse_code_dict_r[mode[1]]
+            self.metadata['pulse_length'] = mode[2]
+
+            # Grid data
+            ut_unix_1 = table_vars['ut1_unix'][inds_mode]
+            ut_unix_1_unique = np.unique(ut_unix_1)
+            num_ut = len(ut_unix_1_unique)
+            vars_fh5 = {}
+            if len(ut_unix_1) % num_ut == 0:
+                num_gate = int(len(ut_unix_1) / num_ut)
+
+                for k, v in table_vars.items():
+                    vars_fh5[k] = v[inds_mode].reshape((num_ut, num_gate))
+            else:
+                gate_inds = []
+                gate_nums = []
+                for t in ut_unix_1_unique:
+                    ii = np.where(ut_unix_1==t)[0]
+                    ran = table_vars['range'][inds_mode][ii]
+                    diff_ran = np.diff(ran)
+                    if any(diff_ran < 0): # duplicated ranges
+                        iii = range(np.where(diff_ran<0)[0][0]+1)
+                        ii = ii[iii]
+                    gate_inds.append(ii)
+
+                    gate_nums.append(len(gate_inds[-1]))
+                max_gate_num = np.max(gate_nums)
+
+                for k, v in table_vars.items():
+                    vars_fh5[k] = np.empty((num_ut, max_gate_num))
+                    vars_fh5[k][::] = np.nan
+                    for i, inds in enumerate(gate_inds):
+                        vars_fh5[k][i, 0:len(inds)] = v[inds_mode][inds]
+            
+            # Assign data
+            records = fh5['Metadata']['_record_layout'][0]
+            rec_var_names = np.array(list(records.dtype.fields.keys()))
+            rec_vars = {str(rec_var_names[i]): int(records[i]) for i in range(len(rec_var_names))}
+            for var_name, var_name_fh5 in var_name_dict.items():
+                if var_name_fh5 not in vars_fh5.keys():
+                    mylog.StreamLogger.warning(f"The requested variable {var_name_fh5} does not exist in the data file!")
+                    variables[var_name] = None
+                    continue
+                if rec_vars[var_name_fh5] == 1:
+                    variables[var_name] = vars_fh5[var_name_fh5][:, 0][:, np.newaxis]
+                else:
+                    variables[var_name] = vars_fh5[var_name_fh5]
+
+            variables['comp_O_p'] = 1. - variables['comp_mix'] - variables['comp_H_p']
+            variables['comp_O_p_err'] = np.sqrt(variables['comp_mix_err']**2 + variables['comp_H_p_err']**2)
+
+            # need to be check when AZ close to 0.
+            variables['AZ1'] = variables['AZ1'] % 360.
+            variables['AZ2'] = variables['AZ2'] % 360.
+            variables['AZ'] = (variables['AZ1'] + variables['AZ2']) / 2
+            diff_az = np.abs(variables['AZ1'] - variables['AZ2'])
+            variables['AZ'] = np.where(diff_az<180, variables['AZ'], ((variables['AZ1'] + variables['AZ2'] + 360) / 2) % 360)
+            variables['EL'] = (variables['EL1'] + variables['EL2']) / 2 
+            
+            variables['DATETIME_1'] = dttool.convert_unix_time_to_datetime_cftime(vars_fh5['ut1_unix'][:, 0])[:, np.newaxis]
+            variables['DATETIME_2'] = dttool.convert_unix_time_to_datetime_cftime(vars_fh5['ut2_unix'][:, 0])[:, np.newaxis] 
+            variables['DATETIME'] = variables['DATETIME_1'] + (variables['DATETIME_2'] - variables['DATETIME_1']) / 2 
+            variables['T_e'] = variables['T_i'] * variables['T_r']
+            variables['T_e_err'] = variables['T_e'] * np.sqrt((variables['T_i_err'] / variables['T_i']) ** 2
+                                                              + (variables['T_r_err'] / variables['T_r']) ** 2)
+        self.variables = variables
+         
+        return
+
+    def validate_mode(self, modes_unique):
+        mode = None
+
+        # Check antenna
+        modes_antenna_matched = []
+        try:
+            for m in modes_unique:
+                if m[0] == antenna_code_dict[self.antenna]:
+                    modes_antenna_matched.append(m)
+            if len(modes_antenna_matched) == 0:
+                raise KeyError
+            elif len(modes_antenna_matched) == 1:
+                if str(self.pulse_code):
+                    if pulse_code_dict[self.pulse_code] != modes_antenna_matched[0][1]:
+                        raise ValueError
+                if self.pulse_length > 0:
+                    if self.pulse_length != modes_antenna_matched[0][2]:
+                        raise ValueError
+                return modes_antenna_matched[0]
+            else:
+                pass
+        except Exception as e:
+            if str(self.antenna):
+                mylog.StreamLogger.error("Antenna {} is not found!".format(self.antenna.upper()))
+            else:
+                mylog.StreamLogger.error("Antenna must be specified!")
+            self.list_modes()
+            return None
+
+        modes_pulse_code_matched = []
+        try:
+            for m in modes_antenna_matched:
+                if m[1] == pulse_code_dict[self.pulse_code]:
+                    modes_pulse_code_matched.append(m)
+            if len(modes_pulse_code_matched) == 0:
+                raise KeyError
+            elif len(modes_pulse_code_matched) == 1:
+                if self.pulse_length > 0:
+                    if self.pulse_length != modes_antenna_matched[0][2]:
+                        raise ValueError
+                return modes_pulse_code_matched[0]
+            else:
+                pass
+        except Exception as e:
+            if str(self.pulse_code):
+                mylog.StreamLogger.error("Pulse code {} is not found!".format(self.pulse_code.upper()))
+            else:
+                mylog.StreamLogger.error("Pulse code must be specified!")
+            self.list_modes()
+            return None
+
+        modes_pulse_length_matched = []
+        try:
+            for m in modes_pulse_code_matched:
+                if m[2] == self.pulse_length:
+                    modes_pulse_length_matched.append(m)
+            if len(modes_pulse_length_matched) == 0:
+                raise ValueError
+            elif len(modes_pulse_length_matched) == 1:
+                return modes_pulse_length_matched[0]
+            else:
+                mylog.StreamLogger.error("Multiple modes found!")
+                raise ValueError
+        except Exception as e:
+            if self.pulse_length > 0:
+                mylog.StreamLogger.error("Pulse length {} is not found!".format(self.pulse_code.upper()))
+            else:
+                mylog.StreamLogger.error("Pulse length must be specified!")
+            self.list_modes()
+            return None
+        return mode
+
+    def list_modes(self):
+        mylog.simpleinfo.info("List of the experiment modes:")
+
+        for i, m in enumerate(self.metadata['MODES']):
+            s = '{}, '.format(i)
+            for k, v in m.items():
+                s = s + "{}: {}, ".format(k, v)
+            mylog.simpleinfo.info(s)
+        return
+    def get_modes(self, modes):
+
+        modes_unique = np.unique(modes)
+
+        self.metadata['MODES'] = []
+        num_modes = self.metadata['NUM_MODES'] = len(modes_unique)
+        for i in range(num_modes):
+            self.metadata['MODES'].append({
+                'ANTENNA_ID': modes_unique[i][0],
+                'ANTENNA': antenna_code_dict_r[modes_unique[i][0]],
+                'PULSE_CODE_ID': modes_unique[i][1],
+                'PULSE_CODE': pulse_code_dict_r[modes_unique[i][1]],
+                'PULSE_LENGTH': modes_unique[i][2]
+            })
+        return modes_unique
 
     def load(self):
         variables = {}
@@ -182,8 +378,12 @@ class Loader:
             variables['comp_O_p_err'] = np.sqrt(variables['comp_mix_err']**2 + variables['comp_H_p_err']**2)
 
             # need to be check when AZ close to 0.
-            variables['AZ'] = variables['AZ1'] % 360.
-            variables['EL'] = variables['EL1']
+            variables['AZ1'] = variables['AZ1'] % 360.
+            variables['AZ2'] = variables['AZ2'] % 360.
+            variables['AZ'] = (variables['AZ1'] + variables['AZ2']) / 2
+            diff_az = np.abs(variables['AZ1'] - variables['AZ2'])
+            variables['AZ'] = np.where(diff_az<180, variables['AZ'], ((variables['AZ1'] + variables['AZ2'] + 360) / 2) % 360)
+            variables['EL'] = (variables['EL1'] + variables['EL2']) / 2 
 
             variables['RANGE'] = np.tile(vars_fh5['range'], [variables['n_e'].shape[0], 1])
             variables['DATETIME'] = dttool.convert_unix_time_to_datetime_cftime(vars_fh5['timestamps'])
@@ -194,8 +394,3 @@ class Loader:
         self.variables = variables
         self.metadata = metadata
 
-
-if __name__ == "__main__":
-    import pathlib
-    fp = pathlib.Path("/home/lei/afys-data/Madrigal/Millstone_ISR/2016/20160314/Millstone_ISR_combined_20160314.005.hdf5")
-    Loader(file_path=fp)

geospacelab/datahub/sources/madrigal/isr/millstonehill/basic/variable_config.py

@@ -87,7 +87,7 @@ axis[1].data = "@d.HEIGHT.value"
 axis[2].data = "@v.value"
 axis[1].label = 'h'
 axis[1].unit = 'km'
-axis[2].lim = [8e9, 9e11]
+axis[2].lim = [8e9, 3e12]
 axis[2].scale = 'log'
 axis[2].label = '@v.label'
 axis[2].unit = '@v.unit_label'

geospacelab/datahub/sources/madrigal/isr/pfisr/fitted/__init__.py

@@ -232,9 +232,8 @@ class Dataset(datahub.DatasetSourced):
                 search_pattern = "*EID-*/"
                 exp_dirs = list(initial_file_dir.glob(search_pattern))
 
-                if not list(exp_dirs):
-                    done = False
-                    continue
+                if not list(exp_dirs) and self.allow_download:
+                    self.download_data()
 
                 def dir_parser(dirs):
                     dirs_out = []
@@ -252,9 +251,8 @@ class Dataset(datahub.DatasetSourced):
                     return dirs_out
                 file_dirs = dir_parser(exp_dirs)
 
-                if not list(file_dirs):
-                    done=False
-                    continue
+                if not list(file_dirs) and self.allow_download:
+                    self.download_data() 
 
                 for fd in file_dirs:
                     if isinstance(self.exp_name_pattern, list):
@@ -281,7 +279,7 @@ class Dataset(datahub.DatasetSourced):
                         initial_file_dir=fd, search_pattern=search_pattern,
                         allow_multiple_files=True, recursive=recursive)
 
-                # Validate file paths
+                    # Validate file paths
 
                     if not done and self.allow_download:
                         done = self.download_data()

geospacelab/datahub/sources/madrigal/isr/pfisr/fitted/loader.py

@@ -141,6 +141,12 @@ class Loader:
         
         return metadata
 
+    def load_from_table_layout(self):
+        variables = {}
+        metadata = {}
+
+
+
     def load(self):
         variables = {}
         metadata = {}
@@ -217,9 +223,3 @@ class Loader:
             self.beam_el = np.nanmedian(variables['EL'])
             
             self.metadata = metadata
-
-
-if __name__ == "__main__":
-    import pathlib
-    fp = pathlib.Path("/home/lei/Downloads/pfa140102.002.hdf5")
-    Loader(file_path=fp, beam_az=290.5, beam_el=70.)

geospacelab/datahub/sources/madrigal/isr/pfisr/vi/__init__.py

@@ -159,9 +159,8 @@ class Dataset(datahub.DatasetSourced):
                 search_pattern = "*EID-*/"
                 exp_dirs = list(initial_file_dir.glob(search_pattern))
 
-                if not list(exp_dirs):
-                    done = False
-                    continue
+                if not list(exp_dirs) and self.allow_download:
+                    self.download_data() 
 
                 def dir_parser(dirs):
                     dirs_out = []
@@ -180,9 +179,8 @@ class Dataset(datahub.DatasetSourced):
 
                 file_dirs = dir_parser(exp_dirs)
 
-                if not list(file_dirs):
-                    done = False
-                    continue
+                if not list(file_dirs) and self.allow_download:
+                    self.download_data() 
 
                 for fd in file_dirs:
                     if isinstance(self.exp_name_pattern, list):

geospacelab/datahub/sources/madrigal/isr/risr_n/downloader.py

@@ -0,0 +1,155 @@
+# Licensed under the BSD 3-Clause License
+# Copyright (C) 2021 GeospaceLab (geospacelab)
+# Author: Lei Cai, Space Physics and Astronomy, University of Oulu
+
+__author__ = "Lei Cai"
+__copyright__ = "Copyright 2021, GeospaceLab"
+__license__ = "BSD-3-Clause License"
+__email__ = "lei.cai@oulu.fi"
+__docformat__ = "reStructureText"
+
+from cProfile import label
+import datetime
+from dateutil.relativedelta import relativedelta
+from dateutil.parser import parse as dtparse
+import numpy as np
+import re
+import requests
+import bs4
+import os
+import pathlib
+
+from geospacelab.config import prf
+import geospacelab.datahub.sources.madrigal as madrigal
+import geospacelab.toolbox.utilities.pylogging as mylog
+import geospacelab.toolbox.utilities.pydatetime as dttool
+from geospacelab.datahub.sources.madrigal.downloader import Downloader as DownloaderBase
+
+
+def test():
+    dt_fr = datetime.datetime(2016, 3, 14)
+    dt_to = datetime.datetime(2016, 3, 14, 23)
+    eid = 100213152
+    download_obj = Downloader(
+        dt_fr, dt_to, include_exp_ids=[100213152, 100213190],
+        data_product='vi',
+        include_file_name_patterns= [['ran']],
+        include_file_type_patterns=[['velocity']],
+        exclude_file_type_patterns=['.*uncorrected.*', '.*power.*'],
+        force_download=True,
+        dry_run=False)
+
+
+class Downloader(DownloaderBase):
+
+    def __init__(
+            self, dt_fr: datetime.datetime, dt_to: datetime,
+            data_product=None,
+            include_exp_name_patterns: list=None,
+            exclude_exp_name_patterns: list=None,
+            include_exp_ids: list = None,
+            exclude_exp_ids: list=[],
+            include_file_name_patterns: list = None,
+            exclude_file_name_patterns: list = None,
+            include_file_type_patterns=None,
+            exclude_file_type_patterns=None,
+            data_file_root_dir: str = None,
+            direct_download = True,
+            force_download = False,
+            dry_run: bool=False,
+            madrigal_url: str = "https://data.amisr.com/madrigal",
+            user_fullname: str=madrigal.default_user_fullname,
+            user_email: str=madrigal.default_user_email,
+            user_affiliation: str=madrigal.default_user_affiliation):
+
+        icodes = [91,]
+        self.data_product = data_product if isinstance(data_product, str) else ''
+        
+        dt_fr = dttool.get_start_of_the_day(dt_fr)
+        dt_to = dttool.get_end_of_the_day(dt_to)
+
+        super().__init__(
+            dt_fr=dt_fr, dt_to=dt_to, icodes=icodes,
+            include_exp_name_patterns=include_exp_name_patterns, 
+            exclude_exp_name_patterns=exclude_exp_name_patterns, 
+            include_exp_ids=include_exp_ids,
+            exclude_exp_ids=exclude_exp_ids,
+            include_file_name_patterns=include_file_name_patterns, 
+            exclude_file_name_patterns=exclude_file_name_patterns,
+            include_file_type_patterns=include_file_type_patterns, 
+            exclude_file_type_patterns=exclude_file_type_patterns,  
+            data_file_root_dir=data_file_root_dir,
+            force_download=force_download, direct_download=direct_download, dry_run=dry_run,
+            madrigal_url=madrigal_url,
+            user_fullname=user_fullname, user_email=user_email, user_affiliation=user_affiliation)
+
+    def download(self, **kwargs):
+
+        exps, database = self.get_exp_list(
+            dt_fr=self.dt_fr,
+            dt_to=self.dt_to,
+            include_exp_name_patterns=self.include_exp_name_patterns,
+            exclude_exp_name_patterns=self.exclude_exp_name_patterns,
+            include_exp_ids=self.include_exp_ids,
+            exclude_exp_ids=self.exclude_exp_ids,
+            icodes=self.icodes,
+            madrigal_url=self.madrigal_url,
+            display=True)
+        self.exp_list = list(exps)
+        self.database = database
+
+        exps, exps_error = self.get_online_file_list(
+            exp_list=self.exp_list, database=database,
+            include_file_name_patterns=self.include_file_name_patterns,
+            exclude_file_name_patterns=self.exclude_file_name_patterns,
+            include_file_type_patterns=self.include_file_type_patterns,
+            exclude_file_type_patterns=self.exclude_file_type_patterns,
+            display=True
+        )
+        self.exp_list_error = list(exps_error)
+
+        file_paths = [] 
+        for exp in exps:
+            dt_fr_exp = datetime.datetime(
+                exp.startyear, exp.startmonth, exp.startday, exp.starthour, exp.startmin, exp.startsec
+            )
+            dt_to_exp = datetime.datetime(
+                exp.endyear, exp.endmonth, exp.endday, exp.endhour, exp.endmin, exp.endsec
+            )
+            for file in list(exp.files):
+
+                file_path_remote = pathlib.Path(file.name)
+                file_name_remote = file_path_remote.name
+
+                res = re.search(r'/([\d]{2}[a-z]{3}[\d]{2})', file.doi)
+                dtstr = res.groups()[0]
+                dtstr = dtstr[0:2] + dtstr[2].upper() + dtstr[3:]
+                thisday = datetime.datetime.strptime(dtstr, "%d%b%y")
+
+                exp_name_patterns = re.findall('[\w.]+', exp.name)
+                if len(exp_name_patterns) > 5:
+                    exp_name_patterns = exp_name_patterns[:5]
+                    exp_name_patterns[0] = exp_name_patterns[0].replace('.risrn', '') 
+                exp_name = '_'.join(exp_name_patterns).lower()
+                file_dir_local = self.data_file_root_dir / thisday.strftime("%Y") / \
+                                ('RISR-N_EID-' + str(exp.id) + '_'
+                                 + dt_fr_exp.strftime("%Y%m%d%H%M%S") + '_'
+                                 + dt_to_exp.strftime("%Y%m%d%H%M%S") + '_' + exp_name)
+                file_dir_local.mkdir(parents=True, exist_ok=True)
+                file_name_local = 'RISR-N_' + thisday.strftime('%Y%m%d') + '_' + \
+                                  self.data_product.replace(' ', '_') + \
+                                  '.' + '.'.join(file_name_remote.split('.')[1:])
+                file_path_local = file_dir_local / file_name_local
+                if file_path_local.is_file():
+                    mylog.simpleinfo.info("The file has been downloaded: {}.".format(file_path_local))
+                    self.done = True
+                    continue
+                super().download(
+                    file_path_remote=file.name, file_path_local=file_path_local,
+                    file_format='hdf5')
+                file_paths.append(file_path_local)
+        return file_paths
+
+
+if __name__ == "__main__":
+    test()