geospacelab 0.9.4__py3-none-any.whl → 0.10.1__py3-none-any.whl

geospacelab/__init__.py

@@ -6,7 +6,7 @@ __author__ = "Lei Cai"
 __copyright__ = "Copyright 2021, GeospaceLAB"
 __credits__ = ["Lei Cai"]
 __license__ = "BSD-3-Clause License"
-__version__ = "0.9.4"
+__version__ = "0.10.1"
 __maintainer__ = "Lei Cai"
 __email__ = "lei.cai@oulu.fi"
 __status__ = "Developing"

geospacelab/datahub/sources/cdaweb/downloader.py

@@ -43,6 +43,7 @@ class Downloader(object):
         self.dt_fr = dt_fr
         self.dt_to = dt_to
         self.source_file_paths = []
+        self.done = False
 
         if data_file_root_dir is None:
             self.data_file_root_dir = prf.datahub_data_root_dir

geospacelab/datahub/sources/madrigal/downloader.py

@@ -116,6 +116,7 @@ class Downloader(object):
         if file_path_local.is_file():
             mylog.simpleinfo.info("The file {} has been downloaded.".format(file_path_local.name))
             if not self.force_download:
+                self.done = True
                 return
         files_error = []
         mylog.simpleinfo.info("Downloading {} ...".format(file_path_remote))
@@ -192,8 +193,12 @@ class Downloader(object):
                 for file in files:
                     matching = 0
                     for fnp in include_file_name_patterns:
+                        if isinstance(fnp, str):
+                            fnp = [fnp]
                         if isinstance(fnp, list):
-                            fnp = '.*' + '.*'.join(fnp) + '.*'
+                            fnp = r'.*' + '.*'.join(fnp) + '.*'
+                        else:
+                            raise AttributeError
                         rc = re.compile(fnp)
                         file_name = pathlib.Path(file.name).name
                         rm = rc.match(file_name.lower())
@@ -212,8 +217,12 @@ class Downloader(object):
                 for file in files:
                     matching = 1
                     for fnp in exclude_file_name_patterns:
+                        if isinstance(fnp, str):
+                            fnp = [fnp]
                         if isinstance(fnp, list):
-                            fnp = '.*' + '.*'.join(fnp) + '.*'
+                            fnp = r'.*' + '.*'.join(fnp) + '.*'
+                        else:
+                            raise AttributeError
                         rc = re.compile(fnp)
                         file_name = pathlib.Path(file.name).name
                         rm = rc.match(file_name.lower())
@@ -232,8 +241,12 @@ class Downloader(object):
                 for file in files:
                     matching = 0
                     for fnp in include_file_type_patterns:
+                        if isinstance(fnp, str):
+                            fnp = [fnp]
                         if isinstance(fnp, list):
-                            fnp = '.*' + '.*'.join(fnp) + '.*'
+                            fnp = r'.*' + '.*'.join(fnp) + '.*'
+                        else:
+                            raise AttributeError
                         rc = re.compile(fnp)
                         rm = rc.match(file.kindatdesc.lower())
                         if rm is not None:
@@ -251,8 +264,12 @@ class Downloader(object):
                 for file in files:
                     matching = 1
                     for fnp in exclude_file_type_patterns:
+                        if isinstance(fnp, str):
+                            fnp = [fnp]
                         if isinstance(fnp, list):
-                            fnp = '.*' + '.*'.join(fnp) + '.*'
+                            fnp = r'.*' + '.*'.join(fnp) + '.*'
+                        else:
+                            raise AttributeError
                         rc = re.compile(fnp)
                         rm = rc.match(file.kindatdesc.lower())
                         if rm is not None:
@@ -281,7 +298,7 @@ class Downloader(object):
         if display:
             mylog.simpleinfo.info("Listing matched experiments and files ...")
             exp_info = Downloader.get_exp_info(exps, include_file_info=True)
-            mylog.simpleinfo.info("{:>10s}\t{:<24s}\t{:<24s}\t{:<16s}\t{:<15s}\t{:<40.40s}\t{:<30.30s}\t{:<80.80s}".format(
+            mylog.simpleinfo.info("{:>10s}\t{:<24s}\t{:<24s}\t{:<16s}\t{:<15s}\t{:<40.40s}\t{:<50.50s}\t{:<80.80s}".format(
                 'EXP NUM', 'START TIME', 'END TIME', 'DURATION (hour)', 'EXP ID', 'EXP Name', 'File Name', 'File Type'
             )
             )
@@ -293,7 +310,7 @@ class Downloader(object):
             ):
                 
                 for file in exp_info['FILES'][ind]:
-                    line_str = "{:>10d}\t{:<24s}\t{:<24s}\t{:<16.1f}\t{:<15d}\t{:<40.40s}\t{:<30.30s}\t{:<80.80s}".format(
+                    line_str = "{:>10d}\t{:<24s}\t{:<24s}\t{:<16.1f}\t{:<15d}\t{:<40.40s}\t{:<50.50s}\t{:<80.80s}".format(
                         ind + 1,
                         dt_fr.strftime("%Y-%m-%d %H:%M:%S"),
                         dt_to.strftime("%Y-%m-%d %H:%M:%S"),
@@ -437,8 +454,12 @@ class Downloader(object):
                 for exp in exps:
                     matching = 0
                     for enp in include_exp_name_patterns:
+                        if isinstance(enp, str):
+                            enp = [enp]
                         if isinstance(enp, list):
-                            enp = '.*' + '.*'.join(enp) + '.*'
+                            enp = r'.*' + '.*'.join(enp) + '.*'
+                        else:
+                            raise AttributeError
                         rc = re.compile(enp)
                         rm = rc.match(exp.name.lower())
                         if rm is not None:
@@ -456,8 +477,12 @@ class Downloader(object):
                 for exp in exps:
                     matching = 1
                     for enp in exclude_exp_name_patterns:
+                        if isinstance(enp, str):
+                            enp = [enp]
                         if isinstance(enp, list):
-                            enp = '.*' + '.*'.join(enp) + '.*'
+                            enp = r'.*' + '.*'.join(enp) + '.*'
+                        else:
+                            raise AttributeError
                         rc = re.compile(enp)
                         rm = rc.match(exp.name.lower())
                         if rm is not None:

geospacelab/datahub/sources/madrigal/isr/eiscat/__init__.py

@@ -10,6 +10,8 @@ __docformat__ = "reStructureText"
 
 
 import datetime
+import re
+
 import numpy as np
 
 import geospacelab.datahub as datahub
@@ -27,9 +29,9 @@ default_dataset_attrs = {
     'kind': 'sourced',
     'database': madrigal_database,
     'facility': 'EISCAT',
-    'data_file_type': 'eiscat-hdf5',
+    'data_file_type': 'madrigal-hdf5',
     'data_file_ext': 'hdf5',
-    'data_root_dir': prf.datahub_data_root_dir / 'Madrigal' / 'EISCAT' / 'analyzed',
+    'data_root_dir': prf.datahub_data_root_dir / 'Madrigal' / 'EISCAT' / 'Analyzed',
     'allow_download': True,
     'status_control': False,
     'rasidual_contorl': False,
@@ -49,7 +51,7 @@ default_variable_names = [
 
 # default_data_search_recursive = True
 
-default_attrs_required = ['site', 'antenna', 'modulation']
+default_attrs_required = ['site', 'antenna',]
 
 
 class Dataset(datahub.DatasetSourced):
@@ -63,13 +65,15 @@ class Dataset(datahub.DatasetSourced):
         self.site = kwargs.pop('site', '')
         self.antenna = kwargs.pop('antenna', '')
         self.experiment = kwargs.pop('experiment', '')
+        self.experiment_ids = kwargs.pop('exp_ids', [])
         self.pulse_code = kwargs.pop('pulse_code', '')
         self.scan_mode = kwargs.pop('scan_mode', '')
         self.modulation = kwargs.pop('modulation', '')
         self.data_file_type = kwargs.pop('data_file_type', '')
         self.affiliation = kwargs.pop('affiliation', '')
         self.allow_download = kwargs.pop('allow_download', True)
-        self.metadata = None
+        self.gate_num = kwargs.pop('gate_num', None)
+        self.metadata = {}
         self.add_AACGM = kwargs.pop('add_AACGM', True)
         self.add_APEX = kwargs.pop('add_APEX', False)
 
@@ -102,6 +106,8 @@ class Dataset(datahub.DatasetSourced):
 
         if str(self.data_file_type):
             self.data_file_ext = self.data_file_type.split('-')[1]
+            if (self.load_mode == 'AUTO') and (self.data_file_type=='eiscat-mat'):
+                raise AttributeError
 
     def label(self, **kwargs):
         label = super().label()
@@ -111,7 +117,9 @@ class Dataset(datahub.DatasetSourced):
         self.check_data_files(**kwargs)
 
         for file_path in self.data_file_paths:
-            load_obj = self.loader(file_path, file_type=self.data_file_type)
+            load_obj = self.loader(file_path, file_type=self.data_file_type, gate_num=self.gate_num)
+            if self.gate_num is None:
+                self.gate_num = load_obj.gate_num
 
             for var_name in self._variables.keys():
                 self._variables[var_name].join(load_obj.variables[var_name])
@@ -125,6 +133,12 @@ class Dataset(datahub.DatasetSourced):
             self.experiment = rawdata_path.split('/')[-1].split('@')[0]
             self.affiliation = load_obj.metadata['affiliation']
             self.metadata = load_obj.metadata
+
+        inds_cmb = np.argsort(self['DATETIME'].flatten())
+        if any(np.diff(np.array(inds_cmb))<0):
+            for var_name in self.keys():
+                self[var_name].value = self[var_name].value[inds_cmb, :]
+
         if self.add_AACGM or self.add_APEX:
             self.calc_lat_lon()
             # self.select_beams(field_aligned=True)
@@ -253,57 +267,142 @@ class Dataset(datahub.DatasetSourced):
     def search_data_files(self, **kwargs):
         dt_fr = self.dt_fr
         dt_to = self.dt_to
-        diff_days = dttool.get_diff_days(dt_fr, dt_to)
-        day0 = dttool.get_start_of_the_day(dt_fr)
-        for i in range(diff_days + 1):
-            thisday = day0 + datetime.timedelta(days=i)
-            initial_file_dir = self.data_root_dir / self.site / thisday.strftime('%Y')
-
-            file_patterns = []
-            if self.data_file_type == 'eiscat-hdf5':
-                file_patterns.append('EISCAT')
-            elif self.data_file_type == 'madrigal-hdf5':
-                file_patterns.append('MAD6400')
-            elif self.data_file_type == 'eiscat-mat':
-                pass
-            file_patterns.append(thisday.strftime('%Y-%m-%d'))
-            file_patterns.append(self.modulation)
-            file_patterns.append(self.antenna.lower())
-
-            # remove empty str
-            file_patterns = [pattern for pattern in file_patterns if str(pattern)]
-
-            search_pattern = '*' + '*'.join(file_patterns) + '*'
-            if self.data_file_type == 'eiscat-mat':
-                search_pattern = search_pattern + '/'
-            done = super().search_data_files(
-                initial_file_dir=initial_file_dir, search_pattern=search_pattern)
-
-            # Validate file paths
-
-            if not done and self.allow_download:
-                done = self.download_data()
-                if done:
-                    done = super().search_data_files(
-                        initial_file_dir=initial_file_dir, search_pattern=search_pattern)
+        done = False
+        if not list(self.experiment_ids):
+            diff_days = dttool.get_diff_days(dt_fr, dt_to)
+            day0 = dttool.get_start_of_the_day(dt_fr)
+            for i in range(diff_days + 1):
+                thisday = day0 + datetime.timedelta(days=i)
+                initial_file_dir = self.data_root_dir / self.site / thisday.strftime('%Y')
+
+                file_patterns = []
+                if self.data_file_type == 'eiscat-hdf5':
+                    file_patterns.append('EISCAT')
+                elif self.data_file_type == 'madrigal-hdf5':
+                    file_patterns.append('MAD6400')
+                else:
+                    raise NotImplementedError
+                file_patterns.append(thisday.strftime('%Y-%m-%d'))
+                if str(self.pulse_code):
+                    file_patterns.append(self.pulse_code)
+                if str(self.modulation):
+                    file_patterns.append(self.modulation)
+                file_patterns.append(self.antenna.lower())
+
+                # remove empty str
+                file_patterns = [pattern for pattern in file_patterns if str(pattern)]
+
+                search_pattern = '*' + '*'.join(file_patterns) + '*'
+                done = super().search_data_files(
+                    initial_file_dir=initial_file_dir,
+                    search_pattern=search_pattern, allow_multiple_files=True)
+
+                # Validate file paths
+                if not done and self.allow_download:
+                    done = self.download_data()
+                    if done:
+                        done = super().search_data_files(
+                            initial_file_dir=initial_file_dir,
+                            search_pattern=search_pattern,
+                            allow_multiple_files=True
+                        )
+                    else:
+                        print('Cannot find files from the online database!')
+        else:
+            initial_file_dir = self.data_root_dir
+            for exp_id in self.experiment_ids:
+                file_patterns = []
+                if self.data_file_type == 'eiscat-hdf5':
+                    file_patterns.append('EISCAT')
+                elif self.data_file_type == 'madrigal-hdf5':
+                    file_patterns.append('MAD')
                 else:
-                    print('Cannot find files from the online database!')
+                    raise NotImplementedError
+                file_patterns.append(thisday.strftime('%Y-%m-%d'))
+                if str(self.pulse_code):
+                    file_patterns.append(self.pulse_code)
+                if str(self.modulation):
+                    file_patterns.append(self.modulation)
+                file_patterns.append(self.antenna.lower())
+                # remove empty str
+                file_patterns = [pattern for pattern in file_patterns if str(pattern)]
+                search_pattern = f"*EID-{exp_id}*/*{'*'.join(file_patterns)}*"
+                done = super().search_data_files(
+                    initial_file_dir=initial_file_dir,
+                    search_pattern=search_pattern, recursive=True, allow_multiple_files=True
+                )
+
+                if not done and self.allow_download:
+                    done = self.download_data()
+                    if done:
+                        done = super().search_data_files(
+                            initial_file_dir=initial_file_dir,
+                            search_pattern=search_pattern, recursive=True, allow_multiple_files=True
+                        )
+                    else:
+                        print('The requested experiment (ID: {}) does not exist in the online database!'.format(exp_id))
+                if len(done) > 1:
+                    mylog.StreamLogger.warning(
+                        "Multiple data files detected! " +
+                        "Specify the experiment pulse code and modulation may constrain the searching condition.")
+                    for fp in done:
+                        mylog.simpleinfo.info(str(fp))
+        self._check_multiple_files()
 
         return done
 
+    def _check_multiple_files(self):
+        file_paths = self.data_file_paths
+        exp_ids = []
+        for fp in file_paths:
+            rc = re.compile(r"EID\-([\d]+)")
+            res = rc.search(str(fp))
+            exp_ids.append(res.groups()[0])
+        exp_ids_unique = [eid for eid in np.unique(exp_ids)]
+
+        file_paths_new = []
+        for eid in exp_ids_unique:
+            inds_id = np.where(np.array(exp_ids)==eid)[0]
+            fps_sub = []
+            for ii in inds_id:
+                fp = file_paths[ii]
+                rc = re.compile(r".*_([\d]{8}T[\d]{6}).*_([\d]{8}T[\d]{6}).*[\d]{4}\-[\d]{2}\-[\d]{2}_([\w.]+)@.*")
+                res = rc.search(str(fp))
+                dt_0 = datetime.datetime.strptime(res.groups()[0], '%Y%m%dT%H%M%S')
+                dt_1 = datetime.datetime.strptime(res.groups()[1], '%Y%m%dT%H%M%S')
+                if (dt_0 >= self.dt_to) or (dt_1<=self.dt_fr):
+                    continue
+                if str(self.pulse_code):
+                    if self.pulse_code not in res.groups()[2].lower():
+                        continue
+                if str(self.modulation):
+                    if self.modulation not in res.groups()[2].lower():
+                        continue
+                if '_v' in res.groups()[2].lower():
+                    continue
+                fps_sub.extend([fp])
+            if len(fps_sub) > 1:
+                mylog.StreamLogger.warning("Multiple data files for a single experiment detected!")
+                # for fp in fps_sub:
+                #     mylog.simpleinfo.info(str(fp))
+                # fps_sub = fps_sub[0]
+            file_paths_new.extend(fps_sub)
+        self.data_file_paths = file_paths_new
+
+
     def download_data(self):
         if self.data_file_type == 'eiscat-hdf5':
             download_obj = self.downloader(dt_fr=self.dt_fr, dt_to=self.dt_to,
-                                           sites=[self.site], kind_data='eiscat',
-                                           data_file_root_dir=self.data_root_dir)
+                                           antennas=[self.antenna], kind_data='eiscat',
+                                           data_file_root_dir=self.data_root_dir,
+                                           exclude_file_type_patterns=['pp']
+            )
         elif self.data_file_type == 'madrigal-hdf5':
             download_obj = self.downloader(dt_fr=self.dt_fr, dt_to=self.dt_to,
-                                           sites=[self.site], kind_data='madrigal',
-                                           data_file_root_dir=self.data_root_dir)
-        elif self.data_file_type == 'eiscat-mat':
-            download_obj = self.downloader(dt_fr=self.dt_fr, dt_to=self.dt_to,
-                                                 sites=[self.site], kind_data='eiscat',
-                                           data_file_root_dir=self.data_root_dir)
+                                           antennas=[self.antenna], kind_data='madrigal',
+                                           data_file_root_dir=self.data_root_dir,
+                                           exclude_file_type_patterns=['pp']
+                                           )
         else:
             raise TypeError
         return download_obj.done

geospacelab/datahub/sources/madrigal/isr/eiscat/downloader.py

@@ -28,17 +28,242 @@ import geospacelab.datahub.sources.madrigal as madrigal
 import geospacelab.toolbox.utilities.pylogging as mylog
 import geospacelab.toolbox.utilities.pydatetime as dttool
 
+from geospacelab.datahub.sources.madrigal.downloader import Downloader as DownloaderBase
+
+
+class Downloader(DownloaderBase):
+
+    def __init__(
+            self, 
+            dt_fr: datetime.datetime, dt_to: datetime,
+            antennas=None, 
+            data_file_root_dir=None, 
+            kind_data="madrigal",
+            user_fullname=madrigal.default_user_fullname,
+            user_email=madrigal.default_user_email,
+            user_affiliation=madrigal.default_user_affiliation,
+            include_exp_name_patterns: list=None,
+            exclude_exp_name_patterns: list=None,
+            include_exp_ids: list=None,
+            exclude_exp_ids: list=None,
+            include_file_name_patterns: list = None,
+            exclude_file_name_patterns: list = None,
+            include_file_type_patterns=None,
+            exclude_file_type_patterns=None,
+            direct_download = True,
+            force_download = False,
+            dry_run: bool=False,
+            madrigal_url: str = "http://madrigal.eiscat.se/",
+        ):
+
+        # dt_fr = dttool.get_start_of_the_day(dt_fr)
+        # dt_to = dttool.get_end_of_the_day(dt_to)
+        
+        icodes = []
+        for a in antennas:
+            icodes.extend(instrument_codes[a])  
+        icodes = [int(i) for i in np.unique(icodes)]
+        
+        self.kind_data=kind_data
 
-def test():
-    sites = ['UHF']
-    dt_fr = datetime.datetime(2014, 1, 1,)
-    dt_to = datetime.datetime(2014, 12, 31)
-    # download_obj = Downloader(dt_fr, dt_to, sites=sites, kind_data="madrigal")
-    # schedule = EISCATSchedule(dt_fr=dt_fr, dt_to=dt_to)
-    # schedule.to_txt()
+        if data_file_root_dir is None:
+            self.data_file_root_dir = pfr.datahub_data_root_dir / 'Madrigal' / 'EISCAT' / 'Analyzed'
+        else:
+            self.data_file_root_dir = data_file_root_dir
 
+        super().__init__(
+            dt_fr=dt_fr, dt_to=dt_to, icodes=icodes,
+            include_exp_name_patterns=include_exp_name_patterns, 
+            exclude_exp_name_patterns=exclude_exp_name_patterns, 
+            include_exp_ids=include_exp_ids,
+            exclude_exp_ids=exclude_exp_ids,
+            include_file_name_patterns=include_file_name_patterns, 
+            exclude_file_name_patterns=exclude_file_name_patterns,
+            include_file_type_patterns=include_file_type_patterns, 
+            exclude_file_type_patterns=exclude_file_type_patterns,  
+            data_file_root_dir=data_file_root_dir,
+            force_download=force_download, direct_download=direct_download, dry_run=dry_run,
+            madrigal_url=madrigal_url,
+            user_fullname=user_fullname, user_email=user_email, user_affiliation=user_affiliation)
+
+    def download(self, **kwargs):
+        if self.kind_data.lower() == 'eiscat':
+            self.download_EISCAT()
+        elif self.kind_data.lower() == 'madrigal':
+            self.download_MAD()
+        else:
+            raise NotImplementedError    
+        return 
+    
+    def download_EISCAT(self):
+        exps, database = self.get_exp_list(
+            dt_fr=dttool.get_start_of_the_day(self.dt_fr),
+            dt_to=dttool.get_end_of_the_day(self.dt_to),
+            include_exp_name_patterns=self.include_exp_name_patterns,
+            exclude_exp_name_patterns=self.exclude_exp_name_patterns,
+            include_exp_ids=self.include_exp_ids,
+            exclude_exp_ids=self.exclude_exp_ids,
+            icodes=self.icodes,
+            madrigal_url=self.madrigal_url,
+            display=True)
+        self.exp_list = list(exps)
+        self.database = database
+        
+        cookies = {
+                'user_email': self.user_email,
+                'user_fullname': self.user_fullname,
+                'user_affiliation': self.user_affiliation
+            }
+        for exp in exps:
+            dt_fr_exp = datetime.datetime(
+                exp.startyear, exp.startmonth, exp.startday, exp.starthour, exp.startmin, exp.startsec
+            )
+            dt_to_exp = datetime.datetime(
+                exp.endyear, exp.endmonth, exp.endday, exp.endhour, exp.endmin, exp.endsec
+            )
+            if (dt_fr_exp >= self.dt_to) or (dt_to_exp <= self.dt_fr):
+                continue
+            try:
+                res = re.search(r'([\d]{4}\-[\d]{2}\-[\d]{2})_(\w+)@(\w+)', exp.name)
+                thisday = datetime.datetime.strptime(res.groups()[0], "%Y-%m-%d")
+                pulse_code = res.groups()[1]
+                antenna_ = res.groups()[2]
+                if 'uhf' in antenna_.lower():
+                    antenna = 'UHF'
+                elif 'vhf' in antenna_.lower():
+                    antenna = 'VHF'
+                elif any([a in antenna_.lower() for a in ['32m', '42m', 'esr']]):
+                    antenna = 'ESR'
+                elif 'sod' in antenna_.lower():
+                    antenna = 'SOD'
+                elif 'kir' in antenna_.lower():
+                    antenna = 'KIR'
+                else:
+                    raise NotImplementedError
+            except Exception as e:
+                print(e)
+                mylog.StreamLogger.warning("Parsing the experiment name was failed! EXP: {}".format(exp.name))
+            
+            url = "https://madrigal.eiscat.se/madrigal/showExperiment?experiment_list=" + str(exp.id) + "&show_plots="
+            
+            r = requests.get(url, cookies=cookies)
+            soup = bs4.BeautifulSoup(r.text, 'html.parser')
+            links = soup.find_all('a', href=True)
+            for link in links:
+                href = link['href']
+                if any(href.endswith(s) for s in ['.png', '.tar.gz', '.hdf5']):
+                    filename = href.split('/')[-1]
+                    file_dir_local = self.data_file_root_dir / antenna / thisday.strftime("%Y") / \
+                                (exp.name + '_EID-' + str(exp.id) + '_'
+                                 + dt_fr_exp.strftime("%Y%m%dT%H%M%S") + '_'
+                                 + dt_to_exp.strftime("%Y%m%dT%H%M%S"))
+                    file_dir_local.mkdir(parents=True, exist_ok=True)
+
+                    remote_file = requests.get(href)
+                    file_path = file_dir_local / filename
+                    if file_path.is_file() and not self.force_download:
+                        print("The file {} has been downloaded.".format(filename))
+                        self.done=True
+                        continue
+                    mylog.simpleinfo.info(
+                        'Downloading "{} ..."'.format(filename)
+                    )
+                    with open(file_path, "wb") as eiscat:
+                        eiscat.write(remote_file.content)
+                    mylog.simpleinfo.info('Saved to {}.'.format(file_dir_local))
+                    self.done = True
+        return 
+    
+    def download_MAD(self, **kwargs):
+
+        exps, database = self.get_exp_list(
+            dt_fr=dttool.get_start_of_the_day(self.dt_fr),
+            dt_to=dttool.get_end_of_the_day(self.dt_to),
+            include_exp_name_patterns=self.include_exp_name_patterns,
+            exclude_exp_name_patterns=self.exclude_exp_name_patterns,
+            include_exp_ids=self.include_exp_ids,
+            exclude_exp_ids=self.exclude_exp_ids,
+            icodes=self.icodes,
+            madrigal_url=self.madrigal_url,
+            display=True)
+        self.exp_list = list(exps)
+        self.database = database
+
+        exps, exps_error = self.get_online_file_list(
+            exp_list=self.exp_list, database=database,
+            include_file_name_patterns=self.include_file_name_patterns,
+            exclude_file_name_patterns=self.exclude_file_name_patterns,
+            include_file_type_patterns=self.include_file_type_patterns,
+            exclude_file_type_patterns=self.exclude_file_type_patterns,
+            display=True
+        )
+        self.exp_list_error = list(exps_error)
+
+        file_paths = [] 
+        for exp in exps:
+            dt_fr_exp = datetime.datetime(
+                exp.startyear, exp.startmonth, exp.startday, exp.starthour, exp.startmin, exp.startsec
+            )
+            dt_to_exp = datetime.datetime(
+                exp.endyear, exp.endmonth, exp.endday, exp.endhour, exp.endmin, exp.endsec
+            )
+            # if (dt_fr_exp >= self.dt_to) or (dt_to_exp <= self.dt_fr):
+            #     continue
+            for file in list(exp.files):
+
+                file_path_remote = pathlib.Path(file.name)
+                file_name_remote = file_path_remote.name
+
+                try:
+                    res = re.search(r'([\d]{4}\-[\d]{2}\-[\d]{2})_(\w+)@(\w+)', exp.name)
+                    thisday = datetime.datetime.strptime(res.groups()[0], "%Y-%m-%d")
+                    pulse_code = res.groups()[1]
+                    antenna_ = res.groups()[2]
+                    if 'uhf' in antenna_.lower():
+                        antenna = 'UHF'
+                    elif 'vhf' in antenna_.lower():
+                        antenna = 'VHF'
+                    elif any([a in antenna_.lower() for a in ['32m', '42m', 'esr']]):
+                        antenna = 'ESR'
+                    elif 'sod' in antenna_.lower():
+                        antenna = 'SOD'
+                    elif 'kir' in antenna_.lower():
+                        antenna = 'KIR'
+                    else:
+                        raise NotImplementedError
+                except Exception as e:
+                    print(e)
+                    mylog.StreamLogger.warning("Parsing the experiment name was failed! EXP: {}".format(exp.name))
+
+                file_dir_local = self.data_file_root_dir / antenna / thisday.strftime("%Y") / \
+                                (exp.name + '_EID-' + str(exp.id) + '_'
+                                 + dt_fr_exp.strftime("%Y%m%dT%H%M%S") + '_'
+                                 + dt_to_exp.strftime("%Y%m%dT%H%M%S"))
+                file_dir_local.mkdir(parents=True, exist_ok=True)
 
-class Downloader(object):
+                file_name_local = file_name_remote
+                file_path_local = file_dir_local / file_name_local
+
+                super().download(
+                    file_path_remote=file.name, file_path_local=file_path_local,
+                    file_format='hdf5')
+                file_paths.append(file_path_local)
+        return file_paths
+
+
+def test():
+    dt_0 = datetime.datetime(2016, 3, 9, 19)
+    dt_1 = datetime.datetime(2016, 3, 9, 23, 59)
+    downloader = DownloaderNew(
+        dt_fr=dt_0, dt_to=dt_1, 
+        kind_data='EISCAT',
+        antennas=['UHF', 'VHF'],
+        force_download=True,
+        exclude_file_type_patterns=['pp']
+        )
+    pass
+
+class DownloaderOld(object):
     """Download the quickplots and archieved analyzed results from EISCAT schedule webpage
     """
 
@@ -220,7 +445,8 @@ class Downloader(object):
 instrument_codes = {
     'UHF': [72],
     'VHF': [74],
-    'ESR': [95],
+    '42m': [95],
+    '32m': [95],
     'SOD': [73, 76],
     'KIR': [71, 75]
 }