geospacelab 0.11.4__py3-none-any.whl → 0.12.0__py3-none-any.whl

geospacelab/datahub/sources/cdaweb/omni/__init__.py

@@ -100,6 +100,8 @@ class Dataset(datahub.DatasetSourced):
         self.omni_res = kwargs.pop('omni_res', '1min')
         self.data_file_type = kwargs.pop('data_file_type','')
         self.allow_download = kwargs.pop('allow_download', True)
+        self.force_download = kwargs.pop('force_download', False)
+        self.download_dry_run = kwargs.pop('download_dry_run', False)
 
         self.metadata = None
 
@@ -165,7 +167,18 @@ class Dataset(datahub.DatasetSourced):
                                        '{:4d}'.format(thismonth.year)
                 file_patterns = [
                     self.omni_res,
-                    thismonth.strftime('%Y%m%d')
+                    thismonth.strftime('%Y%m')
+                ]
+            elif self.omni_res == '1h':
+                initial_file_dir = kwargs.pop('initial_file_dir', None)
+                if initial_file_dir is None:
+                    initial_file_dir = self.data_root_dir / \
+                        'OMNI2_low_res_1h' / \
+                        '{:4d}'.format(thismonth.year)
+                file_patterns = [
+                    'omni2',    
+                    'mrg1hr',
+                    thismonth.strftime('%Y')
                 ]
             else:
                 raise NotImplementedError
@@ -181,28 +194,27 @@ class Dataset(datahub.DatasetSourced):
 
             # Validate file paths
 
-            if not done and self.allow_download:
+            if (not done and self.allow_download) or (self.force_download):
                 done = self.download_data()
                 if done:
                     done = super().search_data_files(
                         initial_file_dir=initial_file_dir, search_pattern=search_pattern)
                 else:
                     print('Cannot find files from the online database!')
-
+        if list(self.data_file_paths):
+            self.data_file_paths = list(set(self.data_file_paths))
+            self.data_file_paths.sort()
         return done
 
     def download_data(self):
-        if self.data_file_type == 'hres-cdf':
-            if self.omni_type.upper() == 'OMNI':
-                new_omni = False
-            elif self.omni_type.upper() == 'OMNI2':
-                new_omni = True
-            download_obj = self.downloader(dt_fr=self.dt_fr, dt_to=self.dt_to,
-                                           res=self.omni_res, new_omni=new_omni,
-                                           data_file_root_dir=self.data_root_dir)
-        else:
-            raise NotImplementedError
-        return download_obj.done
+
+        download_obj = self.downloader(
+            dt_fr=self.dt_fr, dt_to=self.dt_to,
+            time_res=self.omni_res, product=self.omni_type,
+            root_dir_local=self.data_root_dir, 
+            force_download=self.force_download,
+            dry_run=self.download_dry_run,)
+        return any(download_obj.done)
 
     # _validate_IMF_cs = staticmethod(_validate_IMF_cs)
 

geospacelab/datahub/sources/cdaweb/omni/downloader.py

@@ -13,102 +13,115 @@ import datetime
 import requests
 import bs4
 import re
+import pathlib
 
 import geospacelab.toolbox.utilities.pydatetime as dttool
 import geospacelab.toolbox.utilities.pylogging as mylog
 from geospacelab.config import prf
 
-
-class Downloader(object):
-    def __init__(self, dt_fr,  dt_to, res='1min', new_omni=True, data_file_root_dir=None, version=None):
-        if res == '1h':
-            new_omni = True
-        self.dt_fr = dt_fr
-        self.dt_to = dt_to
-        self.res = res
-        self.new_omni = new_omni
+from geospacelab.datahub.sources.cdaweb.downloader import CDAWebHTTPDownloader as DownloaderBase
+
+
+class Downloader(DownloaderBase):
+    
+    def __init__(
+        self, 
+        dt_fr,  dt_to, 
+        time_res='1min', 
+        product='OMNI2', 
+        version='',
+        root_dir_local=None,
+        direct_download=True,
+        force_download=False,
+        dry_run=False,
+        ):
+
+        self.time_res = time_res
+        self.product = product
         self.version = version
-        self.done = False
-        if data_file_root_dir is None:
-            self.data_file_root_dir = prf.datahub_data_root_dir / "CDAWeb" / 'OMNI'
-        else:
-            self.data_file_root_dir = data_file_root_dir
-
-        self.url_base = "https://cdaweb.gsfc.nasa.gov/pub/data/omni/omni_cdaweb/"
-
-        self.download()
-
-    def download(self):
-        if self.res in ['1min', '5min']:
-            self.download_high_res_omni()
-        elif self.res == '1h':
-            self.download_low_res_omni()
-
-    def download_high_res_omni(self):
-        if self.new_omni:
-            omni_type = "hro2"
-            omni_dir_name = 'OMNI2'
+        
+        if root_dir_local is None:
+            root_dir_local = prf.datahub_data_root_dir / "CDAWeb" / 'OMNI' 
         else:
-            omni_type = "hro"
-            omni_dir_name = "OMNI"
-
-        num_month = (self.dt_to.year - self.dt_fr.year) * 12 + self.dt_to.month - self.dt_fr.month + 1
-        dt0 = datetime.datetime(self.dt_fr.year, self.dt_fr.month, 1)
-        for nm in range(num_month):
-            dt1 = dttool.get_next_n_months(dt0, nm)
-            url = self.url_base + omni_type + '_' + self.res + '/' + '{:4d}'.format(dt1.year) + '/'
-
-            r = requests.get(url)
-
-            soup = bs4.BeautifulSoup(r.text, 'html.parser')
-            a_tags = soup.find_all('a', href=True)
-            hrefs = []
-            for a_tag in a_tags:
-                href = a_tag['href']
-                pattern = omni_type + '_' + self.res + '_' + dt1.strftime("%Y%m%d")
-                if pattern in href:
-                    hrefs.append(href)
-            if len(hrefs) == 0:
-                mylog.StreamLogger.info("Cannot find the queried data file!")
-                return
-            if len(hrefs) > 1:
-                mylog.StreamLogger.warning("Find multiple matched files!")
-                print(hrefs)
-                return
-
-            href = hrefs[0]
-            ma = re.search('v[0-5][0-9]', href)
-            version = ma.group(0)
-            if self.version is None:
-                self.version = version
-            elif self.version != version:
-                mylog.StreamLogger.info("Cannot find the queried data file! Version={}.".format(version))
-
-            r_file = requests.get(url + href, allow_redirects=True)
-            file_name = href
-            file_path = self.data_file_root_dir / (omni_dir_name + '_high_res_' + self.res)
-            file_path = file_path / '{:4d}'.format(dt1.year) / file_name
-            if file_path.is_file():
-                mylog.simpleinfo.info(
-                    "The file {} exists in the directory {}.".format(file_path.name, file_path.parent.resolve()))
-            else:
-                file_path.parent.resolve().mkdir(parents=True, exist_ok=True)
-                with open(file_path, "wb") as omni:
-                    mylog.simpleinfo.info(
-                        "Downloading {} to the directory {} ...".format(file_path.name, file_path.parent.resolve())
-                    )
-                    omni.write(r_file.content)
-                    mylog.simpleinfo.info("Done")
-            self.done = True
-
-    def download_low_res_omni(self):
-        raise NotImplemented
+            root_dir_local = pathlib.Path(root_dir_local)
+        root_dir_remote = '/'.join([
+            self.root_dir_remote, 'omni', 'omni_cdaweb'
+            ])
+        super().__init__(
+            dt_fr, dt_to,
+            root_dir_local=root_dir_local,
+            root_dir_remote=root_dir_remote,
+            direct_download=direct_download,
+            force_download=force_download,
+            dry_run=dry_run,
+        ) 
+        
+    def search_from_http(self, *args, **kwargs):
+        dt_fr_1 = self.dt_fr
+        dt_to_1 = self.dt_to
+        diff_months = (dt_to_1.year - dt_fr_1.year) * 12 + dt_to_1.month - dt_fr_1.month
+        file_paths_remote = []
+        for nm in range(diff_months + 1):
+            this_month = dttool.get_next_n_months(dt_fr_1, nm)
+            
+            subdirs = []
+            
+            if self.time_res in ['1min', '5min']:
+                subdirs.append(f'{omni_product_dict[self.product]}_{self.time_res}')
+            elif self.time_res == '1h':
+                subdirs.append('hourly')
+            subdirs.append('{:4d}'.format(this_month.year))
+            
+            if self.time_res in ['1min', '5min',]:
+                file_name_patterns = [
+                    'omni', omni_product_dict[self.product], self.time_res, this_month.strftime("%Y%m")
+                    ]
+            elif self.time_res == '1h':
+                file_name_patterns = [
+                    'omni2', 'mrg1hr', this_month.strftime("%Y")
+                    ]
+            if str(self.version):
+                file_name_patterns.append(self.version)
+                
+            paths = super().search_from_http(subdirs=subdirs, file_name_patterns=file_name_patterns)
+            
+            if len(paths) > 1 and self.time_res != '1h':
+                mylog.StreamLogger.error("Find multiple matched files!")
+                print(paths)
+
+            file_paths_remote.extend(paths)
+        
+        return list(set(file_paths_remote))
+    
+    def save_files_from_http(self, file_paths_local=None, root_dir_remote=None):
+        
+        if file_paths_local is None:
+            file_paths_local = []
+            for fp_remote in self.file_paths_remote:
+                if self.time_res in ['1min', '5min']:
+                    subdir = f'{self.product}_high_res_{self.time_res}'
+                elif self.time_res == '1h':
+                    subdir = 'OMNI2_low_res_1h'
+            
+                fp_local = fp_remote.replace(self.base_url + '/' + self.root_dir_remote + '/', '')
+                pattern_replaced = fp_local.split('/')[0]
+                fp_local = fp_local.replace(
+                    pattern_replaced, subdir, 1
+                )
+                file_paths_local.append(self.root_dir_local / fp_local)
+        return super().save_files_from_http(file_paths_local, root_dir_remote)
+        
+
+omni_product_dict = {
+    'OMNI': 'hro',
+    'OMNI2': 'hro2',
+}
 
 
 def test():
     dt_fr = datetime.datetime(2020, 3, 4)
     dt_to = datetime.datetime(2020, 5, 3)
-    download_obj = Downloader(dt_fr, dt_to)
+    download_obj = Downloader(dt_fr, dt_to, force_download=True, time_res='1h')
     pass
 
 

geospacelab/datahub/sources/esa_eo/swarm/advanced/efi_tct02/__init__.py

@@ -170,7 +170,7 @@ class Dataset(datahub.DatasetSourced):
     def add_GEO_LST(self):
         lons = self['SC_GEO_LON'].flatten()
         uts = self['SC_DATETIME'].flatten()
-        lsts = [ut + datetime.timedelta(hours=lon / 15.) for ut, lon in zip(uts, lons)]
+        lsts = [ut + datetime.timedelta(seconds=int(lon / 15. * 3600)) for ut, lon in zip(uts, lons)]
         lsts = [lst.hour + lst.minute / 60. + lst.second / 3600. for lst in lsts]
         var = self.add_variable(var_name='SC_GEO_LST')
         var.value = np.array(lsts)[:, np.newaxis]

geospacelab/datahub/sources/esa_eo/swarm/advanced/efi_tct02/downloader.py

@@ -33,7 +33,7 @@ class Downloader(DownloaderModel):
             raise NotImplementedError
 
         if ftp_data_dir is None:
-            ftp_data_dir = f'Advanced/Plasma_Data/{frq_str}_TII_Cross-track_Dataset/Sat_{sat_id.upper()}'
+            ftp_data_dir = f'Advanced/Plasma_Data/{frq_str}_TII_Cross-track_Dataset/New_version/Sat_{sat_id.upper()}'
 
         if data_file_root_dir is None:
             data_file_root_dir = prf.datahub_data_root_dir / "ESA" / "SWARM" / "Advanced" / "EFI-TII" / data_type

geospacelab/datahub/sources/gfz/hpo/__init__.py

@@ -104,7 +104,7 @@ class Dataset(datahub.DatasetSourced):
                 import matplotlib.dates as mdates
                 self._variables['Hp'].visual.plot_config.bar['width'] = \
                     (mdates.date2num(datetime.datetime(2000, 1, 1,) + datetime.timedelta(minutes=self.data_res)) \
-                    - mdates.date2num(datetime.datetime(2000, 1, 1))) * 0.9
+                    - mdates.date2num(datetime.datetime(2000, 1, 1))) * 1
             self._variables['ap'].config(name=f'Hp{self.data_res}', label=f'ap{self.data_res}')
             # self.select_beams(field_aligned=True)
         if self.time_clip:

geospacelab/datahub/sources/gfz/hpo/variable_config.py

@@ -60,7 +60,9 @@ plot_config.bar = {
     'vmin': 0,
     'vmax': 10,
     'colormap': cm.cmap_for_kp(),
-    'width': mdates.date2num(datetime.datetime(1971, 1, 1, 0, 25)) - mdates.date2num(datetime.datetime(1971, 1, 1, 0))
+    'align': 'center',
+    'width': 1.
+    # 'width': mdates.date2num(datetime.datetime(1971, 1, 1, 0, 25)) - mdates.date2num(datetime.datetime(1971, 1, 1, 0))
 }
 # set axis attrs
 axis = var.visual.axis

geospacelab/datahub/sources/madrigal/isr/pfisr/fitted/loader.py

@@ -183,7 +183,7 @@ class Loader:
                 if var_name == 'nel':
                     vars_fh5['ne'] = 10**np.array(fh5_vars_2d[var_name]).T
                 elif var_name == 'dnel':
-                    vars_fh5['dne'] = 10 ** np.array(fh5_vars_2d[var_name]).T
+                    vars_fh5['dne'] = 10**np.array(fh5_vars_2d[var_name]).T
                 else:
                     arr = np.array(fh5_vars_2d[var_name]).T
 

geospacelab/datahub/sources/madrigal/satellites/dmsp/downloader.py

@@ -36,7 +36,8 @@ data_type_dict = {
     's4': 'plasma temp',
     'e': 'flux/energy',
     'ssies2': 'SSIES-2',
-    'hp':     'Hemispherical power'
+    'hp':     'Hemispherical power',
+    'ssies3':   'SSIES-3'
 }
 
 

geospacelab/datahub/sources/tud/champ/dns_acc/__init__.py

@@ -25,7 +25,7 @@ default_dataset_attrs = {
     'instrument': 'ACC',
     'product': 'DNS-ACC',
     'data_file_ext': 'txt',
-    'product_version': 'v01',
+    'product_version': 'v02',
     'data_root_dir': prf.datahub_data_root_dir / 'TUD' / 'CHAMP',
     'allow_load': True,
     'allow_download': True,
@@ -47,7 +47,19 @@ default_variable_names_v01 = [
     'SC_GEO_LST',
     'rho_n',
     ]
-default_variable_names_v02 = []
+default_variable_names_v02 = [
+    'SC_DATETIME',
+    'SC_GEO_LAT',
+    'SC_GEO_LON',
+    'SC_GEO_ALT',
+    'SC_ARG_LAT',
+    'SC_GEO_LST',
+    'rho_n',
+    'rho_n_MEAN',
+    'FLAG',
+    'FLAG_MEAN',
+    ]
+
 
 # default_data_search_recursive = True
 
@@ -64,10 +76,12 @@ class Dataset(datahub.DatasetSourced):
         self.facility = kwargs.pop('facility', 'CHAMP')
         self.instrument = kwargs.pop('instrument', 'ACC')
         self.product = kwargs.pop('product', 'DNS-ACC')
-        self.product_version = kwargs.pop('product_version', 'v01')
+        self.product_version = kwargs.pop('product_version', 'v02')
         self.local_latest_version = ''
         self.allow_download = kwargs.pop('allow_download', False)
         self.force_download = kwargs.pop('force_download', False)
+        self.download_dry_run = kwargs.pop('download_dry_run', False)
+        
         self.add_AACGM = kwargs.pop('add_AACGM', False) 
         self.add_APEX = kwargs.pop('add_APEX', False)
         self._data_root_dir = self.data_root_dir    # Record the initial root dir
@@ -195,7 +209,9 @@ class Dataset(datahub.DatasetSourced):
             file_patterns = [
                 'CH',
                 self.product.upper().replace('-', '_'),
-                this_day.strftime('%Y_%m'),
+                this_day.strftime('%Y'),
+                this_day.strftime('%m'),
+                self.product_version,
             ]
             # remove empty str
             file_patterns = [pattern for pattern in file_patterns if str(pattern)]
@@ -217,7 +233,7 @@ class Dataset(datahub.DatasetSourced):
                         search_pattern=search_pattern,
                         allow_multiple_files=False
                     )
-
+        self.data_file_paths = np.unique(self.data_file_paths)
         return done
 
     def download_data(self, dt_fr=None, dt_to=None):
@@ -229,10 +245,11 @@ class Dataset(datahub.DatasetSourced):
             dt_fr, dt_to,
             product=self.product,
             version=self.product_version,
-            force=self.force_download
+            force_download=self.force_download,
+            dry_run=self.download_dry_run
         )
 
-        return download_obj.done
+        return any(download_obj.done)
 
     @property
     def database(self):

geospacelab/datahub/sources/tud/champ/dns_acc/downloader.py

@@ -8,42 +8,35 @@ __license__ = "BSD-3-Clause License"
 __email__ = "lei.cai@oulu.fi"
 __docformat__ = "reStructureText"
 
+import datetime
 from geospacelab.config import prf
 
-from geospacelab.datahub.sources.tud.downloader import Downloader as DownloaderModel
-
-
-class Downloader(DownloaderModel):
-
-    def __init__(
-            self, dt_fr, dt_to,
-            product='DNS-ACC',
-            version='v01',
-            force=True, direct_download=True, **kwargs
-    ):
-        if version == 'v01':
-            v_str = "version_01"
-        elif version == 'v02':
-            raise NotImplementedError
-        else:
-            raise NotImplementedError
-        data_file_root_dir = prf.datahub_data_root_dir / "TUD" / "CHAMP" / product.upper() / version
-        ftp_data_dir = f'{v_str}/CHAMP_data'
-        file_name_patterns = ['CH', product.replace('-', '_')]
-        super(Downloader, self).__init__(
-            dt_fr, dt_to,
-            data_file_root_dir=data_file_root_dir, ftp_data_dir=ftp_data_dir, force=force,
-            direct_download=direct_download, file_name_patterns=file_name_patterns, **kwargs
+from geospacelab.datahub.sources.tud.downloader import TUDownloader as DownloaderBase   
+
+
+class Downloader(DownloaderBase):
+    
+    def __init__(self, 
+        dt_fr, dt_to, 
+        mission='CHAMP', 
+        sat_id='',
+        version='v02',  
+        product='DNS-ACC',
+        direct_download=True,
+        force_download=False,
+        dry_run=False,
+        ):
+        self.mission = mission
+        self.version = version
+        
+        super().__init__(
+            dt_fr, dt_to, 
+            mission=mission,
+            sat_id=sat_id, 
+            version=version,
+            product=product,
+            direct_download=direct_download,
+            force_download=force_download,
+            dry_run=dry_run
         )
-
-    def download(self, **kwargs):
-
-        done = super(Downloader, self).download(**kwargs)
-        return done
-
-    def search_files(self, **kwargs):
-
-        file_list = super(Downloader, self).search_files(**kwargs)
-
-        return file_list
-        # version control
+        

geospacelab/datahub/sources/tud/champ/dns_acc/loader.py

@@ -30,7 +30,7 @@ class Loader(object):
         if self.version == 'v01':
             self.load_v01()
         elif self.version == 'v02':
-            raise NotImplementedError 
+            self.load_v02()
         else:
             raise NotImplementedError
 
@@ -57,4 +57,30 @@ class Loader(object):
             self.variables['SC_GEO_LST'] = np.array(results[5]).astype(np.float32).reshape(num_rec, 1)
             self.variables['SC_ARG_LAT'] = np.array(results[6]).astype(np.float32).reshape(num_rec, 1)
             self.variables['rho_n'] = np.array(results[7]).astype(np.float32).reshape(num_rec, 1)
-            
+    
+    def load_v02(self):
+        with open(self.file_path, 'r') as f:
+            text = f.read()
+            results = re.findall(
+            r"^(\d{4}-\d{2}-\d{2}\s*\d{2}\:\d{2}\:\d{2}\.\d{3})\s*(\w{3})\s*" 
+            + r"([\-\d.]+)\s*([\-\d.]+)\s*([\-\d.]+)\s*([\-\d.]+)\s*([\-\d.]+)\s*([+\-\d.Ee]+)\s*([+\-\d.Ee]+)\s*([\-\d.]+)\s*([\-\d.]+)",
+            text,
+            re.M)
+            results = list(zip(*results))
+            dts = [datetime.datetime.strptime(dtstr + '000', "%Y-%m-%d %H:%M:%S.%f") for dtstr in results[0]]
+            num_rec = len(dts)
+            if results[1][0] == 'GPS':
+                self.variables['SC_GPSTIME'] = np.array(dts).reshape(num_rec, 1)
+                t_gps = [(dt - dttool._GPS_DATETIME_0).total_seconds() for dt in dts]
+                dts = dttool.convert_gps_time_to_datetime(t_gps, weeks=None)
+
+            self.variables['SC_DATETIME'] = np.array(dts).reshape(num_rec, 1)
+            self.variables['SC_GEO_ALT'] = np.array(results[2]).astype(np.float32).reshape(num_rec, 1) * 1e-3   # in km
+            self.variables['SC_GEO_LON'] = np.array(results[3]).astype(np.float32).reshape(num_rec, 1)
+            self.variables['SC_GEO_LAT'] = np.array(results[4]).astype(np.float32).reshape(num_rec, 1)
+            self.variables['SC_GEO_LST'] = np.array(results[5]).astype(np.float32).reshape(num_rec, 1)
+            self.variables['SC_ARG_LAT'] = np.array(results[6]).astype(np.float32).reshape(num_rec, 1)
+            self.variables['rho_n'] = np.array(results[7]).astype(np.float32).reshape(num_rec, 1)
+            self.variables['rho_n_MEAN'] = np.array(results[8]).astype(np.float32).reshape(num_rec, 1)
+            self.variables['FLAG'] = np.array(results[9]).astype(np.float32).reshape(num_rec, 1)
+            self.variables['FLAG_MEAN'] = np.array(results[10]).astype(np.float32).reshape(num_rec, 1)