geospacelab 0.9.1__tar.gz → 0.9.3__tar.gz

{geospacelab-0.9.1/geospacelab.egg-info → geospacelab-0.9.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: geospacelab
-Version: 0.9.1
+Version: 0.9.3
 Summary: Collect, manage, and visualize geospace data.
 Home-page: https://github.com/JouleCai/geospacelab
 Author: Lei Cai
@@ -104,6 +104,7 @@ on [readthedocs.io](https://geospacelab.readthedocs.io/en/latest/).
 | Data Source                  | Variables                          | File Format           | Downloadable  | Express                       | Status | 
 |------------------------------|------------------------------------|-----------------------|---------------|-------------------------------|--------|
 | CDAWeb/OMNI                  | Solar wind and IMF                 |*cdf*                 | *True*        | __OMNIDashboard__             | stable |
+| CDAWeb/DMSP/SSUSI/EDR_AUR    | DMSP SSUSI EDR_AUR emission lines  | *netcdf*              | *True*        | __DMSPSSUSIDashboard__        | stable |
 | Madrigal/EISCAT              | Ionospheric Ne, Te, Ti, ...        | *EISCAT-hdf5*, *Madrigal-hdf5* | *True* | __EISCATDashboard__           | stable |
 | Madrigal/GNSS/TECMAP         | Ionospheric GPS TEC map            | *hdf5*                | *True*        | -                             | beta   |
 | Madrigal/DMSP/s1             | DMSP SSM, SSIES, etc               | *hdf5*                | *True*        | __DMSPTSDashboard__           | stable |
@@ -111,7 +112,6 @@ on [readthedocs.io](https://geospacelab.readthedocs.io/en/latest/).
 | Madrigal/DMSP/e              | DMSP SSJ                           | *hdf5*                | *True*        | __DMSPTSDashboard__           | stable |
 | Madrigal/Millstone Hill ISR+ | Millstone Hill ISR                 | *hdf5*                | *True*        | __MillstoneHillISRDashboard__ | stable |
 | Madrigal/Poker Flat ISR      | Poker Flat ISR                     | *hdf5*                | *True*        | __-_                          | stable |
-| JHUAPL/DMSP/SSUSI            | DMSP SSUSI                         | *netcdf*              | *True*        | __DMSPSSUSIDashboard__        | stable |
 | JHUAPL/AMPERE/fitted         | AMPERE FAC                         | *netcdf*              | *False*        | __AMPEREDashboard__           | stable |
 | SuperDARN/POTMAP             | SuperDARN potential map            | *ascii*             | *False*       | -                             | stable |                  
 | WDC/Dst                      | Dst index                          | *IAGA2002-ASCII*      | *True*        | -                             | stable |
@@ -263,7 +263,7 @@ modulation = 'ant'
 dh = DataHub(dt_fr, dt_to)
 # dock the first dataset (dataset index starts from 0)
 ds_isr = dh.dock(datasource_contents=[database_name, 'isr', facility_name],
-                      site=site, antenna=antenna, modulation=modulation, data_file_type='eiscat-hdf5')
+                      site=site, antenna=antenna, modulation=modulation, data_file_type='madrigal-hdf5')
 # load data
 ds_isr.load_data()
 # assign a variable from its own dataset to the datahub
@@ -304,7 +304,8 @@ antenna = 'UHF'
 modulation = '60'
 load_mode = 'AUTO'
 dashboard = eiscat.EISCATDashboard(
-    dt_fr, dt_to, site=site, antenna=antenna, modulation=modulation, load_mode='AUTO'
+    dt_fr, dt_to, site=site, antenna=antenna, modulation=modulation, load_mode='AUTO', 
+    data_file_type="madrigal-hdf5"
 )
 dashboard.quicklook()
 
@@ -373,6 +374,9 @@ Output:
 > ![alt text](https://github.com/JouleCai/geospacelab/blob/master/examples/OMNI_1min_20160314-060000-20160320-060000.png?raw=true)
 
 ### Example 4: Mapping geospatial data in the polar map.
+> **_NOTE_**: JHUAPL stopped supporting DMSP SSUSI on 14 Feb 2025. 
+From then on, the data source has been switched from JHUAPL to CDAWeb.
+ 
 ```python
 import datetime
 import matplotlib.pyplot as plt
@@ -392,12 +396,16 @@ dashboard = geomap.GeoDashboard(dt_fr=dt_fr, dt_to=dt_to, figure_config={'figsiz
 # If the orbit_id is specified, only one file will be downloaded. This option saves the downloading time.
 # dashboard.dock(datasource_contents=['jhuapl', 'dmsp', 'ssusi', 'edraur'], pole='N', sat_id='f17', orbit_id='46863')
 # If not specified, the data during the whole day will be downloaded.
-dashboard.dock(datasource_contents=['jhuapl', 'dmsp', 'ssusi', 'edraur'], pole=pole, sat_id=sat_id, orbit_id=None)
+dashboard.dock(
+    datasource_contents=dashboard.dock(datasource_contents=['cdaweb', 'dmsp', 'ssusi', 'edr_aur'], 
+    pole=pole, sat_id=sat_id, orbit_id=None
+)
 ds_s1 = dashboard.dock(
     datasource_contents=['madrigal', 'satellites', 'dmsp', 's1'],
     dt_fr=time1 - datetime.timedelta(minutes=45),
     dt_to=time1 + datetime.timedelta(minutes=45),
-    sat_id=sat_id)
+    sat_id=sat_id
+)
 
 dashboard.set_layout(1, 1)
 

{geospacelab-0.9.1 → geospacelab-0.9.3}/README.md

@@ -48,6 +48,7 @@ on [readthedocs.io](https://geospacelab.readthedocs.io/en/latest/).
 | Data Source                  | Variables                          | File Format           | Downloadable  | Express                       | Status | 
 |------------------------------|------------------------------------|-----------------------|---------------|-------------------------------|--------|
 | CDAWeb/OMNI                  | Solar wind and IMF                 |*cdf*                 | *True*        | __OMNIDashboard__             | stable |
+| CDAWeb/DMSP/SSUSI/EDR_AUR    | DMSP SSUSI EDR_AUR emission lines  | *netcdf*              | *True*        | __DMSPSSUSIDashboard__        | stable |
 | Madrigal/EISCAT              | Ionospheric Ne, Te, Ti, ...        | *EISCAT-hdf5*, *Madrigal-hdf5* | *True* | __EISCATDashboard__           | stable |
 | Madrigal/GNSS/TECMAP         | Ionospheric GPS TEC map            | *hdf5*                | *True*        | -                             | beta   |
 | Madrigal/DMSP/s1             | DMSP SSM, SSIES, etc               | *hdf5*                | *True*        | __DMSPTSDashboard__           | stable |
@@ -55,7 +56,6 @@ on [readthedocs.io](https://geospacelab.readthedocs.io/en/latest/).
 | Madrigal/DMSP/e              | DMSP SSJ                           | *hdf5*                | *True*        | __DMSPTSDashboard__           | stable |
 | Madrigal/Millstone Hill ISR+ | Millstone Hill ISR                 | *hdf5*                | *True*        | __MillstoneHillISRDashboard__ | stable |
 | Madrigal/Poker Flat ISR      | Poker Flat ISR                     | *hdf5*                | *True*        | __-_                          | stable |
-| JHUAPL/DMSP/SSUSI            | DMSP SSUSI                         | *netcdf*              | *True*        | __DMSPSSUSIDashboard__        | stable |
 | JHUAPL/AMPERE/fitted         | AMPERE FAC                         | *netcdf*              | *False*        | __AMPEREDashboard__           | stable |
 | SuperDARN/POTMAP             | SuperDARN potential map            | *ascii*             | *False*       | -                             | stable |                  
 | WDC/Dst                      | Dst index                          | *IAGA2002-ASCII*      | *True*        | -                             | stable |
@@ -207,7 +207,7 @@ modulation = 'ant'
 dh = DataHub(dt_fr, dt_to)
 # dock the first dataset (dataset index starts from 0)
 ds_isr = dh.dock(datasource_contents=[database_name, 'isr', facility_name],
-                      site=site, antenna=antenna, modulation=modulation, data_file_type='eiscat-hdf5')
+                      site=site, antenna=antenna, modulation=modulation, data_file_type='madrigal-hdf5')
 # load data
 ds_isr.load_data()
 # assign a variable from its own dataset to the datahub
@@ -248,7 +248,8 @@ antenna = 'UHF'
 modulation = '60'
 load_mode = 'AUTO'
 dashboard = eiscat.EISCATDashboard(
-    dt_fr, dt_to, site=site, antenna=antenna, modulation=modulation, load_mode='AUTO'
+    dt_fr, dt_to, site=site, antenna=antenna, modulation=modulation, load_mode='AUTO', 
+    data_file_type="madrigal-hdf5"
 )
 dashboard.quicklook()
 
@@ -317,6 +318,9 @@ Output:
 > ![alt text](https://github.com/JouleCai/geospacelab/blob/master/examples/OMNI_1min_20160314-060000-20160320-060000.png?raw=true)
 
 ### Example 4: Mapping geospatial data in the polar map.
+> **_NOTE_**: JHUAPL stopped supporting DMSP SSUSI on 14 Feb 2025. 
+From then on, the data source has been switched from JHUAPL to CDAWeb.
+ 
 ```python
 import datetime
 import matplotlib.pyplot as plt
@@ -336,12 +340,16 @@ dashboard = geomap.GeoDashboard(dt_fr=dt_fr, dt_to=dt_to, figure_config={'figsiz
 # If the orbit_id is specified, only one file will be downloaded. This option saves the downloading time.
 # dashboard.dock(datasource_contents=['jhuapl', 'dmsp', 'ssusi', 'edraur'], pole='N', sat_id='f17', orbit_id='46863')
 # If not specified, the data during the whole day will be downloaded.
-dashboard.dock(datasource_contents=['jhuapl', 'dmsp', 'ssusi', 'edraur'], pole=pole, sat_id=sat_id, orbit_id=None)
+dashboard.dock(
+    datasource_contents=dashboard.dock(datasource_contents=['cdaweb', 'dmsp', 'ssusi', 'edr_aur'], 
+    pole=pole, sat_id=sat_id, orbit_id=None
+)
 ds_s1 = dashboard.dock(
     datasource_contents=['madrigal', 'satellites', 'dmsp', 's1'],
     dt_fr=time1 - datetime.timedelta(minutes=45),
     dt_to=time1 + datetime.timedelta(minutes=45),
-    sat_id=sat_id)
+    sat_id=sat_id
+)
 
 dashboard.set_layout(1, 1)
 

{geospacelab-0.9.1 → geospacelab-0.9.3}/geospacelab/__init__.py

@@ -6,7 +6,7 @@ __author__ = "Lei Cai"
 __copyright__ = "Copyright 2021, GeospaceLAB"
 __credits__ = ["Lei Cai"]
 __license__ = "BSD-3-Clause License"
-__version__ = "0.9.1"
+__version__ = "0.9.3"
 __maintainer__ = "Lei Cai"
 __email__ = "lei.cai@oulu.fi"
 __status__ = "Developing"

geospacelab-0.9.3/geospacelab/datahub/sources/cdaweb/downloader.py

@@ -0,0 +1,152 @@
+# Licensed under the BSD 3-Clause License
+# Copyright (C) 2021 GeospaceLab (geospacelab)
+# Author: Lei Cai, Space Physics and Astronomy, University of Oulu
+
+__author__ = "Lei Cai"
+__copyright__ = "Copyright 2021, GeospaceLab"
+__license__ = "BSD-3-Clause License"
+__email__ = "lei.cai@oulu.fi"
+__docformat__ = "reStructureText"
+
+
+import datetime
+import pathlib
+
+import requests
+import bs4
+import re
+import tqdm
+
+import geospacelab.toolbox.utilities.pydatetime as dttool
+import geospacelab.toolbox.utilities.pylogging as mylog
+from geospacelab.config import prf
+
+
+class Downloader(object):
+
+    def __init__(
+        self, 
+        dt_fr, 
+        dt_to,
+        direct_download=True, 
+        force_download=False,
+        data_file_root_dir=None,
+        dry_run=False,
+        from_ftp=False,
+        ):
+
+        self.url_base = "https://cdaweb.gsfc.nasa.gov/pub/data/"
+        self.force_download = force_download
+        self.dry_run=dry_run
+        self.from_ftp=from_ftp
+
+        self.dt_fr = dt_fr
+        self.dt_to = dt_to
+        self.source_file_paths = []
+
+        if data_file_root_dir is None:
+            self.data_file_root_dir = prf.datahub_data_root_dir
+        else:
+            self.data_file_root_dir = data_file_root_dir
+
+        if direct_download:
+            self.download()
+
+    def download(self,):
+        if self.from_ftp:
+            raise NotImplementedError
+        else:
+            self.download_from_http()
+            
+    def search_from_http(self, subdirs=None, file_name_patterns=None):
+        url = self.url_base + '/'.join(subdirs)
+        r = requests.get(url)
+
+        soup = bs4.BeautifulSoup(r.text, 'html.parser')
+        a_tags = soup.find_all('a', href=True)
+        hrefs = [a_tag['href'] for a_tag in a_tags]
+
+        search_pattern = '.*' + '.*'.join(file_name_patterns) + '.*'
+        fn_regex = re.compile(search_pattern)
+        hrefs = list(filter(fn_regex.match, hrefs))
+
+        paths = []
+        for href in hrefs:
+            paths.append(url + '/' + href)
+        return paths
+
+    def download_from_http(self, ):
+
+        source_file_paths = self.search_from_http()
+
+        for url  in source_file_paths:
+            if self.dry_run:
+                print(f"Dry run: {url}.")
+            else:
+                self.save_file_from_http(url=url)
+
+    def save_file_from_http(self, url, file_dir=None, file_name=None):
+            if file_name is None:
+                file_name = url.split('/')[-1]
+            file_path = file_dir / file_name
+            if file_path.is_file():
+                mylog.simpleinfo.info(
+                    "The file {} exists in the directory {}.".format(file_path.name, file_path.parent.resolve()))
+                if not self.force_download:
+                    self.done = True
+                    return
+
+            file_path.parent.resolve().mkdir(parents=True, exist_ok=True)
+            mylog.simpleinfo.info(f'Downloading {file_name} ...')
+            res = self._download_by_requests_get(url, data_file_path=file_path)
+            if res:
+                mylog.simpleinfo.info(f'Saved in {file_dir}')
+                self.done = True
+            else:
+                mylog.StreamLogger.error(f"Error during downloading. Code: {res}.")
+
+            return
+
+    @staticmethod
+    def _download_by_requests_get(url, data_file_path=None, params=None, stream=True, allow_redirects=True, **kwargs):
+        r = requests.get(url, params=params, stream=stream, allow_redirects=allow_redirects, **kwargs)
+        if r.status_code != 200:
+            return -2
+
+        total_size_in_bytes = int(r.headers.get('content-length', 0))
+        block_size = 1024  # 1 Kibibyte
+        progress_bar = tqdm.tqdm(total=total_size_in_bytes, unit='iB', unit_scale=True)
+
+        with open(data_file_path, 'wb') as file:
+            for data in r.iter_content(block_size):
+                progress_bar.update(len(data))
+                file.write(data)
+        progress_bar.close()
+        if total_size_in_bytes != 0 and progress_bar.n != total_size_in_bytes:
+            mylog.StreamLogger.error("Something wrong during the download!")
+            data_file_path.unlink(missing_ok=True)
+            return -1
+
+        return 1
+
+    def search_from_ftp(self, ftp, file_name_patterns=None):
+        paths = []
+
+        return paths
+
+    def save_file(self, file_dir, file_name, r_source_file):
+        file_path = file_dir / file_name
+
+        if file_path.is_file():
+            mylog.simpleinfo.info(
+                "The file {} exists in the directory {}.".format(file_path.name, file_path.parent.resolve()))
+
+        if self.force_download:
+            file_path.parent.resolve().mkdir(parents=True, exist_ok=True)
+            with open(file_path, "wb") as f:
+                mylog.simpleinfo.info(
+                    "Downloading {} to the directory {} ...".format(file_path.name, file_path.parent.resolve())
+                )
+                f.write(r_source_file.content)
+                mylog.simpleinfo.info("Done")
+

{geospacelab-0.9.1 → geospacelab-0.9.3}/geospacelab/datahub/sources/esa_eo/swarm/downloader.py

@@ -71,14 +71,16 @@ class Downloader(DownloaderBase):
             this_month = dttool.get_next_n_months(self.dt_fr, nm)
             file_name_patterns = copy.deepcopy(default_file_name_patterns)
             file_name_patterns.append(this_month.strftime("%Y%m"))
+            ftp = ftplib.FTP_TLS()
+            ftp.connect(self.ftp_host, self.ftp_port, 30)
             try:
-                ftp = ftplib.FTP_TLS()
-                ftp.connect(self.ftp_host, self.ftp_port, 30)  # 30 timeout
                 ftp.login(user=self.username, passwd=self.__password__)
                 ftp.cwd(self.ftp_data_dir)
                 file_list = ftp.nlst()
                
                 file_names, versions = self.search_files(file_list=file_list, file_name_patterns=file_name_patterns)
+                if file_names is None:
+                    raise FileNotFoundError
                 file_dir_root = self.data_file_root_dir
                 for ind_f, file_name in enumerate(file_names):
                     dt_regex = re.compile(r'(\d{8}T\d{6})_(\d{8}T\d{6})_(\d{4})')
@@ -129,11 +131,11 @@ class Downloader(DownloaderBase):
                     mylog.simpleinfo.info("Done. The zip file has been removed.")
 
                 done = True
-                ftp.quit()
             except Exception as e:
                 print('Error during download from FTP')
                 print(e)
                 done = False
+            ftp.quit()
         return done
 
     def search_files(self, file_list=None, file_name_patterns=None):
@@ -166,7 +168,8 @@ class Downloader(DownloaderBase):
 
         ind_dt = np.where((self.dt_fr <= stop_dts) & (self.dt_to >= start_dts))[0]
         if not list(ind_dt):
-            raise FileExistsError
+            mylog.StreamLogger.info("No matching files found on the ftp")
+            return None, None
         file_list = [file_list[ii] for ii in ind_dt]
         versions = [versions[ii] for ii in ind_dt]
 

{geospacelab-0.9.1 → geospacelab-0.9.3}/geospacelab/datahub/sources/esa_eo/swarm/l2daily/dns_acc/__init__.py

@@ -14,9 +14,9 @@ from geospacelab.config import prf
 import geospacelab.toolbox.utilities.pybasic as basic
 import geospacelab.toolbox.utilities.pylogging as mylog
 import geospacelab.toolbox.utilities.pydatetime as dttool
-from geospacelab.datahub.sources.esa_eo.swarm.l2daily.dns_pod.loader import Loader as default_Loader
-from geospacelab.datahub.sources.esa_eo.swarm.l2daily.dns_pod.downloader import Downloader as default_Downloader
-import geospacelab.datahub.sources.esa_eo.swarm.l2daily.dns_pod.variable_config as var_config
+from geospacelab.datahub.sources.esa_eo.swarm.l2daily.dns_acc.loader import Loader as default_Loader
+from geospacelab.datahub.sources.esa_eo.swarm.l2daily.dns_acc.downloader import Downloader as default_Downloader
+import geospacelab.datahub.sources.esa_eo.swarm.l2daily.dns_acc.variable_config as var_config
 
 
 default_dataset_attrs = {
@@ -47,8 +47,6 @@ default_variable_names = [
     'SC_GEO_ALT',
     'SC_GEO_LST',
     'rho_n',
-    'rho_n_ORBITMEAN',
-    'FLAG',
     ]
 
 # default_data_search_recursive = True
@@ -78,7 +76,7 @@ class Dataset(datahub.DatasetSourced):
 
         self.sat_id = kwargs.pop('sat_id', 'A')
 
-        self.metadata = None
+        self.metadata = {}
 
         allow_load = kwargs.pop('allow_load', False)
 
@@ -212,6 +210,63 @@ class Dataset(datahub.DatasetSourced):
         self['SC_AACGM_LON'].value = cs_aacgm['lon'].reshape(self['SC_DATETIME'].value.shape)
         self['SC_AACGM_MLT'].value = cs_aacgm['mlt'].reshape(self['SC_DATETIME'].value.shape)
 
+    def interp_evenly(self, time_res=None, data_time_res = 30, dt_fr=None, dt_to=None, masked=False):
+        from scipy.interpolate import interp1d
+        import geospacelab.toolbox.utilities.numpymath as nm
+
+        if time_res is None:
+            time_res = data_time_res
+
+        ds_new = datahub.DatasetUser(dt_fr=self.dt_fr, dt_to=self.dt_to, visual=self.visual)
+        ds_new.clone_variables(self)
+        dts = ds_new['SC_DATETIME'].value.flatten()
+        dt0 = dttool.get_start_of_the_day(dts[0])
+        x_0 = np.array([(dt - dt0).total_seconds() for dt in dts])
+        if dt_fr is None:
+            dt_fr = dts[0] - datetime.timedelta(
+                seconds=np.floor(((dts[0] - ds_new.dt_fr).total_seconds() / time_res)) * time_res
+            )
+        if dt_to is None:
+            dt_to = dts[-1] + datetime.timedelta(
+                seconds=np.floor(((ds_new.dt_to - dts[-1]).total_seconds() / time_res)) * time_res
+            )
+        sec_fr = (dt_fr - dt0).total_seconds()
+        sec_to = (dt_to - dt0).total_seconds()
+        x_1 = np.arange(sec_fr, sec_to + time_res / 2, time_res)
+
+        f = interp1d(x_0, x_0, kind='nearest', bounds_error=False, fill_value=(x_0[0], x_0[-1]))
+        pseudo_x = f(x_1)
+        mask = np.abs(pseudo_x - x_1) > data_time_res / 1.5
+
+        dts_new = np.array([dt0 + datetime.timedelta(seconds=sec) for sec in x_1])
+        ds_new['SC_DATETIME'].value = dts_new.reshape((dts_new.size, 1))
+
+        period_var_dict = {'SC_GEO_LON': 360.,
+                           'SC_AACGM_LON': 360.,
+                           'SC_APEX_LON': 360.,
+                           'SC_GEO_LST': 24.,
+                           'SC_AACGM_MLT': 24.,
+                           'SC_APEX_MLT': 24}
+
+        for var_name in ds_new.keys():
+            if var_name in ['SC_DATETIME']:
+                continue
+            if var_name in period_var_dict.keys():
+                var = ds_new[var_name].value.flatten()
+                var_new = nm.interp_period_data(x_0, var, x_1, period=period_var_dict[var_name], method='linear',
+                                                bounds_error=False)
+            else:
+                method = 'linear' if 'FLAG' not in var_name else 'nearest'
+                var = ds_new[var_name].value.flatten()
+                f = interp1d(x_0, var, kind=method, bounds_error=False)
+                var_new = f(x_1)
+            if masked:
+                var_new = np.ma.array(var_new, mask=mask, fill_value=np.nan)
+            else:
+                var_new[mask] = np.nan
+            ds_new[var_name].value = var_new.reshape((dts_new.size, 1))
+        return ds_new
+
     def time_filter_by_quality(self, quality_lim=None):
         if quality_lim is None:
             quality_lim = 0
@@ -253,7 +308,7 @@ class Dataset(datahub.DatasetSourced):
                 sat_id = self.sat_id
             file_patterns = [
                 'DNS' + sat_id.upper(),
-                'POD_2',
+                'ACC_2',
                 this_day.strftime('%Y%m%d') + 'T'
             ]
             # remove empty str
@@ -276,6 +331,7 @@ class Dataset(datahub.DatasetSourced):
                         search_pattern=search_pattern,
                         allow_multiple_files=True
                     )
+                self.allow_download = False
 
         return done
 

{geospacelab-0.9.1 → geospacelab-0.9.3}/geospacelab/datahub/sources/esa_eo/swarm/l2daily/dns_acc/downloader.py

@@ -29,7 +29,7 @@ class Downloader(DownloaderModel):
 
         if ftp_data_dir is None:
             ftp_data_dir = f'Level2daily/Latest_baselines/DNS/ACC/Sat_{sat_id.upper()}'
-
+            # ftp_data_dir = f'Level2daily/Older_baselines/DNS/ACC/Sat_{sat_id.upper()}'
         if data_file_root_dir is None:
             data_file_root_dir = prf.datahub_data_root_dir / "ESA" / "SWARM" / "Level2daily" / "DNS_ACC"
         file_name_patterns = list(self._default_file_name_patterns)

{geospacelab-0.9.1 → geospacelab-0.9.3}/geospacelab/datahub/sources/esa_eo/swarm/l2daily/dns_acc/loader.py

@@ -14,14 +14,12 @@ from geospacelab.datahub.sources.esa_eo.swarm.loader import LoaderModel
 
 # define the default variable name dictionary
 default_variable_name_dict = {
-    'CDF_EPOCH': 'Time',
+    'CDF_EPOCH': 'time',
     'SC_GEO_LAT': 'latitude',
     'SC_GEO_LON': 'longitude',
     'SC_GEO_ALT': 'altitude',
     'SC_GEO_LST': 'local_solar_time',
     'rho_n': 'density',
-    'rho_n_ORBITMEAN': 'density_orbitmean',
-    'FLAG': 'validity_flag',
 }
 
 
@@ -38,4 +36,5 @@ class Loader(LoaderModel):
 
     def load_data(self, **kwargs):
         super(Loader, self).load_data(**kwargs)
+        self.variables['rho_n'][self.variables['rho_n']>1] = np.nan
         self.variables['SC_GEO_ALT'] = self.variables['SC_GEO_ALT'] * 1e-3

{geospacelab-0.9.1 → geospacelab-0.9.3}/geospacelab/datahub/sources/esa_eo/swarm/l2daily/dns_pod/__init__.py

@@ -78,7 +78,7 @@ class Dataset(datahub.DatasetSourced):
 
         self.sat_id = kwargs.pop('sat_id', 'A')
 
-        self.metadata = None
+        self.metadata = {}
 
         allow_load = kwargs.pop('allow_load', False)
 
@@ -167,7 +167,7 @@ class Dataset(datahub.DatasetSourced):
     def add_GEO_LST(self):
         lons = self['SC_GEO_LON'].flatten()
         uts = self['SC_DATETIME'].flatten()
-        lsts = [ut + datetime.timedelta(hours=lon / 15.) for ut, lon in zip(uts, lons)]
+        lsts = [ut + datetime.timedelta(hours=int(lon / 15.)) for ut, lon in zip(uts, lons)]
         lsts = [lst.hour + lst.minute / 60. + lst.second / 3600. for lst in lsts]
         var = self.add_variable(var_name='SC_GEO_LST')
         var.value = np.array(lsts)[:, np.newaxis]
@@ -212,6 +212,63 @@ class Dataset(datahub.DatasetSourced):
         self['SC_AACGM_LON'].value = cs_aacgm['lon'].reshape(self['SC_DATETIME'].value.shape)
         self['SC_AACGM_MLT'].value = cs_aacgm['mlt'].reshape(self['SC_DATETIME'].value.shape)
 
+    def interp_evenly(self, time_res=None, data_time_res = 30, dt_fr=None, dt_to=None, masked=False):
+        from scipy.interpolate import interp1d
+        import geospacelab.toolbox.utilities.numpymath as nm
+
+        if time_res is None:
+            time_res = data_time_res
+
+        ds_new = datahub.DatasetUser(dt_fr=self.dt_fr, dt_to=self.dt_to, visual=self.visual)
+        ds_new.clone_variables(self)
+        dts = ds_new['SC_DATETIME'].value.flatten()
+        dt0 = dttool.get_start_of_the_day(dts[0])
+        x_0 = np.array([(dt - dt0).total_seconds() for dt in dts])
+        if dt_fr is None:
+            dt_fr = dts[0] - datetime.timedelta(
+                seconds=np.floor(((dts[0] - ds_new.dt_fr).total_seconds() / time_res)) * time_res
+            )
+        if dt_to is None:
+            dt_to = dts[-1] + datetime.timedelta(
+                seconds=np.floor(((ds_new.dt_to - dts[-1]).total_seconds() / time_res)) * time_res
+            )
+        sec_fr = (dt_fr - dt0).total_seconds()
+        sec_to = (dt_to - dt0).total_seconds()
+        x_1 = np.arange(sec_fr, sec_to + time_res / 2, time_res)
+
+        f = interp1d(x_0, x_0, kind='nearest', bounds_error=False, fill_value=(x_0[0], x_0[-1]))
+        pseudo_x = f(x_1)
+        mask = np.abs(pseudo_x - x_1) > data_time_res / 1.5
+
+        dts_new = np.array([dt0 + datetime.timedelta(seconds=sec) for sec in x_1])
+        ds_new['SC_DATETIME'].value = dts_new.reshape((dts_new.size, 1))
+
+        period_var_dict = {'SC_GEO_LON': 360.,
+                           'SC_AACGM_LON': 360.,
+                           'SC_APEX_LON': 360.,
+                           'SC_GEO_LST': 24.,
+                           'SC_AACGM_MLT': 24.,
+                           'SC_APEX_MLT': 24}
+
+        for var_name in ds_new.keys():
+            if var_name in ['SC_DATETIME']:
+                continue
+            if var_name in period_var_dict.keys():
+                var = ds_new[var_name].value.flatten()
+                var_new = nm.interp_period_data(x_0, var, x_1, period=period_var_dict[var_name], method='linear',
+                                                bounds_error=False)
+            else:
+                method = 'linear' if 'FLAG' not in var_name else 'nearest'
+                var = ds_new[var_name].value.flatten()
+                f = interp1d(x_0, var, kind=method, bounds_error=False)
+                var_new = f(x_1)
+            if masked:
+                var_new = np.ma.array(var_new, mask=mask, fill_value=np.nan)
+            else:
+                var_new[mask] = np.nan
+            ds_new[var_name].value = var_new.reshape((dts_new.size, 1))
+        return ds_new
+
     def time_filter_by_quality(self, quality_lim=None):
         if quality_lim is None:
             quality_lim = 0

{geospacelab-0.9.1 → geospacelab-0.9.3}/geospacelab/datahub/sources/esa_eo/swarm/l2daily/dns_pod/downloader.py

@@ -29,9 +29,11 @@ class Downloader(DownloaderModel):
 
         if ftp_data_dir is None:
             ftp_data_dir = f'Level2daily/Latest_baselines/DNS/POD/Sat_{sat_id.upper()}'
+            # ftp_data_dir = f'Level2daily/Older_baselines/DNS/POD/Sat_{sat_id.upper()}'
 
         if data_file_root_dir is None:
             data_file_root_dir = prf.datahub_data_root_dir / "ESA" / "SWARM" / "Level2daily" / "DNS_POD"
+
         file_name_patterns = list(self._default_file_name_patterns)
 
         file_name_patterns.extend(['DNS' + sat_id.upper()])

{geospacelab-0.9.1 → geospacelab-0.9.3}/geospacelab/datahub/sources/esa_eo/swarm/l2daily/dns_pod/loader.py

@@ -38,4 +38,7 @@ class Loader(LoaderModel):
 
     def load_data(self, **kwargs):
         super(Loader, self).load_data(**kwargs)
+        self.variables['rho_n'][self.variables['rho_n'] > 1] = np.nan
+        self.variables['rho_n'][self.variables['rho_n'] <= 0] = np.nan
+        self.variables['rho_n'][self.variables['FLAG'] > 0] = np.nan
         self.variables['SC_GEO_ALT'] = self.variables['SC_GEO_ALT'] * 1e-3

{geospacelab-0.9.1 → geospacelab-0.9.3}/geospacelab/datahub/sources/jhuapl/dmsp/ssusi/edraur/variable_config.py

@@ -30,7 +30,7 @@ default_plot_config = {
     'pcolormesh':   {
         'cmap':            default_colormap,
         'c_scale':         'log',
-        'c_lim':           [100, 6000],
+        'c_lim':           [10, 6000],
         'alpha':            0.9,
     }
 }

{geospacelab-0.9.1 → geospacelab-0.9.3}/geospacelab/express/eiscat_dashboard.py

@@ -108,7 +108,7 @@ class EISCATDashboard(TSDashboard):
 
     def save_figure(self, file_name=None, file_dir=None, append_time=True, **kwargs):
         if file_name is None:
-            file_name = kwargs.pop('file_name', self.title.replace(', ', '_'))
+            file_name = '_'.join([s.strip() for s in self.title.split(',')])
         super().save_figure(file_name=file_name, file_dir=file_dir, append_time=append_time, **kwargs)
 
     def add_title(self, x=0.5, y=1.06, title=None, append_time=True, **kwargs):
@@ -158,7 +158,8 @@ def example():
     modulation = '60'
     load_mode = 'AUTO'
     dashboard = EISCATDashboard(
-        dt_fr, dt_to, site=site, antenna=antenna, modulation=modulation, load_mode='AUTO'
+        dt_fr, dt_to, site=site, antenna=antenna, modulation=modulation, load_mode='AUTO',
+        data_file_type="madrigal-hdf5"
     )
     dashboard.quicklook()
 

{geospacelab-0.9.1 → geospacelab-0.9.3}/geospacelab/observatory/orbit/utilities.py

@@ -421,6 +421,7 @@ class LEOToolbox(DatasetUser):
 
         method = 'linear'
         for var_name in variable_names:
+            # print(var_name)
             vrb = self[var_name].value.flatten()[sector>0]
             if along_track_binning:
                 sector_1 = sector[sector>0]

{geospacelab-0.9.1 → geospacelab-0.9.3}/geospacelab/visualization/mpl/dashboards.py

@@ -385,7 +385,7 @@ class TSDashboard(Dashboard):
 
         return results
 
-    def add_shading(self, dt_fr, dt_to, panel_index=0,
+    def add_shading(self, dt_fr, dt_to, panel_index=-1,
                     label=None, label_position=None, top_extend=0., bottom_extend=0., **kwargs):
 
         if label_position is None: