geospacelab 0.11.4__py3-none-any.whl → 0.12.0__py3-none-any.whl

geospacelab/datahub/sources/cdaweb/downloader.py

@@ -12,6 +12,7 @@ __docformat__ = "reStructureText"
 import datetime
 import pathlib
 
+import ftplib
 import requests
 import bs4
 import re
@@ -22,132 +23,430 @@ import geospacelab.toolbox.utilities.pylogging as mylog
 from geospacelab.config import prf
 
 
-class Downloader(object):
-
+class DownloaderBase(object):
+    """
+
+    Parameters
+    ----------
+    dt_fr : datetime.datetime
+        The start datetime for downloading data.
+    dt_to : datetime.datetime
+        The end datetime for downloading data.
+    root_dir_local : str or pathlib.Path
+        The root directory in the local disk to store the downloaded data files.
+    direct_download : bool
+        Whether to start downloading once the Downloader object is created.
+    force_download : bool
+        Whether to force re-download even the data files are already in the local disk.
+    dry_run : bool
+        Whether to only print the downloading information without actual downloading.
+    done : list of bool
+        Whether the downloading is done. The length of the list is the number of files to be downloaded.
+    file_paths_local : list of pathlib.Path
+        The local file paths of the downloaded data files.
+    file_paths_remote : list of str
+        The remote file paths of the data files to be downloaded.
+    
+    """
     def __init__(
-        self, 
-        dt_fr, 
-        dt_to,
-        direct_download=True, 
+        self,
+        dt_fr=None,
+        dt_to=None,
+        root_dir_local=None,
+        direct_download=False,
         force_download=False,
-        data_file_root_dir=None,
         dry_run=False,
-        from_ftp=False,
-        ):
+        download_from = None,
+        **kwargs
+    ):
 
-        self.url_base = "https://cdaweb.gsfc.nasa.gov/pub/data/"
         self.force_download = force_download
-        self.dry_run=dry_run
-        self.from_ftp=from_ftp
+        self.dry_run = dry_run
 
         self.dt_fr = dt_fr
         self.dt_to = dt_to
-        self.source_file_paths = []
-        self.done = False
-
-        if data_file_root_dir is None:
-            self.data_file_root_dir = prf.datahub_data_root_dir
+        self.file_paths_source = []
+        self.file_paths_local = []
+        self.done = None
+        self.file_paths_local = []
+        self.file_paths_remote = []
+
+        if root_dir_local is None:
+            self.root_dir_local = prf.datahub_data_root_dir
         else:
-            self.data_file_root_dir = data_file_root_dir
+            self.root_dir_local = root_dir_local
 
         if direct_download:
             self.download()
+            
+    def download(self, *args, **kwargs):
+        pass
+    
+    def _to_download(self, file_path, with_suffix=None):
+        
+        to_download = True
+        
+        if with_suffix is not None:
+            is_file = file_path.with_suffix(with_suffix).is_file()
+        else:
+            is_file = file_path.is_file()
+        if is_file:
+            if self.force_download:
+                mylog.simpleinfo.info(
+                    "The file {} exists in the directory {}: Forced redownloading the file ...".format(
+                        file_path.name, file_path.parent.resolve()
+                    )
+                )
+            else:
+                mylog.simpleinfo.info(
+                    "The file {} exists in the directory {}: Skipped downloading.".format(
+                        file_path.name, file_path.parent.resolve()
+                    )
+                )
+                to_download = False
+        return to_download
+    
+    def _search_files(self, file_list, file_name_patterns):
+        search_pattern = '.*' + '.*'.join(file_name_patterns) + '.*'
+        fn_regex = re.compile(search_pattern)
+        file_names = list(filter(fn_regex.match, file_list))
+        return file_names
+    
+
+
+class DownloaderFromFTPBase(DownloaderBase):
+    """
+    Base downloader for downloading data files from FTP server.
+
+    Parameters
+    ----------
+    ftp_host : str
+        The FTP host address.
+    ftp_port : int
+        The FTP port [21].
+    root_dir_remote : str
+        The directory in the FTP that stores the data.
+    """
+    
+    root_dir_remote = None
 
-    def download(self,):
-        if self.from_ftp:
-            raise NotImplementedError
+    def __init__(
+        self,
+        dt_fr, dt_to,
+        ftp_host=None, ftp_port=21,
+        username='anonymous', password='', 
+        root_dir_local=None,
+        root_dir_remote=None,
+        direct_download=False, force_download=False, dry_run=False,
+        **kwargs
+        ):
+        
+        self.ftp_host = ftp_host
+        self.ftp_port = ftp_port
+        self.username = username
+        self.password = password
+        if root_dir_remote is not None:
+            self.root_dir_remote = root_dir_remote
         else:
-            self.download_from_http()
+            self.root_dir_remote = '/'
+        
+        super(DownloaderFromFTPBase, self).__init__(
+            dt_fr, dt_to,
+            root_dir_local=root_dir_local, 
+            direct_download=direct_download,
+            force_download=force_download,
+            dry_run=dry_run,
+            **kwargs
+        )
+        
+    def download(
+        self, 
+        with_TLS=False, 
+        subdirs: None | list=None, 
+        file_name_patterns: None | list=None,
+        **kwargs
+        ):
+        
+        timeout = kwargs.pop('timeout', 30) # seconds
+        encoding = kwargs.pop('encoding', 'utf-8')
+        
+        if with_TLS:
+            FTP_CLASS = ftplib.FTP_TLS
+        else:
+            FTP_CLASS = ftplib.FTP
             
-    def search_from_http(self, subdirs=None, file_name_patterns=None):
-        url = self.url_base + '/'.join(subdirs)
-        r = requests.get(url)
+        with FTP_CLASS(encoding=encoding) as ftp:
+            ftp.connect(self.ftp_host, self.ftp_port, timeout)
+            ftp.login(user=self.username, passwd=self.password)
+            ftp.cwd(self.root_dir_remote)
+            
+            self.file_paths_remote = self.search_from_ftp(
+                ftp, subdirs=subdirs, file_name_patterns=file_name_patterns
+            )
+            self.save_files_from_ftp(ftp)
+            
+    def search_from_ftp(self, ftp, subdirs: None | list=None, file_name_patterns: None | list=None):
+        if subdirs is not None:
+            for subdir in subdirs:
+                ftp.cwd(subdir)
+        try:
+            files = ftp.nlst()
+        except ftplib.error_perm as resp:
+            if str(resp) == "550 No files found":
+                mylog.StreamLogger.warning("No files in this directory.")
+            else:
+                raise
+
+        file_names = self._search_files(
+            file_list=files, file_name_patterns=file_name_patterns
+        )        
+        
+        file_paths_remote = []
+        for fn in file_names:
+            file_paths_remote.append(ftp.pwd() + '/' + fn)
+        return file_paths_remote
+    
+    def save_files_from_ftp(self, ftp, file_paths_local=None, root_dir_remote=None):
+        if root_dir_remote is None:
+            root_dir_remote = self.root_dir_remote
+            
+        if file_paths_local is None:
+            self.file_paths_local = []
+            for fp_remote in self.file_paths_remote:
+                self.file_paths_local.append(
+                    self.root_dir_local / fp_remote.replace(root_dir_remote, '')
+                )
+        self.done = [False] * len(self.file_paths_remote)
+        
+        if self.dry_run:
+            for i, (fp_remote, fp_local) in enumerate(zip(self.file_paths_remote, self.file_paths_local)):
+                mylog.simpleinfo.info(
+                    f"Dry run: Downloading the file {fp_remote} to {fp_local} ..."
+                )
+            return
+        
+        for i, (fp_remote, fp_local) in enumerate(zip(self.file_paths_remote, self.file_paths_local)):
+            to_download = self._to_download(fp_local)
+            
+            if not to_download:
+                self.done[i] = True
+                continue
+            mylog.simpleinfo.info(
+                f"Downloading the file: {fp_remote} ..."
+            )
+            
+            file_dir_remote = fp_remote.rsplit('/', 1)[0]
+            file_name_remote = fp_remote.split('/')[-1]
+            
+            file_dir_local = fp_local.parent.resolve()
+            file_dir_local.mkdir(exist_ok=True, parents=True)
+                
+            with open(fp_local, 'w+b') as f:
+                if file_dir_remote != ftp.pwd():
+                    ftp.cwd(file_dir_remote)
+                
+                bufsize=1024
+                total=ftp.size(fp_remote)
+                pbar=tqdm(total=total)
+                def bar(data):
+                    f.write(data)
+                    pbar.update(len(data))
+                res = ftp.retrbinary('RETR '+ file_name_remote, bar, bufsize)
+                pbar.close()
+                # res = ftp.retrbinary('RETR ' + file_name_remote, f.write)
+                # mylog.simpleinfo.info(res)
+                if not res.startswith('226'):
+                    mylog.StreamLogger.warning('The downloading is not compiled...: {}'.format(res))
+                    fp_local.unlink(missing_ok=True)
+                self.done[i] = True
+                mylog.simpleinfo.info("Saved as {}.".format(fp_local))
+            if not self.done[i]:
+                mylog.StreamLogger.error("Error in downloading the file {}.".format(file_name_remote))
+        return
+
+
+class DownloaderFromHTTPBase(DownloaderBase):
+    """
+    Base downloader for downloading data files from HTTP server.
+
+    Parameters
+    ----------
+    base_url : str
+        The base URL of the HTTP server.
+    root_dir_remote : str
+        The directory in the HTTP server that stores the data.
+    """
+    
+    root_dir_remote = None
 
+    def __init__(
+        self,
+        dt_fr, dt_to,
+        base_url=None,
+        root_dir_local=None,
+        root_dir_remote=None,
+        direct_download=False, force_download=False, dry_run=False,
+        **kwargs
+        ):
+        
+        self.base_url = base_url
+        
+        if root_dir_remote is not None:    
+            self.root_dir_remote = root_dir_remote
+        
+        super(DownloaderFromHTTPBase, self).__init__(
+            dt_fr, dt_to,
+            root_dir_local=root_dir_local,
+            direct_download=direct_download,
+            force_download=force_download,
+            dry_run=dry_run,
+            **kwargs
+        )
+        
+    def download(
+        self, 
+        subdirs: None | list=None, 
+        file_name_patterns: None | list=None,
+        **kwargs
+        ):
+        
+        self.file_paths_remote = self.search_from_http(
+            subdirs=subdirs, file_name_patterns=file_name_patterns
+        )
+        self.save_files_from_http()
+        
+        
+    def search_from_http(self, subdirs: None | list=None, file_name_patterns: None | list=None, **kwargs):
+        url_patterns = [self.base_url]
+        if str(self.root_dir_remote):
+            url_patterns.append(self.root_dir_remote)
+        if subdirs is not None:
+            url_patterns.extend(subdirs)
+        url = '/'.join(url_patterns)
+        r = requests.get(url)
         soup = bs4.BeautifulSoup(r.text, 'html.parser')
         a_tags = soup.find_all('a', href=True)
         hrefs = [a_tag['href'] for a_tag in a_tags]
+        
+        hrefs = self._search_files(file_list=hrefs, file_name_patterns=file_name_patterns)
 
-        search_pattern = '.*' + '.*'.join(file_name_patterns) + '.*'
-        fn_regex = re.compile(search_pattern)
-        hrefs = list(filter(fn_regex.match, hrefs))
-
-        paths = []
+        file_paths_remote = []
         for href in hrefs:
-            paths.append(url + '/' + href)
-        return paths
-
-    def download_from_http(self, ):
-
-        source_file_paths = self.search_from_http()
-
-        for url  in source_file_paths:
-            if self.dry_run:
-                print(f"Dry run: {url}.")
-            else:
-                self.save_file_from_http(url=url)
-
-    def save_file_from_http(self, url, file_dir=None, file_name=None):
-            if file_name is None:
-                file_name = url.split('/')[-1]
-            file_path = file_dir / file_name
-            if file_path.is_file():
+            file_paths_remote.append(url + '/' + href)
+        r.close()
+        return file_paths_remote
+
+    def save_files_from_http(self, file_paths_local=None, root_dir_remote=None):
+        if root_dir_remote is None:
+            root_dir_remote = self.root_dir_remote
+        if str(root_dir_remote):
+            root_url = self.base_url + '/' + root_dir_remote
+        else:
+            root_url = self.base_url
+            
+        if file_paths_local is None:
+            self.file_paths_local = []
+            for fp_remote in self.file_paths_remote:
+                self.file_paths_local.append(
+                    self.root_dir_local / fp_remote.replace(root_url, '')
+                )
+        else:
+            self.file_paths_local = file_paths_local
+        self.done = [False] * len(self.file_paths_remote)
+        
+        if self.dry_run:
+            for i, (fp_remote, fp_local) in enumerate(zip(self.file_paths_remote, self.file_paths_local)):
                 mylog.simpleinfo.info(
-                    "The file {} exists in the directory {}.".format(file_path.name, file_path.parent.resolve()))
-                if not self.force_download:
-                    self.done = True
-                    return
-
-            file_path.parent.resolve().mkdir(parents=True, exist_ok=True)
-            mylog.simpleinfo.info(f'Downloading {file_name} ...')
-            res = self._download_by_requests_get(url, data_file_path=file_path)
+                    f"Dry run: Downloading the file {fp_remote} to {fp_local} ..."
+                )
+            return
+        
+        for i, (fp_remote, fp_local) in enumerate(zip(self.file_paths_remote, self.file_paths_local)):
+            to_download = self._to_download(fp_local)
+            
+            if not to_download:
+                self.done[i] = True
+                continue
+            
+            mylog.simpleinfo.info(
+                f"Downloading the file: {fp_remote} ..."
+            )
+
+            res = self._download_by_requests_get(fp_remote, fp_local)
             if res:
-                mylog.simpleinfo.info(f'Saved in {file_dir}')
-                self.done = True
-            else:
-                mylog.StreamLogger.error(f"Error during downloading. Code: {res}.")
+                mylog.simpleinfo.info(f"Saved as {fp_local}.")
+                self.done[i] = True
 
-            return
+        return
 
     @staticmethod
-    def _download_by_requests_get(url, data_file_path=None, params=None, stream=True, allow_redirects=True, **kwargs):
-        r = requests.get(url, params=params, stream=stream, allow_redirects=allow_redirects, **kwargs)
-        if r.status_code != 200:
-            return -2
-
-        total_size_in_bytes = int(r.headers.get('content-length', 0))
-        block_size = 1024  # 1 Kibibyte
-        progress_bar = tqdm.tqdm(total=total_size_in_bytes, unit='iB', unit_scale=True)
-
-        with open(data_file_path, 'wb') as file:
-            for data in r.iter_content(block_size):
-                progress_bar.update(len(data))
-                file.write(data)
-        progress_bar.close()
-        if total_size_in_bytes != 0 and progress_bar.n != total_size_in_bytes:
-            mylog.StreamLogger.error("Something wrong during the download!")
-            data_file_path.unlink(missing_ok=True)
+    def _download_by_requests_get(
+        url, file_path_local, 
+        params=None, stream=True, allow_redirects=True, 
+        file_block_size=1024, 
+        **kwargs
+        ):
+        try: 
+            r = requests.get(url, params=params, stream=stream, allow_redirects=allow_redirects, **kwargs)
+            if r.status_code != 200:
+                return -2
+
+            total_size_in_bytes = int(r.headers.get('content-length', 0))
+
+            progress_bar = tqdm.tqdm(total=total_size_in_bytes, unit='iB', unit_scale=True)
+
+            file_path_local.parent.mkdir(parents=True, exist_ok=True)
+            with open(file_path_local, 'wb') as file:
+                for data in r.iter_content(file_block_size):
+                    progress_bar.update(len(data))
+                    file.write(data)
+            progress_bar.close()
+            if total_size_in_bytes != 0 and progress_bar.n != total_size_in_bytes:
+                mylog.StreamLogger.error("Something wrong during the download!")
+                file_path_local.unlink(missing_ok=True)
+                return -1
+            r.close()
+        except Exception as e:
+            mylog.StreamLogger.error(f"Exception during downloading the file from {url}: {e}")
             return -1
-
         return 1
 
-    def search_from_ftp(self, ftp, file_name_patterns=None):
-        paths = []
-
-        return paths
 
-    def save_file(self, file_dir, file_name, r_source_file):
-        file_path = file_dir / file_name
-
-        if file_path.is_file():
-            mylog.simpleinfo.info(
-                "The file {} exists in the directory {}.".format(file_path.name, file_path.parent.resolve()))
+class CDAWebHTTPDownloader(DownloaderFromHTTPBase):
+    """
+    Downloader for downloading data files from CDAWeb HTTP server.
 
-        if self.force_download:
-            file_path.parent.resolve().mkdir(parents=True, exist_ok=True)
-            with open(file_path, "wb") as f:
-                mylog.simpleinfo.info(
-                    "Downloading {} to the directory {} ...".format(file_path.name, file_path.parent.resolve())
-                )
-                f.write(r_source_file.content)
-                mylog.simpleinfo.info("Done")
+    Parameters
+    ----------
+    base_url : str
+        The base URL of the CDAWeb HTTP server.
+    root_dir_remote : str
+        The directory in the CDAWeb HTTP server that stores the data.
+    """
 
+    root_dir_remote = 'pub/data'
+    
+    def __init__(
+        self,
+        dt_fr, dt_to,
+        root_dir_local=None,
+        root_dir_remote=None,
+        direct_download=False, force_download=False, dry_run=False,
+        **kwargs
+        ):
+        
+        base_url = 'https://cdaweb.gsfc.nasa.gov'
+        
+        if root_dir_local is None:
+            root_dir_local = prf.datahub_data_root_dir / 'CDAWeb'
+        
+        super().__init__(
+            dt_fr, dt_to,
+            base_url=base_url,
+            root_dir_local=root_dir_local,
+            root_dir_remote=root_dir_remote,
+            direct_download=direct_download,
+            force_download=force_download,
+            dry_run=dry_run,
+            **kwargs
+        )

geospacelab/datahub/sources/cdaweb/downloader_backup.py

@@ -0,0 +1,93 @@
+import datetime
+import pathlib
+import copy
+
+import geospacelab.toolbox.utilities.pydatetime as dttool
+import geospacelab.toolbox.utilities.pylogging as mylog
+from geospacelab.config import prf
+from geospacelab.datahub.sources.cdaweb.downloader import Downloader as DownloaderBase
+
+
+class Downloader(DownloaderBase):
+    
+    def __init__(
+            self,
+            dt_fr=None, dt_to=None,
+            sat_id=None,
+            orbit_id=None,
+            direct_download=True,
+            force_download=False,
+            data_file_root_dir = None,
+            dry_run=False,
+        ):
+        product = 'EDR_AUR'
+        if data_file_root_dir is None:
+            data_file_root_dir = prf.datahub_data_root_dir / 'CDAWeb' / 'DMSP' / 'SSUSI' / product
+        self.sat_id = sat_id
+        self.orbit_id = orbit_id
+        self.source_subdirs = ['dmsp', 'dmsp'+self.sat_id.lower(), 'ssusi', 'data', 'edr-aurora']
+
+        super().__init__(
+            dt_fr, dt_to,
+            data_file_root_dir=data_file_root_dir,
+            direct_download=direct_download,force_download=force_download,dry_run=dry_run
+        )
+
+
+    def search_from_http(self, file_name_patterns=None, allow_multiple_files=True):
+
+        dt_fr_1 = self.dt_fr - datetime.timedelta(hours=3)
+        dt_to_1 = self.dt_to + datetime.timedelta(hours=3)
+        diff_days = dttool.get_diff_days(dt_fr_1, dt_to_1)
+        dt0 = dttool.get_start_of_the_day(dt_fr_1)
+        source_file_paths = []
+        for nd in range(diff_days + 1):
+            this_day = dt0 + datetime.timedelta(days=nd)
+            doy = dttool.get_doy(this_day)
+            sdoy = '{:03d}'.format(doy)
+            subdirs = copy.deepcopy(self.source_subdirs)
+            subdirs.extend(
+                [str(this_day.year), sdoy]
+            )
+
+            if self.orbit_id is None:
+                file_name_patterns = [
+                    'dmsp' + self.sat_id.lower(),
+                    'ssusi',
+                    'edr-aurora',
+                    this_day.strftime("%Y") + sdoy + 'T',
+                    '.nc'
+                ]
+            else:
+                file_name_patterns = [
+                    'dmsp' + self.sat_id.lower(),
+                    'ssusi',
+                    'edr-aurora',
+                    'REV',
+                    self.orbit_id,
+                    '.nc'
+                ]
+            paths = super().search_from_http(subdirs=subdirs, file_name_patterns=file_name_patterns)
+            source_file_paths.extend(paths)
+        return source_file_paths
+
+    def save_file_from_http(self, url, file_dir=None, file_name=None):
+
+        sy = url.split('/')[-3]
+        sdoy = url.split('/')[-2]
+        year = int(sy)
+        this_day = dttool.convert_doy_to_datetime(year, int(sdoy))
+        if file_dir is None:
+            file_dir = self.data_file_root_dir /  self.sat_id.upper() / sy / this_day.strftime("%Y%m%d")
+        super().save_file_from_http(url, file_dir=file_dir)
+
+
+
+if __name__ == "__main__":
+    downloader = Downloader(
+        dt_fr = datetime.datetime(2011, 1, 6),
+        dt_to = datetime.datetime(2011, 1, 6, 12),
+        sat_id='F17',
+        orbit_id='21523',
+        dry_run=False,
+    )