weatherdb 1.1.1__tar.gz → 1.1.2__tar.gz

{weatherdb-1.1.1 → weatherdb-1.1.2}/.gitlab-ci.yml

@@ -65,6 +65,7 @@ test_single:
     WEATHERDB_TEST_ARTIFACT_DB_DUMP: ON_FAILURE
     WEATHERDB_TEST_ARTIFACT_LIST_DATA_FILES: ON_FAILURE
     WEATHERDB_TEST_ARTIFACT_COPY_USER_CONFIG: ON_FAILURE
+    WEATHERDB_OPENTOPO_API_KEY: $WEATHERDB_OPENTOPO_API_KEY
     DOCKER_ENV: test
   tags:
     - docker

{weatherdb-1.1.1 → weatherdb-1.1.2}/CHANGES.md

@@ -1,8 +1,14 @@
 # Change-log
 
-## Version 1.0.1
+## Version 1.1.2
+
+- add method to download DEM data from OpenTopography-API
+- update StationBase._expand_timeserie_to_period to remove entries that are older than the min_date config to remove unnessecary database entries
+- fix GroupStation.create_ts: don't throw period changed warning if the change was only due to the expand_to_timestamp output
+
+## Version 1.1.1
 
- - minor fixes to fix previous version
+- minor fixes to fix previous version
 
 ## Version 1.1.0
 

{weatherdb-1.1.1 → weatherdb-1.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: weatherdb
-Version: 1.1.1
+Version: 1.1.2
 Summary: This is a package to work with and to create the Weather Database which handles, checks, fills and corrects DWD Weather Station data.
 Author-email: Max Schmit <max.schmit@hydrology.uni-freiburg.de>
 License:                     GNU GENERAL PUBLIC LICENSE

weatherdb-1.1.2/weatherdb/_version.py

@@ -0,0 +1 @@
+__version__ = "1.1.2"

{weatherdb-1.1.1 → weatherdb-1.1.2}/weatherdb/cli.py

@@ -103,10 +103,16 @@ def download_ma_rasters(which, overwrite, update_user_config):
     """
     click.echo("starting downloading multi annual raster data")
     from weatherdb.utils.get_data import download_ma_rasters
-    download_ma_rasters(overwrite=overwrite)
+    download_ma_rasters(
+        which=which,
+        overwrite=overwrite,
+        update_user_config=update_user_config)
 
 
 @cli.command(short_help="Download the needed digital elevation model raster data from Copernicus to the data folder.")
+@click.option('--out-dir', '-d',
+              type=click.Path(), default=None, show_default=False,
+              help="The directory to save the downloaded DEM data to.")
 @click.option('--overwrite/--no-overwrite', '-o/-no-o',
               type=bool, is_flag=True, default=None, show_default=False,
               help="Should the digital elevation model raster be downloaded even if it already exists?")
@@ -116,7 +122,11 @@ def download_ma_rasters(which, overwrite, update_user_config):
 @click.option("--update-user-config", "-u",
               type=bool, default=False, show_default=True, is_flag=True,
               help="Should the user configuration be updated with the path to the downloaded DEM?")
-def download_dem(overwrite, extent):
+@click.option("--service", "-s",
+              type=str, default=["prism", "openTopography"], show_default=True, multiple=True,
+              help="The service to use to download the DEM. Options are 'prism' or 'openTopography'. " +\
+                   "You can use this option muultiple times to test both in the given order until the file could be downloaded.")
+def download_dem(out_dir, overwrite, extent, update_user_config, service="prism"):
     """Download the newest DEM data from the Copernicus Sentinel dataset.
 
     Only the GLO-30 DEM, wich has a 30m resolution, is downloaded as it is freely available.
@@ -131,7 +141,12 @@ def download_dem(overwrite, extent):
     """
     click.echo("Starting downloading digital elevation model from Copernicus")
     from weatherdb.utils.get_data import download_dem
-    download_dem(overwrite=overwrite, extent=extent)
+    download_dem(
+        out_dir=out_dir,
+        overwrite=overwrite,
+        extent=extent,
+        service=service,
+        update_user_config=update_user_config)
 
 
 # cli statements to update the database

{weatherdb-1.1.1 → weatherdb-1.1.2}/weatherdb/station/GroupStation.py

@@ -526,7 +526,9 @@ class GroupStation(object):
                 period_new = period_filled.union(
                     period,
                     how="inner")
-                if period_new != period:
+                if period_new != period and not (
+                        (period.start == period_new.start) and
+                        ((period.end - period_new.end) <= pd.Timedelta(days=1))):
                     warnings.warn(
                         f"The Period for Station {self.id} got changed from {str(period)} to {str(period_new)}.")
                     period = period_new

{weatherdb-1.1.1 → weatherdb-1.1.2}/weatherdb/station/StationBases.py

@@ -504,10 +504,12 @@ class StationBase:
                     '{interval}'::INTERVAL)::{tstp_dtype} AS timestamp)
             INSERT INTO timeseries."{stid}_{para}"(timestamp)
                 (SELECT wts.timestamp
-                FROM whole_ts wts
-                LEFT JOIN timeseries."{stid}_{para}" ts
+                 FROM whole_ts wts
+                 LEFT JOIN timeseries."{stid}_{para}" ts
                     ON ts.timestamp=wts.timestamp
-                WHERE ts.timestamp IS NULL);
+                 WHERE ts.timestamp IS NULL);
+            DELETE FROM timeseries."{stid}_{para}"
+            WHERE timestamp < '{min_date} 00:00'::{tstp_dtype};
         """.format(
             stid=self.id,
             para=self._para,

{weatherdb-1.1.1 → weatherdb-1.1.2}/weatherdb/utils/get_data.py

@@ -6,9 +6,19 @@ from pathlib import Path
 from distutils.util import strtobool
 import hashlib
 import progressbar as pb
+import keyring
+import os
+import getpass
+import logging
 
 from ..config import config
 
+__all__ = ["download_ma_rasters", "download_dem"]
+
+log = logging.getLogger(__name__)
+
+# Multi annual rasters
+# --------------------
 def download_ma_rasters(which="all", overwrite=None, update_user_config=False):
     """Get the multi annual rasters on which bases the regionalisation is done.
 
@@ -60,7 +70,7 @@ def download_ma_rasters(which="all", overwrite=None, update_user_config=False):
         if file_key in which:
             # check if file is in config
             if f"data:rasters:{file_key}" not in config:
-                print(f"Skipping {file_key} as it is not in your configuration.\nPlease add a section 'data:rasters:{file_key}' to your configuration file.")
+                log.debug(f"Skipping {file_key} as it is not in your configuration.\nPlease add a section 'data:rasters:{file_key}' to your configuration file.")
                 continue
 
             # check if file already exists
@@ -75,7 +85,7 @@ def download_ma_rasters(which="all", overwrite=None, update_user_config=False):
                         "Do you want to overwrite it? [y/n] "))
 
                 if skip:
-                    print(f"Skipping {file_key} as overwriting is not allowed.")
+                    log.debug(f"Skipping {file_key} as overwriting is not allowed.")
                     continue
 
             # check if the directory exists
@@ -126,24 +136,42 @@ def download_ma_rasters(which="all", overwrite=None, update_user_config=False):
                 if config.has_user_config:
                     config.update_user_config(f"data:rasters:{file_key}", "file", str(file_path))
                 else:
-                    print(f"No user configuration file found, therefor the raster '{file_key}' is not set in the user configuration file.")
+                    log.error(f"No user configuration file found, therefor the raster '{file_key}' is not set in the user configuration file.")
 
+# DEM data
+# --------
+def _check_write_fp(fp, overwrite):
+    """Check if a file exists and if it should be overwritten.
 
-def download_dem(overwrite=None, extent=(5.3, 46.1, 15.6, 55.4), update_user_config=False):
-    """Download the newest DEM data from the Copernicus Sentinel dataset.
-
-    Only the GLO-30 DEM, which has a 30m resolution, is downloaded as it is freely available.
-    If you register as a scientific researcher also the EEA-10, with 10 m resolution, is available.
-    You will have to download the data yourself and define it in the configuration file.
-
-    After downloading the data, the files are merged and saved as a single tif file in the data directory in a subfolder called 'DEM'.
-    To use the DEM data in the WeatherDB, you will have to define the path to the tif file in the configuration file.
+    Parameters
+    ----------
+    fp : str or Path
+        The path to the file to check.
+    overwrite : bool
+        Should the file be overwritten?
+
+    Returns
+    -------
+    bool
+        Should the file be written?
+    """
+    fp = Path(fp)
+    if fp.exists():
+        log.info(f"The file already exists at {fp}.")
+        if overwrite is None:
+            overwrite = strtobool(input(f"{fp} already exists. Do you want to overwrite it? [y/n] "))
+        if not overwrite:
+            log.info("Skipping, because overwritting was turned of.")
+            return False
+    return True
 
-    Source:
-    Copernicus DEM - Global and European Digital Elevation Model. Digital Surface Model (DSM) provided in 3 different resolutions (90m, 30m, 10m) with varying geographical extent (EEA: European and GLO: global) and varying format (INSPIRE, DGED, DTED). DOI:10.5270/ESA-c5d3d65.
+def _download_dem_prism(out_dir, overwrite=None, extent=(5.3, 46.1, 15.6, 55.4)):
+    """Download the DEM data from the Copernicus PRISM service.
 
     Parameters
     ----------
+    out_dir: str or Path
+        The directory to save the DEM data to.
     overwrite : bool, optional
         Should the DEM data be downloaded even if it already exists?
         If None the user will be asked.
@@ -151,9 +179,11 @@ def download_dem(overwrite=None, extent=(5.3, 46.1, 15.6, 55.4), update_user_con
     extent : tuple, optional
         The extent in WGS84 of the DEM data to download.
         The default is the boundary of germany + ~40km = (5.3, 46.1, 15.6, 55.4).
-    update_user_config : bool, optional
-        Should the downloaded DEM be set as the used DEM in the user configuration file?
-        The default is False.
+
+    Returns
+    -------
+    fp : Path
+        The path to the downloaded DEM file.
     """
     # import necessary modules
     import rasterio as rio
@@ -164,10 +194,8 @@ def download_dem(overwrite=None, extent=(5.3, 46.1, 15.6, 55.4), update_user_con
     import re
     import json
 
-    # get dem_dir
-    base_dir = Path(config.get("data", "base_dir"))
-    dem_dir = base_dir / "DEM"
-    dem_dir.mkdir(parents=True, exist_ok=True)
+    # check dir
+    out_dir = Path(out_dir)
 
     # get available datasets
     prism_url = "https://prism-dem-open.copernicus.eu/pd-desk-open-access/publicDemURLs"
@@ -191,20 +219,12 @@ def download_dem(overwrite=None, extent=(5.3, 46.1, 15.6, 55.4), update_user_con
         )[-1]["id"]
 
     # check if dataset already exists
-    dem_file = dem_dir / f'{ds_id.replace("/", "__")}.tif'
-    if dem_file.exists():
-        print(f"The DEM data already exists at {dem_file}.")
-        if overwrite is None:
-            overwrite = strtobool(input("Do you want to overwrite it? [y/n] "))
-        if not overwrite:
-            print("Skipping, because overwritting was turned of.")
-            return
-        else:
-            print("Overwriting the dataset.")
-    dem_dir.mkdir(exist_ok=True)
+    dem_file = out_dir / f'{ds_id.replace("/", "__")}.tif'
+    if not _check_write_fp(dem_file, overwrite):
+        return
 
     # selecting DEM tiles
-    print(f"getting available tiles for Copernicus dataset '{ds_id}'")
+    log.info(f"getting available tiles for Copernicus dataset '{ds_id}'")
     ds_files_req = json.loads(
         requests.get(
             f"{prism_url}/{ds_id.replace('/', '__')}",
@@ -225,13 +245,13 @@ def download_dem(overwrite=None, extent=(5.3, 46.1, 15.6, 55.4), update_user_con
         ds_files_all))
 
     # download DEM tiles
-    print("downloading tiles")
+    log.info("downloading tiles")
     with TemporaryDirectory() as tmp_dir:
         tmp_dir_fp = Path(tmp_dir)
         for f in pb.progressbar(ds_files):
             with open(tmp_dir_fp / Path(f["nativeDemUrl"]).name, "wb") as d:
                 d.write(requests.get(f["nativeDemUrl"]).content)
-        print("downloaded all files")
+        log.info("downloaded all files")
 
         # extracting tifs from tars
         for i, f in pb.progressbar(list(enumerate(tmp_dir_fp.glob("*.tar")))):
@@ -269,17 +289,161 @@ def download_dem(overwrite=None, extent=(5.3, 46.1, 15.6, 55.4), update_user_con
         tmp_eula_fp = next(tmp_dir_fp.glob("*.pdf"))
         shutil.copyfile(
             tmp_eula_fp,
-            dem_dir / tmp_eula_fp.name
+            out_dir / tmp_eula_fp.name
         )
 
-    print(f"created DEM at '{dem_file}'.")
+    log.info(f"created DEM at '{dem_file}'.")
+    return dem_file
+
+def _download_dem_opentopo(
+        out_dir,
+        overwrite=None,
+        extent=(5.3, 46.1, 15.6, 55.4),
+        api_key=os.environ.get(
+            "WEATHERDB_OPENTOPO_API_KEY",
+            fallback=keyring.get_password("weatherdb", "opentopo_api_key"))):
+    """Download the DEM data from the OpenTopography service.
+
+    Get an API key from (OpenTopography)[https://portal.opentopography.org/] to use this service.
+
+    Parameters
+    ----------
+    out_dir : str or Path
+        The directory to save the DEM data to.
+    overwrite : bool, optional
+        Should the DEM data be downloaded even if it already exists?
+        If None the user will be asked.
+        The default is None.
+    extent : tuple, optional
+        The extent in WGS84 of the DEM data to download.
+        Should be in the format (south, west, north, east).
+        The default is the boundary of germany + ~40km = (5.3, 46.1, 15.6, 55.4).
+    api_key : str, optional
+        The API key for the OpenTopography service.
+        If None the user will be asked.
+        The default is to check if the environment variable "WEATHERDB_OPENTOPO_API_KEY" is set or if the keyring has a password for "weatherdb" and "opentopo_api_key".
+
+    Returns
+    -------
+    fp : Path
+        The path to the downloaded DEM file.
+    """
+    # check api key
+    if api_key is None:
+        print("No API key for OpenTopography was given or found in the keyring or environment variable.")
+        api_key = getpass("Please enter your API key for OpenTopography: ")
+
+    # make query
+    w, s, e, n = extent
+    url = f"https://portal.opentopography.org/API/globaldem?demtype=COP30&west={w}&south={s}&east={e}&north={n}&outputFormat=GTiff&API_Key={api_key}"
+    r = requests.get(url)
+    if r.status_code == 200:
+        # store api key
+        if api_key != keyring.get_password("weatherdb", "opentopo_api_key"):
+            keyring.set_password("weatherdb", "opentopo_api_key", api_key)
+
+        # get dem file
+        out_fp = out_dir / "OpenTopo_COP30.tif"
+
+        # check if file already exists
+        if not _check_write_fp(out_fp, overwrite):
+            return
+
+        # download file
+        with open(out_fp, "wb") as f:
+            for chunk in r.iter_content(chunk_size=1024):
+                f.write(chunk)
+            log.info(f"Downloaded DEM data from OpenTopography to '{out_fp}'.")
+            return out_fp
+
+    log.error(f"Request to openTopography API with url {r.url.replace(api_key, "[MASKED]")} returned status code {r.status_code}")
+
+def download_dem(out_dir=None,
+                 overwrite=None,
+                 extent=(5.3, 46.1, 15.6, 55.4),
+                 update_user_config=False,
+                 service=["prism", "openTopography"], **kwargs):
+    """Download the newest DEM data from the Copernicus Sentinel dataset.
+
+    Only the GLO-30 DEM, which has a 30m resolution, is downloaded as it is freely available.
+    If you register as a scientific researcher also the EEA-10, with 10 m resolution, is available.
+    You will have to download the data yourself and define it in the configuration file.
+
+    After downloading the data, the files are merged and saved as a single tif file in the data directory in a subfolder called 'DEM'.
+    To use the DEM data in the WeatherDB, you will have to define the path to the tif file in the configuration file.
+
+    Source:
+    Copernicus DEM - Global and European Digital Elevation Model. Digital Surface Model (DSM) provided in 3 different resolutions (90m, 30m, 10m) with varying geographical extent (EEA: European and GLO: global) and varying format (INSPIRE, DGED, DTED). DOI:10.5270/ESA-c5d3d65.
+
+    Parameters
+    ----------
+    out_dir : str or Path, optional
+        The directory to save the DEM data to.
+        If None the data is saved in the data directory in a subfolder called 'DEM'.
+        The default is None.
+    overwrite : bool, optional
+        Should the DEM data be downloaded even if it already exists?
+        If None the user will be asked.
+        The default is None.
+    extent : tuple, optional
+        The extent in WGS84 of the DEM data to download.
+        The default is the boundary of germany + ~40km = (5.3, 46.1, 15.6, 55.4).
+    update_user_config : bool, optional
+        Should the downloaded DEM be set as the used DEM in the user configuration file?
+        The default is False.
+    service : str or list of str, optional
+        The service to use to download the DEM data.
+        Options are "prism" and "openTopography".
+        If Both are given they are executed in the order they are given.
+        If OpenTopography is selected, you will have to provide an API key.
+        The default is ["prism", "openTopography"].
+    """
+    # check service
+    if isinstance(service, str):
+        service = [service]
+
+    # check dir
+    if out_dir is None:
+        out_dir = Path(config.get("data", "base_dir")) / "DEM"
+    else:
+        out_dir = Path(out_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    # download data
+    fp=None
+    for s in service:
+        if s == "prism":
+            try:
+                fp = _download_dem_prism(
+                    out_dir=out_dir,
+                    overwrite=overwrite,
+                    extent=extent,
+                    **kwargs)
+                break
+            except Exception as e:
+                log.debug(f"Error while downloading DEM from PRISM: {e}")
+        elif s == "openTopography":
+            try:
+                fp = _download_dem_opentopo(
+                    out_dir=out_dir,
+                    overwrite=overwrite,
+                    extent=extent,
+                    **kwargs)
+                break
+            except Exception as e:
+                log.debug(f"Error while downloading DEM from OpenTopography: {e}")
+
+    # check if file was downloaded
+    if fp is None:
+        log.error("No DEM data was downloaded.")
+        return
 
     # update user config
     if update_user_config:
         if config.has_user_config:
-            config.update_user_config("data:rasters", "dems", str(dem_file))
+            config.update_user_config("data:rasters", "dems", str(fp))
             return
         else:
-            print("No user configuration file found, therefor the DEM is not set in the user configuration file.")
+            log.info("No user configuration file found, therefor the DEM is not set in the user configuration file.")
 
-    print("To use the DEM data in the WeatherDB, you will have to define the path to the tif file in the user configuration file.")
+    log.info("To use the DEM data in the WeatherDB, you will have to define the path to the tif file in the user configuration file.")

{weatherdb-1.1.1 → weatherdb-1.1.2}/weatherdb.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: weatherdb
-Version: 1.1.1
+Version: 1.1.2
 Summary: This is a package to work with and to create the Weather Database which handles, checks, fills and corrects DWD Weather Station data.
 Author-email: Max Schmit <max.schmit@hydrology.uni-freiburg.de>
 License:                     GNU GENERAL PUBLIC LICENSE

weatherdb-1.1.1/weatherdb/_version.py

@@ -1 +0,0 @@
-__version__ = "1.1.1"