weatherdb 1.1.0__tar.gz → 1.1.2__tar.gz

{weatherdb-1.1.0 → weatherdb-1.1.2}/.gitlab-ci.yml

@@ -65,6 +65,7 @@ test_single:
     WEATHERDB_TEST_ARTIFACT_DB_DUMP: ON_FAILURE
     WEATHERDB_TEST_ARTIFACT_LIST_DATA_FILES: ON_FAILURE
     WEATHERDB_TEST_ARTIFACT_COPY_USER_CONFIG: ON_FAILURE
+    WEATHERDB_OPENTOPO_API_KEY: $WEATHERDB_OPENTOPO_API_KEY
     DOCKER_ENV: test
   tags:
     - docker

{weatherdb-1.1.0 → weatherdb-1.1.2}/CHANGES.md

@@ -1,5 +1,15 @@
 # Change-log
 
+## Version 1.1.2
+
+- add method to download DEM data from OpenTopography-API
+- update StationBase._expand_timeserie_to_period to remove entries that are older than the min_date config to remove unnessecary database entries
+- fix GroupStation.create_ts: don't throw period changed warning if the change was only due to the expand_to_timestamp output
+
+## Version 1.1.1
+
+- minor fixes to fix previous version
+
 ## Version 1.1.0
 
 - !!rename module from weatherDB to weatherdb to be compliant with PEP 8!!

{weatherdb-1.1.0 → weatherdb-1.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: weatherdb
-Version: 1.1.0
+Version: 1.1.2
 Summary: This is a package to work with and to create the Weather Database which handles, checks, fills and corrects DWD Weather Station data.
 Author-email: Max Schmit <max.schmit@hydrology.uni-freiburg.de>
 License:                     GNU GENERAL PUBLIC LICENSE
@@ -704,7 +704,7 @@ Author: [Max Schmit](https://github.com/maxschmi)
 [![Documentation Status](https://readthedocs.org/projects/weatherdb/badge/?version=latest)](https://weatherdb.readthedocs.io/latest)
 [![Pipeline status](https://gitlab.uni-freiburg.de/hydrology/weatherDB/badges/master/pipeline.svg?ignore_skipped=true)](https://gitlab.uni-freiburg.de/hydrology/weatherDB/-/pipelines) 
 
-The weather-DB module offers an API to interact with the automatically filled weather Database.
+The WeatherDB module offers an API to interact with the automatically filled weather Database.
 
 Depending on the Database user privileges you can use more or less methods of the classes.
 

{weatherdb-1.1.0 → weatherdb-1.1.2}/README.md

@@ -6,7 +6,7 @@ Author: [Max Schmit](https://github.com/maxschmi)
 [![Documentation Status](https://readthedocs.org/projects/weatherdb/badge/?version=latest)](https://weatherdb.readthedocs.io/latest)
 [![Pipeline status](https://gitlab.uni-freiburg.de/hydrology/weatherDB/badges/master/pipeline.svg?ignore_skipped=true)](https://gitlab.uni-freiburg.de/hydrology/weatherDB/-/pipelines) 
 
-The weather-DB module offers an API to interact with the automatically filled weather Database.
+The WeatherDB module offers an API to interact with the automatically filled weather Database.
 
 Depending on the Database user privileges you can use more or less methods of the classes.
 

weatherdb-1.1.2/weatherdb/_version.py

@@ -0,0 +1 @@
+__version__ = "1.1.2"

{weatherdb-1.1.0 → weatherdb-1.1.2}/weatherdb/cli.py

@@ -103,10 +103,16 @@ def download_ma_rasters(which, overwrite, update_user_config):
     """
     click.echo("starting downloading multi annual raster data")
     from weatherdb.utils.get_data import download_ma_rasters
-    download_ma_rasters(overwrite=overwrite)
+    download_ma_rasters(
+        which=which,
+        overwrite=overwrite,
+        update_user_config=update_user_config)
 
 
 @cli.command(short_help="Download the needed digital elevation model raster data from Copernicus to the data folder.")
+@click.option('--out-dir', '-d',
+              type=click.Path(), default=None, show_default=False,
+              help="The directory to save the downloaded DEM data to.")
 @click.option('--overwrite/--no-overwrite', '-o/-no-o',
               type=bool, is_flag=True, default=None, show_default=False,
               help="Should the digital elevation model raster be downloaded even if it already exists?")
@@ -116,7 +122,11 @@ def download_ma_rasters(which, overwrite, update_user_config):
 @click.option("--update-user-config", "-u",
               type=bool, default=False, show_default=True, is_flag=True,
               help="Should the user configuration be updated with the path to the downloaded DEM?")
-def download_dem(overwrite, extent):
+@click.option("--service", "-s",
+              type=str, default=["prism", "openTopography"], show_default=True, multiple=True,
+              help="The service to use to download the DEM. Options are 'prism' or 'openTopography'. " +\
+                   "You can use this option muultiple times to test both in the given order until the file could be downloaded.")
+def download_dem(out_dir, overwrite, extent, update_user_config, service="prism"):
     """Download the newest DEM data from the Copernicus Sentinel dataset.
 
     Only the GLO-30 DEM, wich has a 30m resolution, is downloaded as it is freely available.
@@ -131,7 +141,12 @@ def download_dem(overwrite, extent):
     """
     click.echo("Starting downloading digital elevation model from Copernicus")
     from weatherdb.utils.get_data import download_dem
-    download_dem(overwrite=overwrite, extent=extent)
+    download_dem(
+        out_dir=out_dir,
+        overwrite=overwrite,
+        extent=extent,
+        service=service,
+        update_user_config=update_user_config)
 
 
 # cli statements to update the database

{weatherdb-1.1.0 → weatherdb-1.1.2}/weatherdb/station/GroupStation.py

@@ -526,7 +526,9 @@ class GroupStation(object):
                 period_new = period_filled.union(
                     period,
                     how="inner")
-                if period_new != period:
+                if period_new != period and not (
+                        (period.start == period_new.start) and
+                        ((period.end - period_new.end) <= pd.Timedelta(days=1))):
                     warnings.warn(
                         f"The Period for Station {self.id} got changed from {str(period)} to {str(period_new)}.")
                     period = period_new
@@ -599,8 +601,9 @@ class GroupStation(object):
 
             # create header
             if add_meta:
-                header = f"Name: {name}{"\t" * (len(df.columns)-1)}\n" +\
-                         f"Lat: {y[0]}   ,Lon: {x[0]}{"\t" * (len(df.columns)-1)}\n"
+                tabs = "\t" * (len(df.columns)-1)
+                header = f"Name: {name}{tabs}\n" +\
+                         f"Lat: {y[0]}   ,Lon: {x[0]}{tabs}\n"
             else:
                 header = ""
 

{weatherdb-1.1.0 → weatherdb-1.1.2}/weatherdb/station/StationBases.py

@@ -504,10 +504,12 @@ class StationBase:
                     '{interval}'::INTERVAL)::{tstp_dtype} AS timestamp)
             INSERT INTO timeseries."{stid}_{para}"(timestamp)
                 (SELECT wts.timestamp
-                FROM whole_ts wts
-                LEFT JOIN timeseries."{stid}_{para}" ts
+                 FROM whole_ts wts
+                 LEFT JOIN timeseries."{stid}_{para}" ts
                     ON ts.timestamp=wts.timestamp
-                WHERE ts.timestamp IS NULL);
+                 WHERE ts.timestamp IS NULL);
+            DELETE FROM timeseries."{stid}_{para}"
+            WHERE timestamp < '{min_date} 00:00'::{tstp_dtype};
         """.format(
             stid=self.id,
             para=self._para,

{weatherdb-1.1.0 → weatherdb-1.1.2}/weatherdb/utils/get_data.py

@@ -6,9 +6,19 @@ from pathlib import Path
 from distutils.util import strtobool
 import hashlib
 import progressbar as pb
+import keyring
+import os
+import getpass
+import logging
 
 from ..config import config
 
+__all__ = ["download_ma_rasters", "download_dem"]
+
+log = logging.getLogger(__name__)
+
+# Multi annual rasters
+# --------------------
 def download_ma_rasters(which="all", overwrite=None, update_user_config=False):
     """Get the multi annual rasters on which bases the regionalisation is done.
 
@@ -60,7 +70,7 @@ def download_ma_rasters(which="all", overwrite=None, update_user_config=False):
         if file_key in which:
             # check if file is in config
             if f"data:rasters:{file_key}" not in config:
-                print(f"Skipping {file_key} as it is not in your configuration.\nPlease add a section 'data:rasters:{file_key}' to your configuration file.")
+                log.debug(f"Skipping {file_key} as it is not in your configuration.\nPlease add a section 'data:rasters:{file_key}' to your configuration file.")
                 continue
 
             # check if file already exists
@@ -75,7 +85,7 @@ def download_ma_rasters(which="all", overwrite=None, update_user_config=False):
                         "Do you want to overwrite it? [y/n] "))
 
                 if skip:
-                    print(f"Skipping {file_key} as overwriting is not allowed.")
+                    log.debug(f"Skipping {file_key} as overwriting is not allowed.")
                     continue
 
             # check if the directory exists
@@ -126,24 +136,42 @@ def download_ma_rasters(which="all", overwrite=None, update_user_config=False):
                 if config.has_user_config:
                     config.update_user_config(f"data:rasters:{file_key}", "file", str(file_path))
                 else:
-                    print(f"No user configuration file found, therefor the raster '{file_key}' is not set in the user configuration file.")
+                    log.error(f"No user configuration file found, therefor the raster '{file_key}' is not set in the user configuration file.")
 
+# DEM data
+# --------
+def _check_write_fp(fp, overwrite):
+    """Check if a file exists and if it should be overwritten.
 
-def download_dem(overwrite=None, extent=(5.3, 46.1, 15.6, 55.4), update_user_config=False):
-    """Download the newest DEM data from the Copernicus Sentinel dataset.
-
-    Only the GLO-30 DEM, which has a 30m resolution, is downloaded as it is freely available.
-    If you register as a scientific researcher also the EEA-10, with 10 m resolution, is available.
-    You will have to download the data yourself and define it in the configuration file.
-
-    After downloading the data, the files are merged and saved as a single tif file in the data directory in a subfolder called 'DEM'.
-    To use the DEM data in the WeatherDB, you will have to define the path to the tif file in the configuration file.
+    Parameters
+    ----------
+    fp : str or Path
+        The path to the file to check.
+    overwrite : bool
+        Should the file be overwritten?
+
+    Returns
+    -------
+    bool
+        Should the file be written?
+    """
+    fp = Path(fp)
+    if fp.exists():
+        log.info(f"The file already exists at {fp}.")
+        if overwrite is None:
+            overwrite = strtobool(input(f"{fp} already exists. Do you want to overwrite it? [y/n] "))
+        if not overwrite:
+            log.info("Skipping, because overwritting was turned of.")
+            return False
+    return True
 
-    Source:
-    Copernicus DEM - Global and European Digital Elevation Model. Digital Surface Model (DSM) provided in 3 different resolutions (90m, 30m, 10m) with varying geographical extent (EEA: European and GLO: global) and varying format (INSPIRE, DGED, DTED). DOI:10.5270/ESA-c5d3d65.
+def _download_dem_prism(out_dir, overwrite=None, extent=(5.3, 46.1, 15.6, 55.4)):
+    """Download the DEM data from the Copernicus PRISM service.
 
     Parameters
     ----------
+    out_dir: str or Path
+        The directory to save the DEM data to.
     overwrite : bool, optional
         Should the DEM data be downloaded even if it already exists?
         If None the user will be asked.
@@ -151,9 +179,11 @@ def download_dem(overwrite=None, extent=(5.3, 46.1, 15.6, 55.4), update_user_con
     extent : tuple, optional
         The extent in WGS84 of the DEM data to download.
         The default is the boundary of germany + ~40km = (5.3, 46.1, 15.6, 55.4).
-    update_user_config : bool, optional
-        Should the downloaded DEM be set as the used DEM in the user configuration file?
-        The default is False.
+
+    Returns
+    -------
+    fp : Path
+        The path to the downloaded DEM file.
     """
     # import necessary modules
     import rasterio as rio
@@ -164,10 +194,8 @@ def download_dem(overwrite=None, extent=(5.3, 46.1, 15.6, 55.4), update_user_con
     import re
     import json
 
-    # get dem_dir
-    base_dir = Path(config.get("data", "base_dir"))
-    dem_dir = base_dir / "DEM"
-    dem_dir.mkdir(parents=True, exist_ok=True)
+    # check dir
+    out_dir = Path(out_dir)
 
     # get available datasets
     prism_url = "https://prism-dem-open.copernicus.eu/pd-desk-open-access/publicDemURLs"
@@ -191,20 +219,12 @@ def download_dem(overwrite=None, extent=(5.3, 46.1, 15.6, 55.4), update_user_con
         )[-1]["id"]
 
     # check if dataset already exists
-    dem_file = dem_dir / f'{ds_id.replace("/", "__")}.tif'
-    if dem_file.exists():
-        print(f"The DEM data already exists at {dem_file}.")
-        if overwrite is None:
-            overwrite = strtobool(input("Do you want to overwrite it? [y/n] "))
-        if not overwrite:
-            print("Skipping, because overwritting was turned of.")
-            return
-        else:
-            print("Overwriting the dataset.")
-    dem_dir.mkdir(exist_ok=True)
+    dem_file = out_dir / f'{ds_id.replace("/", "__")}.tif'
+    if not _check_write_fp(dem_file, overwrite):
+        return
 
     # selecting DEM tiles
-    print(f"getting available tiles for Copernicus dataset '{ds_id}'")
+    log.info(f"getting available tiles for Copernicus dataset '{ds_id}'")
     ds_files_req = json.loads(
         requests.get(
             f"{prism_url}/{ds_id.replace('/', '__')}",
@@ -225,13 +245,13 @@ def download_dem(overwrite=None, extent=(5.3, 46.1, 15.6, 55.4), update_user_con
         ds_files_all))
 
     # download DEM tiles
-    print("downloading tiles")
+    log.info("downloading tiles")
     with TemporaryDirectory() as tmp_dir:
         tmp_dir_fp = Path(tmp_dir)
         for f in pb.progressbar(ds_files):
             with open(tmp_dir_fp / Path(f["nativeDemUrl"]).name, "wb") as d:
                 d.write(requests.get(f["nativeDemUrl"]).content)
-        print("downloaded all files")
+        log.info("downloaded all files")
 
         # extracting tifs from tars
         for i, f in pb.progressbar(list(enumerate(tmp_dir_fp.glob("*.tar")))):
@@ -269,17 +289,161 @@ def download_dem(overwrite=None, extent=(5.3, 46.1, 15.6, 55.4), update_user_con
         tmp_eula_fp = next(tmp_dir_fp.glob("*.pdf"))
         shutil.copyfile(
             tmp_eula_fp,
-            dem_dir / tmp_eula_fp.name
+            out_dir / tmp_eula_fp.name
         )
 
-    print(f"created DEM at '{dem_file}'.")
+    log.info(f"created DEM at '{dem_file}'.")
+    return dem_file
+
+def _download_dem_opentopo(
+        out_dir,
+        overwrite=None,
+        extent=(5.3, 46.1, 15.6, 55.4),
+        api_key=os.environ.get(
+            "WEATHERDB_OPENTOPO_API_KEY",
+            fallback=keyring.get_password("weatherdb", "opentopo_api_key"))):
+    """Download the DEM data from the OpenTopography service.
+
+    Get an API key from (OpenTopography)[https://portal.opentopography.org/] to use this service.
+
+    Parameters
+    ----------
+    out_dir : str or Path
+        The directory to save the DEM data to.
+    overwrite : bool, optional
+        Should the DEM data be downloaded even if it already exists?
+        If None the user will be asked.
+        The default is None.
+    extent : tuple, optional
+        The extent in WGS84 of the DEM data to download.
+        Should be in the format (south, west, north, east).
+        The default is the boundary of germany + ~40km = (5.3, 46.1, 15.6, 55.4).
+    api_key : str, optional
+        The API key for the OpenTopography service.
+        If None the user will be asked.
+        The default is to check if the environment variable "WEATHERDB_OPENTOPO_API_KEY" is set or if the keyring has a password for "weatherdb" and "opentopo_api_key".
+
+    Returns
+    -------
+    fp : Path
+        The path to the downloaded DEM file.
+    """
+    # check api key
+    if api_key is None:
+        print("No API key for OpenTopography was given or found in the keyring or environment variable.")
+        api_key = getpass("Please enter your API key for OpenTopography: ")
+
+    # make query
+    w, s, e, n = extent
+    url = f"https://portal.opentopography.org/API/globaldem?demtype=COP30&west={w}&south={s}&east={e}&north={n}&outputFormat=GTiff&API_Key={api_key}"
+    r = requests.get(url)
+    if r.status_code == 200:
+        # store api key
+        if api_key != keyring.get_password("weatherdb", "opentopo_api_key"):
+            keyring.set_password("weatherdb", "opentopo_api_key", api_key)
+
+        # get dem file
+        out_fp = out_dir / "OpenTopo_COP30.tif"
+
+        # check if file already exists
+        if not _check_write_fp(out_fp, overwrite):
+            return
+
+        # download file
+        with open(out_fp, "wb") as f:
+            for chunk in r.iter_content(chunk_size=1024):
+                f.write(chunk)
+            log.info(f"Downloaded DEM data from OpenTopography to '{out_fp}'.")
+            return out_fp
+
+    log.error(f"Request to openTopography API with url {r.url.replace(api_key, "[MASKED]")} returned status code {r.status_code}")
+
+def download_dem(out_dir=None,
+                 overwrite=None,
+                 extent=(5.3, 46.1, 15.6, 55.4),
+                 update_user_config=False,
+                 service=["prism", "openTopography"], **kwargs):
+    """Download the newest DEM data from the Copernicus Sentinel dataset.
+
+    Only the GLO-30 DEM, which has a 30m resolution, is downloaded as it is freely available.
+    If you register as a scientific researcher also the EEA-10, with 10 m resolution, is available.
+    You will have to download the data yourself and define it in the configuration file.
+
+    After downloading the data, the files are merged and saved as a single tif file in the data directory in a subfolder called 'DEM'.
+    To use the DEM data in the WeatherDB, you will have to define the path to the tif file in the configuration file.
+
+    Source:
+    Copernicus DEM - Global and European Digital Elevation Model. Digital Surface Model (DSM) provided in 3 different resolutions (90m, 30m, 10m) with varying geographical extent (EEA: European and GLO: global) and varying format (INSPIRE, DGED, DTED). DOI:10.5270/ESA-c5d3d65.
+
+    Parameters
+    ----------
+    out_dir : str or Path, optional
+        The directory to save the DEM data to.
+        If None the data is saved in the data directory in a subfolder called 'DEM'.
+        The default is None.
+    overwrite : bool, optional
+        Should the DEM data be downloaded even if it already exists?
+        If None the user will be asked.
+        The default is None.
+    extent : tuple, optional
+        The extent in WGS84 of the DEM data to download.
+        The default is the boundary of germany + ~40km = (5.3, 46.1, 15.6, 55.4).
+    update_user_config : bool, optional
+        Should the downloaded DEM be set as the used DEM in the user configuration file?
+        The default is False.
+    service : str or list of str, optional
+        The service to use to download the DEM data.
+        Options are "prism" and "openTopography".
+        If Both are given they are executed in the order they are given.
+        If OpenTopography is selected, you will have to provide an API key.
+        The default is ["prism", "openTopography"].
+    """
+    # check service
+    if isinstance(service, str):
+        service = [service]
+
+    # check dir
+    if out_dir is None:
+        out_dir = Path(config.get("data", "base_dir")) / "DEM"
+    else:
+        out_dir = Path(out_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    # download data
+    fp=None
+    for s in service:
+        if s == "prism":
+            try:
+                fp = _download_dem_prism(
+                    out_dir=out_dir,
+                    overwrite=overwrite,
+                    extent=extent,
+                    **kwargs)
+                break
+            except Exception as e:
+                log.debug(f"Error while downloading DEM from PRISM: {e}")
+        elif s == "openTopography":
+            try:
+                fp = _download_dem_opentopo(
+                    out_dir=out_dir,
+                    overwrite=overwrite,
+                    extent=extent,
+                    **kwargs)
+                break
+            except Exception as e:
+                log.debug(f"Error while downloading DEM from OpenTopography: {e}")
+
+    # check if file was downloaded
+    if fp is None:
+        log.error("No DEM data was downloaded.")
+        return
 
     # update user config
     if update_user_config:
         if config.has_user_config:
-            config.update_user_config("data:rasters", "dems", str(dem_file))
+            config.update_user_config("data:rasters", "dems", str(fp))
             return
         else:
-            print("No user configuration file found, therefor the DEM is not set in the user configuration file.")
+            log.info("No user configuration file found, therefor the DEM is not set in the user configuration file.")
 
-    print("To use the DEM data in the WeatherDB, you will have to define the path to the tif file in the user configuration file.")
+    log.info("To use the DEM data in the WeatherDB, you will have to define the path to the tif file in the user configuration file.")

{weatherdb-1.1.0 → weatherdb-1.1.2}/weatherdb.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: weatherdb
-Version: 1.1.0
+Version: 1.1.2
 Summary: This is a package to work with and to create the Weather Database which handles, checks, fills and corrects DWD Weather Station data.
 Author-email: Max Schmit <max.schmit@hydrology.uni-freiburg.de>
 License:                     GNU GENERAL PUBLIC LICENSE
@@ -704,7 +704,7 @@ Author: [Max Schmit](https://github.com/maxschmi)
 [![Documentation Status](https://readthedocs.org/projects/weatherdb/badge/?version=latest)](https://weatherdb.readthedocs.io/latest)
 [![Pipeline status](https://gitlab.uni-freiburg.de/hydrology/weatherDB/badges/master/pipeline.svg?ignore_skipped=true)](https://gitlab.uni-freiburg.de/hydrology/weatherDB/-/pipelines) 
 
-The weather-DB module offers an API to interact with the automatically filled weather Database.
+The WeatherDB module offers an API to interact with the automatically filled weather Database.
 
 Depending on the Database user privileges you can use more or less methods of the classes.
 

{weatherdb-1.1.0 → weatherdb-1.1.2}/weatherdb.egg-info/SOURCES.txt

@@ -27,12 +27,12 @@ docs/source/_static/logo.png
 docs/source/api/api.rst
 docs/source/api/cli.rst
 docs/source/api/weatherDB.broker.rst
-docs/source/api/weatherDB.config.rst
-docs/source/api/weatherDB.db.rst
-docs/source/api/weatherDB.rst
-docs/source/api/weatherDB.station.rst
-docs/source/api/weatherDB.stations.rst
-docs/source/api/weatherDB.utils.rst
+docs/source/api/weatherdb.config.rst
+docs/source/api/weatherdb.db.rst
+docs/source/api/weatherdb.rst
+docs/source/api/weatherdb.station.rst
+docs/source/api/weatherdb.stations.rst
+docs/source/api/weatherdb.utils.rst
 docs/source/setup/Configuration.md
 docs/source/setup/Hosting.md
 docs/source/setup/Install.md

weatherdb-1.1.0/weatherdb/_version.py

@@ -1 +0,0 @@
-__version__ = "1.1.0"