weatherdb 1.1.1__tar.gz → 1.2.0__tar.gz

{weatherdb-1.1.1 → weatherdb-1.2.0}/.gitlab-ci.yml

@@ -1,6 +1,7 @@
 stages:
   - test_single
   - test_matrix
+  - test_manual
   - build
   - deploy
 
@@ -21,10 +22,10 @@ test_single:
   before_script:
     - python3 -m pip install --upgrade pip --root-user-action=ignore
     - python3 -m pip install unittest-xml-reporting --root-user-action=ignore
-  script:
     - python3 -m pip install .[optionals] --root-user-action=ignore
     - mkdir -p $WEATHERDB_DATA_BASE_DIR
     - mkdir -p test-reports
+  script:
     - python3 -m xmlrunner discover -s tests -p testSuite.py -o test-reports
   cache:
     - key: weatherdb-data-cache-$CI_COMMIT_REF_SLUG
@@ -61,10 +62,13 @@ test_single:
     WEATHERDB_LOGGING_HANDLER: console
     WEATHERDB_LOGGING_LEVEL: DEBUG
     WEATHERDB_HORIZON_RADIUS: 40000
+    WEATHERDB_OPENTOPO_API_KEY: $WEATHERDB_OPENTOPO_API_KEY
     WEATHERDB_TEST_ARTIFACT_DIR: $CI_PROJECT_DIR/artifacts
     WEATHERDB_TEST_ARTIFACT_DB_DUMP: ON_FAILURE
     WEATHERDB_TEST_ARTIFACT_LIST_DATA_FILES: ON_FAILURE
     WEATHERDB_TEST_ARTIFACT_COPY_USER_CONFIG: ON_FAILURE
+    WEATHERDB_TEST_DR_COPY_RASTERS: True
+    WEATHERDB_TEST_DR_UPDATE_USER_CONFIG: True
     DOCKER_ENV: test
   tags:
     - docker
@@ -72,14 +76,26 @@ test_single:
 test_matrix:
   stage: test_matrix
   extends: test_single
-  dependencies:
-    - test_single
   parallel:
     matrix:
       - PY_VERSION: ["3.9","3.10","3.11","3.12","3.13"]
   variables:
     PYTHON_VERSION: $PY_VERSION
 
+test_download_rasters:
+  extends: test_single
+  stage: test_manual
+  rules:
+    - when: manual
+      allow_failure: true
+  script:
+    - python3 -m xmlrunner discover -s tests -p test_downloadRasters.py --complete -o test-reports
+  variables:
+    PYTHON_VERSION: $PY_VERSION
+  parallel:
+    matrix:
+      - PY_VERSION: ["3.9","3.10","3.11","3.12","3.13"]
+
 build:
   rules:
     - if: $CI_COMMIT_TAG =~ /^[Vv]\d+\.\d+\.\d+$/

{weatherdb-1.1.1 → weatherdb-1.2.0}/CHANGES.md

@@ -1,8 +1,19 @@
 # Change-log
 
-## Version 1.0.1
+## Version 1.2.0
+
+- tests: add test rasters and make downloading the rasters a manual job on GitLab as there were many problems
+- fillup: add method to linearly interpolate residual missing values
+
+## Version 1.1.2
+
+- add method to download DEM data from OpenTopography-API
+- update StationBase._expand_timeserie_to_period to remove entries that are older than the min_date config to remove unnessecary database entries
+- fix GroupStation.create_ts: don't throw period changed warning if the change was only due to the expand_to_timestamp output
+
+## Version 1.1.1
 
- - minor fixes to fix previous version
+- minor fixes to fix previous version
 
 ## Version 1.1.0
 

{weatherdb-1.1.1 → weatherdb-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: weatherdb
-Version: 1.1.1
+Version: 1.2.0
 Summary: This is a package to work with and to create the Weather Database which handles, checks, fills and corrects DWD Weather Station data.
 Author-email: Max Schmit <max.schmit@hydrology.uni-freiburg.de>
 License:                     GNU GENERAL PUBLIC LICENSE

{weatherdb-1.1.1 → weatherdb-1.2.0}/docker/docker-compose.yaml

@@ -14,12 +14,14 @@ services:
       WEATHERDB_USER_CONFIG_FILE: /home/config_user.yaml
       WEATHERDB_DATA_BASE_DIR: /home/data
       WEATHERDB_HANDLE_NON_EXISTING_CONFIG: create
+      WEATHERDB_OPENTOPO_API_KEY: /run/secrets/opentopo_api_key
       DOCKER_ENV: main
     volumes:
       - weatherdb_home:/home
     command: ["sleep", "infinity"] # to keep awake
     secrets:
       - db_password
+      - opentopo_api_key
     develop:
       watch:
         - action: rebuild
@@ -53,6 +55,8 @@ volumes:
 secrets:
   db_password:
     file: ./db_password.docker-secret
+  opentopo_api_key:
+    file: ./opentopo_api_key.docker-secret
 
 # start from parent folder with `docker compose -f docker\\docker-compose.yaml up --build`
-# To connect to ther weatherdb service use `docker-compose exec weatherdb bash`
+# To connect to the weatherdb service use `docker-compose exec weatherdb bash`

{weatherdb-1.1.1 → weatherdb-1.2.0}/docs/source/Methode.md

@@ -117,7 +117,11 @@ N_{neighbor} * \dfrac{N_{station,ma,winter}}{N_{neighbor,ma,winter}} \space if\s
 N_{neighbor} * \dfrac{N_{station,ma,summer}}{N_{neighbor,ma,summer}} \space if\space month\notin[4:9]
 \end{cases}$
 
-For the precipitation stations only stations within a 100 km radius are taken to fill missing values. For the potential Evapotranspiration and the temperature this radius is 150km. For the temperature stations the median of the regionalised values from the 5 closest stations (but not more than 150 km away) is taken to fill missing values.
+For the precipitation stations only stations within a 110 km radius are taken to fill missing values. For the potential Evapotranspiration and the temperature this radius is 150km. For the temperature stations the median of the regionalised values from the 5 closest stations (but not more than 150 km away) is taken to fill missing values. Those limits can get configured in the user configurations.
+
+### linear interpolation
+
+After missing values got filled with regionalised values from the neighboring stations there can still be missing values left. To fill those residual missing values, they get interpolated linearly. Those linear interpolations have configurable limits for how big the interval of missing values can be. The default values are 1h for precipitation and 2 days for temperature and evapotranspiration.
 
 ### adjusting precipitation to daily station measurements
 

{weatherdb-1.1.1 → weatherdb-1.2.0}/docs/source/setup/Quickstart.md

@@ -46,7 +46,8 @@ So there is:
 - "raw" : the raw measurements as on the DWD server
 - "qc"  : The quality checked data
 - "filled" : The filled timeseries
-- "filled_by" : The station ID of the station from which the data was taken to fill the measurements
+- "filled_by" : The station ID of the station from which the data was taken to fill the measurements.
+                If the value was filled by linear intrepolation "filled_by" is `-1`
 - "corr"    : The Richter corrected timeserie.
 
 If you want more than just one kind of timeseries, e.g. the filled timeseries, together with the id from which station the respective field got filled with use:

weatherdb-1.2.0/tests/test-data/DEM/README.md

@@ -0,0 +1,3 @@
+# Readme
+
+The DEM file in this folder originates from Copernicus and got clipped to the region of analysis in the test environment + 40km

weatherdb-1.2.0/tests/test-data/regionalisation/README.md

@@ -0,0 +1,3 @@
+# Readme
+
+The files in this folder originate from https://zenodo.org/records/10066045 and got clipped to the region of analysis in the test environment

weatherdb-1.2.0/tests/test-data/test-data-config.ini

@@ -0,0 +1,28 @@
+; Those are configs that should get loaded, when the test files are loaded
+[data:rasters]
+DEMS = ${data:BASE_DIR}/DEM/COP-DEM_GLO-30-DGED__2023_1_clipped.tif
+
+[data:rasters:hyras]
+FILE = ${data:BASE_DIR}/regionalisation/HYRAS_ma_1991_2020_DGM25_clipped.tif
+BAND_P_WIHY = n_hyras_wihj
+BAND_P_SUHY = n_hyras_sohj
+BAND_P_YEAR = n_hyras_year
+SRID = 3035
+FACTOR_P_WIHY = 1
+FACTOR_P_SUHY = 1
+FACTOR_P_YEAR = 1
+
+[data:rasters:dwd]
+FILE = ${data:BASE_DIR}/regionalisation/DWD-grid_ma_1991_2020_DGM25_clipped.tif
+BAND_P_WIHY = n_wihj
+BAND_P_SUHY = n_sohj
+BAND_P_YEAR = n_year
+BAND_T_YEAR = t_year
+BAND_ET_YEAR = et_year
+SRID = 3035
+FACTOR_P_WIHY = 1
+FACTOR_P_SUHY = 1
+FACTOR_P_YEAR = 1
+FACTOR_T_YEAR = 0.1
+FACTOR_ET_YEAR = 1
+

weatherdb-1.2.0/weatherdb/_version.py

@@ -0,0 +1 @@
+__version__ = "1.2.0"

{weatherdb-1.1.1 → weatherdb-1.2.0}/weatherdb/cli.py

@@ -103,10 +103,16 @@ def download_ma_rasters(which, overwrite, update_user_config):
     """
     click.echo("starting downloading multi annual raster data")
     from weatherdb.utils.get_data import download_ma_rasters
-    download_ma_rasters(overwrite=overwrite)
+    download_ma_rasters(
+        which=which,
+        overwrite=overwrite,
+        update_user_config=update_user_config)
 
 
 @cli.command(short_help="Download the needed digital elevation model raster data from Copernicus to the data folder.")
+@click.option('--out-dir', '-d',
+              type=click.Path(), default=None, show_default=False,
+              help="The directory to save the downloaded DEM data to.")
 @click.option('--overwrite/--no-overwrite', '-o/-no-o',
               type=bool, is_flag=True, default=None, show_default=False,
               help="Should the digital elevation model raster be downloaded even if it already exists?")
@@ -116,7 +122,11 @@ def download_ma_rasters(which, overwrite, update_user_config):
 @click.option("--update-user-config", "-u",
               type=bool, default=False, show_default=True, is_flag=True,
               help="Should the user configuration be updated with the path to the downloaded DEM?")
-def download_dem(overwrite, extent):
+@click.option("--service", "-s",
+              type=str, default=["prism", "openTopography"], show_default=True, multiple=True,
+              help="The service to use to download the DEM. Options are 'prism' or 'openTopography'. " +\
+                   "You can use this option muultiple times to test both in the given order until the file could be downloaded.")
+def download_dem(out_dir, overwrite, extent, update_user_config, service="prism"):
     """Download the newest DEM data from the Copernicus Sentinel dataset.
 
     Only the GLO-30 DEM, wich has a 30m resolution, is downloaded as it is freely available.
@@ -131,7 +141,12 @@ def download_dem(overwrite, extent):
     """
     click.echo("Starting downloading digital elevation model from Copernicus")
     from weatherdb.utils.get_data import download_dem
-    download_dem(overwrite=overwrite, extent=extent)
+    download_dem(
+        out_dir=out_dir,
+        overwrite=overwrite,
+        extent=extent,
+        service=service,
+        update_user_config=update_user_config)
 
 
 # cli statements to update the database

{weatherdb-1.1.1 → weatherdb-1.2.0}/weatherdb/config/config_default.ini

@@ -143,12 +143,25 @@ RASTER_BUFFER_CRS = ${weatherdb:HORIZON_CRS}
 ; The default is 1999-01-01
 MIN_DATE = 1999-01-01
 
+; When there are still NAs after filling with neighboring stations, the module can use linear interpolation to fill the gaps
+; The limit defines the maximum interval that is allowed to fill with linear interpolation
+; The value is given with a unit, e.g. 3 days, 30 minutes, 1 hour
+; For further explanation about the format see https://www.postgresql.org/docs/current/datatype-datetime.html#DATATYPE-INTERVAL-INPUT
+; for precipitation, where the timeseries resolution is 10 minutes, 30 minutes means 3 missing values are interpolated, but not more.
+; If there shouldn't be any linear interpolation, set the limit to 0
+; For precipitation, the default is 1 hour
+LINEAR_INTERPOLATION_LIMIT_P = 1 hour
+; For temperature, the default is 2 days
+LINEAR_INTERPOLATION_LIMIT_T = 2 days
+; For Evapotranspiration, the default is 2 days
+LINEAR_INTERPOLATION_LIMIT_ET = 2 days
+
 
 [weatherdb:max_fillup_distance]
 ; The maximum distance in meters to use for the filling of the station data
 ; For each parameter (P, T, ET) the module uses a different distance
 ; Precipitation (P)
-P = 110000
+P = 130000
 ; Temperature (T)
 T = 150000
 ; Evapotranspiration (ET)

{weatherdb-1.1.1 → weatherdb-1.2.0}/weatherdb/station/GroupStation.py

@@ -526,7 +526,9 @@ class GroupStation(object):
                 period_new = period_filled.union(
                     period,
                     how="inner")
-                if period_new != period:
+                if period_new != period and not (
+                        (period.start == period_new.start) and
+                        ((period.end - period_new.end) <= pd.Timedelta(days=1))):
                     warnings.warn(
                         f"The Period for Station {self.id} got changed from {str(period)} to {str(period_new)}.")
                     period = period_new

{weatherdb-1.1.1 → weatherdb-1.2.0}/weatherdb/station/StationBases.py

@@ -504,10 +504,12 @@ class StationBase:
                     '{interval}'::INTERVAL)::{tstp_dtype} AS timestamp)
             INSERT INTO timeseries."{stid}_{para}"(timestamp)
                 (SELECT wts.timestamp
-                FROM whole_ts wts
-                LEFT JOIN timeseries."{stid}_{para}" ts
+                 FROM whole_ts wts
+                 LEFT JOIN timeseries."{stid}_{para}" ts
                     ON ts.timestamp=wts.timestamp
-                WHERE ts.timestamp IS NULL);
+                 WHERE ts.timestamp IS NULL);
+            DELETE FROM timeseries."{stid}_{para}"
+            WHERE timestamp < '{min_date} 00:00'::{tstp_dtype};
         """.format(
             stid=self.id,
             para=self._para,
@@ -1489,6 +1491,70 @@ class StationBase:
                 extra_fillup_where=sql_format_dict["extra_fillup_where"] +\
                     ' OR ts."filled_by" IS DISTINCT FROM new."filled_by"'))
 
+        # linear interpolation for the last missing values
+        lr_limit = config.get(
+            "weatherdb",
+            "LINEAR_INTERPOLATION_LIMIT_{self._para_base}",
+            fallback="0")
+        if lr_limit != "0":
+            sql_format_dict.update(dict(
+                sql_linear_interpolation=f"""
+                    DO
+                    $do$
+                        DECLARE reg_borders record;
+                        BEGIN
+                            FOR reg_borders IN
+                                WITH empty_periods AS (
+                                    SELECT *
+                                    FROM (  SELECT
+                                                CASE WHEN dist_next>'{self._interval}'::interval
+                                                     THEN timestamp ELSE NULL
+                                                END AS start,
+                                                CASE WHEN dist_next>'{self._interval}'::interval
+                                                     THEN LEAD(timestamp) OVER (ORDER BY timestamp)
+                                                     ELSE NULL
+                                                END AS end
+                                            FROM (
+                                                SELECT *,
+                                                    timestamp - lag(timestamp, 1) OVER ( ORDER BY timestamp) AS dist_prev,
+                                                    lead(timestamp, 1) OVER ( ORDER BY timestamp) - timestamp AS dist_next
+                                                FROM new_filled_{self.id}_{self._para}
+                                                WHERE filled IS NOT NULL) t
+                                            WHERE t.dist_prev > '{self._interval}'::interval
+                                                  OR t.dist_next > '{self._interval}'::interval
+                                         ) p
+                                    WHERE p.start IS NOT NULL AND p.end IS NOT NULL)
+                                SELECT
+                                    ep.start AS timestamp_start,
+                                    ttss.filled AS value_start,
+                                    ep.end AS timestamp_end,
+                                    ttse.filled AS value_end,
+                                    (ttse.filled - ttss.filled)::numeric/(EXTRACT(EPOCH FROM (ep.end - ep.start))/EXTRACT(EPOCH FROM '{self._interval}'::interval))::numeric as slope
+                                FROM empty_periods ep
+                                LEFT JOIN new_filled_{self.id}_{self._para} ttss ON ep.start = ttss.timestamp
+                                LEFT JOIN new_filled_{self.id}_{self._para} ttse ON ep.end = ttse.timestamp
+                                where (ep.end - ep.start - '{self._interval}'::interval) <= '{lr_limit}'::interval
+                            loop
+                                execute FORMAT(
+                                $$
+                                UPDATE new_filled_{self.id}_{self._para} ts
+                                SET filled=%2$L + (EXTRACT(EPOCH FROM ts.timestamp - %1$L)::numeric/(EXTRACT(EPOCH FROM '{self._interval}'::interval))::numeric * %5$L),
+                                    filled_by=-1
+                                WHERE ts.timestamp>%1$L and ts.timestamp<%3$L;
+                                $$,
+                                reg_borders.timestamp_start,
+                                reg_borders.value_start,
+                                reg_borders.timestamp_end,
+                                reg_borders.value_end,
+                                reg_borders.slope
+                                );
+                            END loop;
+                        END
+                    $do$;"""))
+        else:
+            sql_format_dict.update(dict(sql_linear_interpolation=""))
+
+
         # Make SQL statement to fill the missing values with values from nearby stations
         sql = """
             CREATE TEMP TABLE new_filled_{stid}_{para}
@@ -1561,15 +1627,16 @@ class StationBase:
                         FROM new_filled_{stid}_{para}
                         WHERE "filled" IS NULL {extra_unfilled_period_where};
                     END LOOP;
-                    {sql_extra_after_loop}
-                    UPDATE timeseries."{stid}_{para}" ts
-                    SET filled = new.filled, {extra_cols_fillup}
-                        filled_by = new.filled_by
-                    FROM new_filled_{stid}_{para} new
-                    WHERE ts.timestamp = new.timestamp
-                        AND (ts."filled" IS DISTINCT FROM new."filled" {extra_fillup_where}) ;
                 END
             $do$;
+            {sql_linear_interpolation}
+            {sql_extra_after_loop}
+            UPDATE timeseries."{stid}_{para}" ts
+            SET filled = new.filled, {extra_cols_fillup}
+                filled_by = new.filled_by
+            FROM new_filled_{stid}_{para} new
+            WHERE ts.timestamp = new.timestamp
+                AND (ts."filled" IS DISTINCT FROM new."filled" {extra_fillup_where}) ;
         """.format(**sql_format_dict)
 
         # execute