weatherdb 1.1.2__tar.gz → 1.2.0__tar.gz

{weatherdb-1.1.2 → weatherdb-1.2.0}/.gitlab-ci.yml

@@ -1,6 +1,7 @@
 stages:
   - test_single
   - test_matrix
+  - test_manual
   - build
   - deploy
 
@@ -21,10 +22,10 @@ test_single:
   before_script:
     - python3 -m pip install --upgrade pip --root-user-action=ignore
     - python3 -m pip install unittest-xml-reporting --root-user-action=ignore
-  script:
     - python3 -m pip install .[optionals] --root-user-action=ignore
     - mkdir -p $WEATHERDB_DATA_BASE_DIR
     - mkdir -p test-reports
+  script:
     - python3 -m xmlrunner discover -s tests -p testSuite.py -o test-reports
   cache:
     - key: weatherdb-data-cache-$CI_COMMIT_REF_SLUG
@@ -61,11 +62,13 @@ test_single:
     WEATHERDB_LOGGING_HANDLER: console
     WEATHERDB_LOGGING_LEVEL: DEBUG
     WEATHERDB_HORIZON_RADIUS: 40000
+    WEATHERDB_OPENTOPO_API_KEY: $WEATHERDB_OPENTOPO_API_KEY
     WEATHERDB_TEST_ARTIFACT_DIR: $CI_PROJECT_DIR/artifacts
     WEATHERDB_TEST_ARTIFACT_DB_DUMP: ON_FAILURE
     WEATHERDB_TEST_ARTIFACT_LIST_DATA_FILES: ON_FAILURE
     WEATHERDB_TEST_ARTIFACT_COPY_USER_CONFIG: ON_FAILURE
-    WEATHERDB_OPENTOPO_API_KEY: $WEATHERDB_OPENTOPO_API_KEY
+    WEATHERDB_TEST_DR_COPY_RASTERS: True
+    WEATHERDB_TEST_DR_UPDATE_USER_CONFIG: True
     DOCKER_ENV: test
   tags:
     - docker
@@ -73,14 +76,26 @@ test_single:
 test_matrix:
   stage: test_matrix
   extends: test_single
-  dependencies:
-    - test_single
   parallel:
     matrix:
       - PY_VERSION: ["3.9","3.10","3.11","3.12","3.13"]
   variables:
     PYTHON_VERSION: $PY_VERSION
 
+test_download_rasters:
+  extends: test_single
+  stage: test_manual
+  rules:
+    - when: manual
+      allow_failure: true
+  script:
+    - python3 -m xmlrunner discover -s tests -p test_downloadRasters.py --complete -o test-reports
+  variables:
+    PYTHON_VERSION: $PY_VERSION
+  parallel:
+    matrix:
+      - PY_VERSION: ["3.9","3.10","3.11","3.12","3.13"]
+
 build:
   rules:
     - if: $CI_COMMIT_TAG =~ /^[Vv]\d+\.\d+\.\d+$/

{weatherdb-1.1.2 → weatherdb-1.2.0}/CHANGES.md

@@ -1,5 +1,10 @@
 # Change-log
 
+## Version 1.2.0
+
+- tests: add test rasters and make downloading the rasters a manual job on GitLab as there were many problems
+- fillup: add method to linearly interpolate residual missing values
+
 ## Version 1.1.2
 
 - add method to download DEM data from OpenTopography-API

{weatherdb-1.1.2 → weatherdb-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: weatherdb
-Version: 1.1.2
+Version: 1.2.0
 Summary: This is a package to work with and to create the Weather Database which handles, checks, fills and corrects DWD Weather Station data.
 Author-email: Max Schmit <max.schmit@hydrology.uni-freiburg.de>
 License:                     GNU GENERAL PUBLIC LICENSE

{weatherdb-1.1.2 → weatherdb-1.2.0}/docker/docker-compose.yaml

@@ -14,12 +14,14 @@ services:
       WEATHERDB_USER_CONFIG_FILE: /home/config_user.yaml
       WEATHERDB_DATA_BASE_DIR: /home/data
       WEATHERDB_HANDLE_NON_EXISTING_CONFIG: create
+      WEATHERDB_OPENTOPO_API_KEY: /run/secrets/opentopo_api_key
       DOCKER_ENV: main
     volumes:
       - weatherdb_home:/home
     command: ["sleep", "infinity"] # to keep awake
     secrets:
       - db_password
+      - opentopo_api_key
     develop:
       watch:
         - action: rebuild
@@ -53,6 +55,8 @@ volumes:
 secrets:
   db_password:
     file: ./db_password.docker-secret
+  opentopo_api_key:
+    file: ./opentopo_api_key.docker-secret
 
 # start from parent folder with `docker compose -f docker\\docker-compose.yaml up --build`
-# To connect to ther weatherdb service use `docker-compose exec weatherdb bash`
+# To connect to the weatherdb service use `docker-compose exec weatherdb bash`

{weatherdb-1.1.2 → weatherdb-1.2.0}/docs/source/Methode.md

@@ -117,7 +117,11 @@ N_{neighbor} * \dfrac{N_{station,ma,winter}}{N_{neighbor,ma,winter}} \space if\s
 N_{neighbor} * \dfrac{N_{station,ma,summer}}{N_{neighbor,ma,summer}} \space if\space month\notin[4:9]
 \end{cases}$
 
-For the precipitation stations only stations within a 100 km radius are taken to fill missing values. For the potential Evapotranspiration and the temperature this radius is 150km. For the temperature stations the median of the regionalised values from the 5 closest stations (but not more than 150 km away) is taken to fill missing values.
+For the precipitation stations only stations within a 110 km radius are taken to fill missing values. For the potential Evapotranspiration and the temperature this radius is 150km. For the temperature stations the median of the regionalised values from the 5 closest stations (but not more than 150 km away) is taken to fill missing values. Those limits can get configured in the user configurations.
+
+### linear interpolation
+
+After missing values got filled with regionalised values from the neighboring stations there can still be missing values left. To fill those residual missing values, they get interpolated linearly. Those linear interpolations have configurable limits for how big the interval of missing values can be. The default values are 1h for precipitation and 2 days for temperature and evapotranspiration.
 
 ### adjusting precipitation to daily station measurements
 

{weatherdb-1.1.2 → weatherdb-1.2.0}/docs/source/setup/Quickstart.md

@@ -46,7 +46,8 @@ So there is:
 - "raw" : the raw measurements as on the DWD server
 - "qc"  : The quality checked data
 - "filled" : The filled timeseries
-- "filled_by" : The station ID of the station from which the data was taken to fill the measurements
+- "filled_by" : The station ID of the station from which the data was taken to fill the measurements.
+                If the value was filled by linear intrepolation "filled_by" is `-1`
 - "corr"    : The Richter corrected timeserie.
 
 If you want more than just one kind of timeseries, e.g. the filled timeseries, together with the id from which station the respective field got filled with use:

weatherdb-1.2.0/tests/test-data/DEM/README.md

@@ -0,0 +1,3 @@
+# Readme
+
+The DEM file in this folder originates from Copernicus and got clipped to the region of analysis in the test environment + 40km

weatherdb-1.2.0/tests/test-data/regionalisation/README.md

@@ -0,0 +1,3 @@
+# Readme
+
+The files in this folder originate from https://zenodo.org/records/10066045 and got clipped to the region of analysis in the test environment

weatherdb-1.2.0/tests/test-data/test-data-config.ini

@@ -0,0 +1,28 @@
+; Those are configs that should get loaded, when the test files are loaded
+[data:rasters]
+DEMS = ${data:BASE_DIR}/DEM/COP-DEM_GLO-30-DGED__2023_1_clipped.tif
+
+[data:rasters:hyras]
+FILE = ${data:BASE_DIR}/regionalisation/HYRAS_ma_1991_2020_DGM25_clipped.tif
+BAND_P_WIHY = n_hyras_wihj
+BAND_P_SUHY = n_hyras_sohj
+BAND_P_YEAR = n_hyras_year
+SRID = 3035
+FACTOR_P_WIHY = 1
+FACTOR_P_SUHY = 1
+FACTOR_P_YEAR = 1
+
+[data:rasters:dwd]
+FILE = ${data:BASE_DIR}/regionalisation/DWD-grid_ma_1991_2020_DGM25_clipped.tif
+BAND_P_WIHY = n_wihj
+BAND_P_SUHY = n_sohj
+BAND_P_YEAR = n_year
+BAND_T_YEAR = t_year
+BAND_ET_YEAR = et_year
+SRID = 3035
+FACTOR_P_WIHY = 1
+FACTOR_P_SUHY = 1
+FACTOR_P_YEAR = 1
+FACTOR_T_YEAR = 0.1
+FACTOR_ET_YEAR = 1
+

weatherdb-1.2.0/weatherdb/_version.py

@@ -0,0 +1 @@
+__version__ = "1.2.0"

{weatherdb-1.1.2 → weatherdb-1.2.0}/weatherdb/config/config_default.ini

@@ -143,12 +143,25 @@ RASTER_BUFFER_CRS = ${weatherdb:HORIZON_CRS}
 ; The default is 1999-01-01
 MIN_DATE = 1999-01-01
 
+; When there are still NAs after filling with neighboring stations, the module can use linear interpolation to fill the gaps
+; The limit defines the maximum interval that is allowed to fill with linear interpolation
+; The value is given with a unit, e.g. 3 days, 30 minutes, 1 hour
+; For further explanation about the format see https://www.postgresql.org/docs/current/datatype-datetime.html#DATATYPE-INTERVAL-INPUT
+; for precipitation, where the timeseries resolution is 10 minutes, 30 minutes means 3 missing values are interpolated, but not more.
+; If there shouldn't be any linear interpolation, set the limit to 0
+; For precipitation, the default is 1 hour
+LINEAR_INTERPOLATION_LIMIT_P = 1 hour
+; For temperature, the default is 2 days
+LINEAR_INTERPOLATION_LIMIT_T = 2 days
+; For Evapotranspiration, the default is 2 days
+LINEAR_INTERPOLATION_LIMIT_ET = 2 days
+
 
 [weatherdb:max_fillup_distance]
 ; The maximum distance in meters to use for the filling of the station data
 ; For each parameter (P, T, ET) the module uses a different distance
 ; Precipitation (P)
-P = 110000
+P = 130000
 ; Temperature (T)
 T = 150000
 ; Evapotranspiration (ET)

{weatherdb-1.1.2 → weatherdb-1.2.0}/weatherdb/station/StationBases.py

@@ -1491,6 +1491,70 @@ class StationBase:
                 extra_fillup_where=sql_format_dict["extra_fillup_where"] +\
                     ' OR ts."filled_by" IS DISTINCT FROM new."filled_by"'))
 
+        # linear interpolation for the last missing values
+        lr_limit = config.get(
+            "weatherdb",
+            "LINEAR_INTERPOLATION_LIMIT_{self._para_base}",
+            fallback="0")
+        if lr_limit != "0":
+            sql_format_dict.update(dict(
+                sql_linear_interpolation=f"""
+                    DO
+                    $do$
+                        DECLARE reg_borders record;
+                        BEGIN
+                            FOR reg_borders IN
+                                WITH empty_periods AS (
+                                    SELECT *
+                                    FROM (  SELECT
+                                                CASE WHEN dist_next>'{self._interval}'::interval
+                                                     THEN timestamp ELSE NULL
+                                                END AS start,
+                                                CASE WHEN dist_next>'{self._interval}'::interval
+                                                     THEN LEAD(timestamp) OVER (ORDER BY timestamp)
+                                                     ELSE NULL
+                                                END AS end
+                                            FROM (
+                                                SELECT *,
+                                                    timestamp - lag(timestamp, 1) OVER ( ORDER BY timestamp) AS dist_prev,
+                                                    lead(timestamp, 1) OVER ( ORDER BY timestamp) - timestamp AS dist_next
+                                                FROM new_filled_{self.id}_{self._para}
+                                                WHERE filled IS NOT NULL) t
+                                            WHERE t.dist_prev > '{self._interval}'::interval
+                                                  OR t.dist_next > '{self._interval}'::interval
+                                         ) p
+                                    WHERE p.start IS NOT NULL AND p.end IS NOT NULL)
+                                SELECT
+                                    ep.start AS timestamp_start,
+                                    ttss.filled AS value_start,
+                                    ep.end AS timestamp_end,
+                                    ttse.filled AS value_end,
+                                    (ttse.filled - ttss.filled)::numeric/(EXTRACT(EPOCH FROM (ep.end - ep.start))/EXTRACT(EPOCH FROM '{self._interval}'::interval))::numeric as slope
+                                FROM empty_periods ep
+                                LEFT JOIN new_filled_{self.id}_{self._para} ttss ON ep.start = ttss.timestamp
+                                LEFT JOIN new_filled_{self.id}_{self._para} ttse ON ep.end = ttse.timestamp
+                                where (ep.end - ep.start - '{self._interval}'::interval) <= '{lr_limit}'::interval
+                            loop
+                                execute FORMAT(
+                                $$
+                                UPDATE new_filled_{self.id}_{self._para} ts
+                                SET filled=%2$L + (EXTRACT(EPOCH FROM ts.timestamp - %1$L)::numeric/(EXTRACT(EPOCH FROM '{self._interval}'::interval))::numeric * %5$L),
+                                    filled_by=-1
+                                WHERE ts.timestamp>%1$L and ts.timestamp<%3$L;
+                                $$,
+                                reg_borders.timestamp_start,
+                                reg_borders.value_start,
+                                reg_borders.timestamp_end,
+                                reg_borders.value_end,
+                                reg_borders.slope
+                                );
+                            END loop;
+                        END
+                    $do$;"""))
+        else:
+            sql_format_dict.update(dict(sql_linear_interpolation=""))
+
+
         # Make SQL statement to fill the missing values with values from nearby stations
         sql = """
             CREATE TEMP TABLE new_filled_{stid}_{para}
@@ -1563,15 +1627,16 @@ class StationBase:
                         FROM new_filled_{stid}_{para}
                         WHERE "filled" IS NULL {extra_unfilled_period_where};
                     END LOOP;
-                    {sql_extra_after_loop}
-                    UPDATE timeseries."{stid}_{para}" ts
-                    SET filled = new.filled, {extra_cols_fillup}
-                        filled_by = new.filled_by
-                    FROM new_filled_{stid}_{para} new
-                    WHERE ts.timestamp = new.timestamp
-                        AND (ts."filled" IS DISTINCT FROM new."filled" {extra_fillup_where}) ;
                 END
             $do$;
+            {sql_linear_interpolation}
+            {sql_extra_after_loop}
+            UPDATE timeseries."{stid}_{para}" ts
+            SET filled = new.filled, {extra_cols_fillup}
+                filled_by = new.filled_by
+            FROM new_filled_{stid}_{para} new
+            WHERE ts.timestamp = new.timestamp
+                AND (ts."filled" IS DISTINCT FROM new."filled" {extra_fillup_where}) ;
         """.format(**sql_format_dict)
 
         # execute

{weatherdb-1.1.2 → weatherdb-1.2.0}/weatherdb/utils/get_data.py

@@ -301,7 +301,7 @@ def _download_dem_opentopo(
         extent=(5.3, 46.1, 15.6, 55.4),
         api_key=os.environ.get(
             "WEATHERDB_OPENTOPO_API_KEY",
-            fallback=keyring.get_password("weatherdb", "opentopo_api_key"))):
+            default=keyring.get_password("weatherdb", "opentopo_api_key"))):
     """Download the DEM data from the OpenTopography service.
 
     Get an API key from (OpenTopography)[https://portal.opentopography.org/] to use this service.
@@ -322,6 +322,7 @@ def _download_dem_opentopo(
         The API key for the OpenTopography service.
         If None the user will be asked.
         The default is to check if the environment variable "WEATHERDB_OPENTOPO_API_KEY" is set or if the keyring has a password for "weatherdb" and "opentopo_api_key".
+        If the value is a valid filepath the content of the file is used as the API key.
 
     Returns
     -------
@@ -332,6 +333,9 @@ def _download_dem_opentopo(
     if api_key is None:
         print("No API key for OpenTopography was given or found in the keyring or environment variable.")
         api_key = getpass("Please enter your API key for OpenTopography: ")
+    if Path(api_key).exists():
+        with open(api_key) as f:
+            api_key = f.read().strip()
 
     # make query
     w, s, e, n = extent

{weatherdb-1.1.2 → weatherdb-1.2.0}/weatherdb.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: weatherdb
-Version: 1.1.2
+Version: 1.2.0
 Summary: This is a package to work with and to create the Weather Database which handles, checks, fills and corrects DWD Weather Station data.
 Author-email: Max Schmit <max.schmit@hydrology.uni-freiburg.de>
 License:                     GNU GENERAL PUBLIC LICENSE

{weatherdb-1.1.2 → weatherdb-1.2.0}/weatherdb.egg-info/SOURCES.txt

@@ -38,12 +38,13 @@ docs/source/setup/Hosting.md
 docs/source/setup/Install.md
 docs/source/setup/Quickstart.md
 docs/source/setup/setup.rst
-weatherDB.egg-info/PKG-INFO
-weatherDB.egg-info/SOURCES.txt
-weatherDB.egg-info/dependency_links.txt
-weatherDB.egg-info/entry_points.txt
-weatherDB.egg-info/requires.txt
-weatherDB.egg-info/top_level.txt
+tests/test-data/test-data-config.ini
+tests/test-data/DEM/COP-DEM_GLO-30-DGED__2023_1_clipped.tif
+tests/test-data/DEM/README.md
+tests/test-data/DEM/eula_F.pdf
+tests/test-data/regionalisation/DWD-grid_ma_1991_2020_DGM25_clipped.tif
+tests/test-data/regionalisation/HYRAS_ma_1991_2020_DGM25_clipped.tif
+tests/test-data/regionalisation/README.md
 weatherdb/__init__.py
 weatherdb/_version.py
 weatherdb/broker.py