weatherdb 1.1.2__tar.gz → 1.2.1__tar.gz

{weatherdb-1.1.2 → weatherdb-1.2.1}/.gitlab-ci.yml

@@ -1,8 +1,9 @@
 stages:
   - test_single
   - test_matrix
+  - test_manual
   - build
-  - deploy
+  - release
 
 workflow:
   auto_cancel:
@@ -21,10 +22,10 @@ test_single:
   before_script:
     - python3 -m pip install --upgrade pip --root-user-action=ignore
     - python3 -m pip install unittest-xml-reporting --root-user-action=ignore
-  script:
     - python3 -m pip install .[optionals] --root-user-action=ignore
     - mkdir -p $WEATHERDB_DATA_BASE_DIR
     - mkdir -p test-reports
+  script:
     - python3 -m xmlrunner discover -s tests -p testSuite.py -o test-reports
   cache:
     - key: weatherdb-data-cache-$CI_COMMIT_REF_SLUG
@@ -61,11 +62,13 @@ test_single:
     WEATHERDB_LOGGING_HANDLER: console
     WEATHERDB_LOGGING_LEVEL: DEBUG
     WEATHERDB_HORIZON_RADIUS: 40000
+    WEATHERDB_OPENTOPO_API_KEY: $WEATHERDB_OPENTOPO_API_KEY
     WEATHERDB_TEST_ARTIFACT_DIR: $CI_PROJECT_DIR/artifacts
     WEATHERDB_TEST_ARTIFACT_DB_DUMP: ON_FAILURE
     WEATHERDB_TEST_ARTIFACT_LIST_DATA_FILES: ON_FAILURE
     WEATHERDB_TEST_ARTIFACT_COPY_USER_CONFIG: ON_FAILURE
-    WEATHERDB_OPENTOPO_API_KEY: $WEATHERDB_OPENTOPO_API_KEY
+    WEATHERDB_TEST_DR_COPY_RASTERS: True
+    WEATHERDB_TEST_DR_UPDATE_USER_CONFIG: True
     DOCKER_ENV: test
   tags:
     - docker
@@ -73,20 +76,32 @@ test_single:
 test_matrix:
   stage: test_matrix
   extends: test_single
-  dependencies:
-    - test_single
   parallel:
     matrix:
       - PY_VERSION: ["3.9","3.10","3.11","3.12","3.13"]
   variables:
     PYTHON_VERSION: $PY_VERSION
 
+test_download_rasters:
+  extends: test_single
+  stage: test_manual
+  rules:
+    - when: manual
+      allow_failure: true
+  script:
+    - python3 -m xmlrunner discover -s tests -p test_downloadRasters.py --complete -o test-reports
+  variables:
+    PYTHON_VERSION: $PY_VERSION
+  parallel:
+    matrix:
+      - PY_VERSION: ["3.9","3.10","3.11","3.12","3.13"]
+
 build:
+  stage: build
   rules:
-    - if: $CI_COMMIT_TAG =~ /^[Vv]\d+\.\d+\.\d+$/
+    - if: $CI_COMMIT_TAG =~ /^[Vv]*\d+\.\d+\.\d+$/
   dependencies:
     - test_matrix
-  stage: build
   image: python:3.11
   script:
     - python -m pip install --upgrade pip --root-user-action=ignore
@@ -100,12 +115,12 @@ build:
   tags:
     - docker
 
-deploy:
+release_pypi:
+  stage: release
   rules:
     - !reference [build, rules]
   dependencies:
     - build
-  stage: deploy
   image: python:3.11
   script:
     - python -m pip install --upgrade pip --root-user-action=ignore
@@ -116,4 +131,19 @@ deploy:
     TWINE_PASSWORD: $TWINE_PASSWORD
   tags:
     - docker
-  interruptible: false
+  interruptible: false
+
+release_gitlab:
+  stage: release
+  rules:
+    - !reference [build, rules]
+  image: registry.gitlab.com/gitlab-org/release-cli:latest
+  dependencies:
+    - build
+    - release_pypi
+  script:
+    - echo "running release_gitlab"
+  release:
+    name: '$CI_COMMIT_TAG'
+    tag_name: '$CI_COMMIT_TAG'
+    description: '$CI_COMMIT_TAG_MESSAGE'

{weatherdb-1.1.2 → weatherdb-1.2.1}/CHANGES.md

@@ -1,5 +1,22 @@
 # Change-log
 
+## Version 1.2.1
+
+- fix: filllup linear regression did not take configuration value and therefor did never work
+- config: change LINEAR_INTERPOLATION_LIMIT to be a category
+- cli: add version parameter
+- multiprocessing: make process count configurable through user configuration
+- broker: prevent broker inactivation by other broker
+- broker: add force deactivate all method to forcefully change the database flag of active brokers
+
+## Version 1.2.0
+
+> [!WARNING]
+> This versions update did have an error and the lienear regression doesn't work at all. PLease use version 1.2.1
+
+- tests: add test rasters and make downloading the rasters a manual job on GitLab as there were many problems
+- fillup: add method to linearly interpolate residual missing values
+
 ## Version 1.1.2
 
 - add method to download DEM data from OpenTopography-API
@@ -12,7 +29,10 @@
 
 ## Version 1.1.0
 
-- !!rename module from weatherDB to weatherdb to be compliant with PEP 8!!
+> [!NOTE]
+> **major change:** The module got renamed to `weatherdb`, you need to adapt this in your import and cli statements
+
+- rename module from weatherDB to weatherdb to be compliant with PEP 8
   So you need to import the module now with `import weatherdb`
 - fix some missing stationN variables to stationsP
 - fix: TimestampPeriod._check_period

{weatherdb-1.1.2 → weatherdb-1.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: weatherdb
-Version: 1.1.2
+Version: 1.2.1
 Summary: This is a package to work with and to create the Weather Database which handles, checks, fills and corrects DWD Weather Station data.
 Author-email: Max Schmit <max.schmit@hydrology.uni-freiburg.de>
 License:                     GNU GENERAL PUBLIC LICENSE

{weatherdb-1.1.2 → weatherdb-1.2.1}/docker/docker-compose.yaml

@@ -14,12 +14,14 @@ services:
       WEATHERDB_USER_CONFIG_FILE: /home/config_user.yaml
       WEATHERDB_DATA_BASE_DIR: /home/data
       WEATHERDB_HANDLE_NON_EXISTING_CONFIG: create
+      WEATHERDB_OPENTOPO_API_KEY: /run/secrets/opentopo_api_key
       DOCKER_ENV: main
     volumes:
       - weatherdb_home:/home
     command: ["sleep", "infinity"] # to keep awake
     secrets:
       - db_password
+      - opentopo_api_key
     develop:
       watch:
         - action: rebuild
@@ -53,6 +55,8 @@ volumes:
 secrets:
   db_password:
     file: ./db_password.docker-secret
+  opentopo_api_key:
+    file: ./opentopo_api_key.docker-secret
 
 # start from parent folder with `docker compose -f docker\\docker-compose.yaml up --build`
-# To connect to ther weatherdb service use `docker-compose exec weatherdb bash`
+# To connect to the weatherdb service use `docker-compose exec weatherdb bash`

{weatherdb-1.1.2 → weatherdb-1.2.1}/docs/source/Methode.md

@@ -117,7 +117,11 @@ N_{neighbor} * \dfrac{N_{station,ma,winter}}{N_{neighbor,ma,winter}} \space if\s
 N_{neighbor} * \dfrac{N_{station,ma,summer}}{N_{neighbor,ma,summer}} \space if\space month\notin[4:9]
 \end{cases}$
 
-For the precipitation stations only stations within a 100 km radius are taken to fill missing values. For the potential Evapotranspiration and the temperature this radius is 150km. For the temperature stations the median of the regionalised values from the 5 closest stations (but not more than 150 km away) is taken to fill missing values.
+For the precipitation stations only stations within a 110 km radius are taken to fill missing values. For the potential Evapotranspiration and the temperature this radius is 150km. For the temperature stations the median of the regionalised values from the 5 closest stations (but not more than 150 km away) is taken to fill missing values. Those limits can get configured in the user configurations.
+
+### linear interpolation
+
+After missing values got filled with regionalised values from the neighboring stations there can still be missing values left. To fill those residual missing values, they get interpolated linearly. Those linear interpolations have configurable limits for how big the interval of missing values can be. The default values are 1h for precipitation and 2 days for temperature and evapotranspiration.
 
 ### adjusting precipitation to daily station measurements
 

weatherdb-1.2.1/docs/source/api/weatherdb.broker.rst

@@ -0,0 +1,7 @@
+broker
+------
+
+.. autoclass:: weatherdb.broker.Broker
+
+
+

{weatherdb-1.1.2 → weatherdb-1.2.1}/docs/source/api/weatherdb.db.rst

@@ -19,5 +19,15 @@ base models
 
 .. autoclass:: weatherdb.db.models.MetaBaseQC
 
+views
+-----
 
+Those are the database views
+
+.. automodule:: weatherdb.db.views
+
+connections
+-----------
+
+.. automodule:: weatherdb.db.connections
 

{weatherdb-1.1.2 → weatherdb-1.2.1}/docs/source/api/weatherdb.rst

@@ -1,7 +1,6 @@
 weatherdb
 =========
 
-
 .. toctree::
    :maxdepth: 2
 

weatherdb-1.2.1/docs/source/api/weatherdb.station.GroupStation.rst

@@ -0,0 +1,5 @@
+GroupStation
+============
+
+.. py:module:: weatherdb.station
+.. autoclass:: weatherdb.station.GroupStation

weatherdb-1.2.1/docs/source/api/weatherdb.station.StationBases.rst

@@ -0,0 +1,18 @@
+StationBases
+============
+
+Those are the base station classes on which the real station classes above depend on.
+None of them is working on its own, because the class variables are not yet set correctly.
+
+.. py:module:: weatherdb.station.StationBases
+
+.. autoclass:: weatherdb.station.StationBases.StationBase
+
+
+.. autoclass:: weatherdb.station.StationBases.StationPBase
+
+
+.. autoclass:: weatherdb.station.StationBases.StationCanVirtualBase
+
+
+.. autoclass:: weatherdb.station.StationBases.StationTETBase

weatherdb-1.2.1/docs/source/api/weatherdb.station.StationET.rst

@@ -0,0 +1,5 @@
+StationET
+=========
+
+.. py:module:: weatherdb.station
+.. autoclass:: weatherdb.station.StationET

weatherdb-1.2.1/docs/source/api/weatherdb.station.StationP.rst

@@ -0,0 +1,5 @@
+StationP
+========
+
+.. py:module:: weatherdb.station
+.. autoclass:: weatherdb.station.StationP

weatherdb-1.2.1/docs/source/api/weatherdb.station.StationPD.rst

@@ -0,0 +1,6 @@
+StationPD
+=========
+
+.. py:module:: weatherdb.station
+.. autoclass:: weatherdb.station.StationPD
+  :exclude-members: quality_check, last_imp_quality_check, get_corr, get_adj, get_qc

weatherdb-1.2.1/docs/source/api/weatherdb.station.StationT.rst

@@ -0,0 +1,5 @@
+StationT
+========
+
+.. py:module:: weatherdb.station
+.. autoclass:: weatherdb.station.StationT

weatherdb-1.2.1/docs/source/api/weatherdb.station.rst

@@ -0,0 +1,17 @@
+station
+--------
+
+.. py:module:: weatherdb.station
+
+.. toctree::
+   :maxdepth: 1
+   :titlesonly:
+
+   weatherdb.station.StationP
+   weatherdb.station.StationT
+   weatherdb.station.StationET
+   weatherdb.station.StationPD
+   weatherdb.station.GroupStation
+   weatherdb.station.StationBases
+
+

weatherdb-1.2.1/docs/source/api/weatherdb.stations.StationsBase.rst

@@ -0,0 +1,15 @@
+StationsBase
+============
+
+Those are the base stations classes on which the real stations classes above depend on.
+None of them is working on its own, because the class variables are not yet set correctly.
+
+.. py:module:: weatherdb.stations.StationsBase
+
+.. autoclass:: weatherdb.stations.StationsBase.StationsBase
+
+  .. autoclasstoc::
+
+.. autoclass:: weatherdb.stations.StationsBaseTET.StationsBaseTET
+
+   .. autoclasstoc::

weatherdb-1.2.1/docs/source/api/weatherdb.stations.StationsET.rst

@@ -0,0 +1,6 @@
+StationsET
+==========
+
+.. py:module:: weatherdb.stations
+
+.. autoclass:: weatherdb.stations.StationsET

weatherdb-1.2.1/docs/source/api/weatherdb.stations.StationsP.rst

@@ -0,0 +1,6 @@
+StationsP
+=========
+
+.. py:module:: weatherdb.stations
+
+.. autoclass:: weatherdb.stations.StationsP

weatherdb-1.2.1/docs/source/api/weatherdb.stations.StationsPD.rst

@@ -0,0 +1,7 @@
+StationsPD
+==========
+
+.. py:module:: weatherdb.stations
+
+.. autoclass:: weatherdb.stations.StationsP
+

weatherdb-1.2.1/docs/source/api/weatherdb.stations.StationsT.rst

@@ -0,0 +1,7 @@
+StationsT
+=========
+
+.. py:module:: weatherdb.stations
+
+.. autoclass:: weatherdb.stations.StationsT
+

weatherdb-1.2.1/docs/source/api/weatherdb.stations.rst

@@ -0,0 +1,15 @@
+stations
+--------
+
+.. py:module:: weatherdb.stations
+
+.. toctree::
+   :maxdepth: 1
+   :titlesonly:
+
+   weatherdb.stations.StationsP
+   weatherdb.stations.StationsT
+   weatherdb.stations.StationsET
+   weatherdb.stations.StationsPD
+   weatherdb.stations.GroupStations
+   weatherdb.stations.StationsBase

{weatherdb-1.1.2 → weatherdb-1.2.1}/docs/source/api/weatherdb.utils.rst

@@ -3,6 +3,8 @@ utils
 
 In this module there are several utilities used by the WeatherDB package, you shouldn't need to use them directly.
 
+TimestampPeriod
+---------------
 .. autoclass:: weatherdb.utils.TimestampPeriod
 
 dwd

{weatherdb-1.1.2 → weatherdb-1.2.1}/docs/source/conf.py

@@ -84,6 +84,9 @@ autoclasstoc_sections = [
 # Autosummary options
 autosummary_generate = True
 autosummary_generate_overwrite = True
+suppress_warnings = [
+    'autosummary.import_cycle',
+]
 
 # intersphinx
 intersphinx_mapping = {

{weatherdb-1.1.2 → weatherdb-1.2.1}/docs/source/setup/Quickstart.md

@@ -46,7 +46,8 @@ So there is:
 - "raw" : the raw measurements as on the DWD server
 - "qc"  : The quality checked data
 - "filled" : The filled timeseries
-- "filled_by" : The station ID of the station from which the data was taken to fill the measurements
+- "filled_by" : The station ID of the station from which the data was taken to fill the measurements.
+                If the value was filled by linear intrepolation "filled_by" is `-1`
 - "corr"    : The Richter corrected timeserie.
 
 If you want more than just one kind of timeseries, e.g. the filled timeseries, together with the id from which station the respective field got filled with use:

weatherdb-1.2.1/tests/test-data/DEM/README.md

@@ -0,0 +1,3 @@
+# Readme
+
+The DEM file in this folder originates from Copernicus and got clipped to the region of analysis in the test environment + 40km

weatherdb-1.2.1/tests/test-data/regionalisation/README.md

@@ -0,0 +1,3 @@
+# Readme
+
+The files in this folder originate from https://zenodo.org/records/10066045 and got clipped to the region of analysis in the test environment

weatherdb-1.2.1/tests/test-data/test-data-config.ini

@@ -0,0 +1,28 @@
+; Those are configs that should get loaded, when the test files are loaded
+[data:rasters]
+DEMS = ${data:BASE_DIR}/DEM/COP-DEM_GLO-30-DGED__2023_1_clipped.tif
+
+[data:rasters:hyras]
+FILE = ${data:BASE_DIR}/regionalisation/HYRAS_ma_1991_2020_DGM25_clipped.tif
+BAND_P_WIHY = n_hyras_wihj
+BAND_P_SUHY = n_hyras_sohj
+BAND_P_YEAR = n_hyras_year
+SRID = 3035
+FACTOR_P_WIHY = 1
+FACTOR_P_SUHY = 1
+FACTOR_P_YEAR = 1
+
+[data:rasters:dwd]
+FILE = ${data:BASE_DIR}/regionalisation/DWD-grid_ma_1991_2020_DGM25_clipped.tif
+BAND_P_WIHY = n_wihj
+BAND_P_SUHY = n_sohj
+BAND_P_YEAR = n_year
+BAND_T_YEAR = t_year
+BAND_ET_YEAR = et_year
+SRID = 3035
+FACTOR_P_WIHY = 1
+FACTOR_P_SUHY = 1
+FACTOR_P_YEAR = 1
+FACTOR_T_YEAR = 0.1
+FACTOR_ET_YEAR = 1
+

weatherdb-1.2.1/weatherdb/_version.py

@@ -0,0 +1 @@
+__version__ = "1.2.1"

{weatherdb-1.1.2 → weatherdb-1.2.1}/weatherdb/broker.py

@@ -647,8 +647,9 @@ class Broker(object):
         is_active : bool
             Whether the broker is active.
         """
+        if self._is_active != is_active:
+            self.set_setting("broker_active", str(is_active))
         self._is_active = is_active
-        self.set_setting("broker_active", str(is_active))
 
     def _deactivate(self):
         self.is_active = False
@@ -664,4 +665,13 @@ class Broker(object):
             yield self
         finally:
             self._deactivate()
-            atexit.unregister(self._deactivate)
+            atexit.unregister(self._deactivate)
+
+    def force_deactivate_all(self):
+        """Forcefully set the active broker flag in the database to deactivated.
+
+        This is useful if the broker got exited before it could deactivate itself.
+        """
+        if self.is_active:
+            self._deactivate()
+        self.set_setting("broker_active", "False")

{weatherdb-1.1.2 → weatherdb-1.2.1}/weatherdb/cli.py

@@ -16,6 +16,7 @@ import weatherdb
                  show_default=True
                  )
             )
+@click.version_option(weatherdb.__version__)
 @click.option('--do-logging/--no-logging',
               is_flag=True, default=True, show_default=True,
               help="Should the logging be done to the console?")
@@ -98,8 +99,11 @@ def create_user_config(file, on_exists):
 def download_ma_rasters(which, overwrite, update_user_config):
     """Get the multi annual rasters on which bases the regionalisation is done.
 
-    The refined multi annual datasets, that are downloaded are published on Zenodo:
-    Schmit, M.; Weiler, M. (2023). German weather services (DWD) multi annual meteorological rasters for the climate period 1991-2020 refined to 25m grid (1.0.0) [Data set]. Zenodo. https://doi.org/10.5281/zenodo.10066045
+    The refined multi annual datasets, that are downloaded are published on Zenodo. [1]_
+
+    References:
+    -----------
+    .. [1]  Schmit, M.; Weiler, M. (2023). German weather services (DWD) multi annual meteorological rasters for the climate period 1991-2020 refined to 25m grid (1.0.0) [Data set]. Zenodo. `DOI:10.5281/zenodo.10066045 <https://doi.org/10.5281/zenodo.10066045>`_
     """
     click.echo("starting downloading multi annual raster data")
     from weatherdb.utils.get_data import download_ma_rasters
@@ -127,7 +131,7 @@ def download_ma_rasters(which, overwrite, update_user_config):
               help="The service to use to download the DEM. Options are 'prism' or 'openTopography'. " +\
                    "You can use this option muultiple times to test both in the given order until the file could be downloaded.")
 def download_dem(out_dir, overwrite, extent, update_user_config, service="prism"):
-    """Download the newest DEM data from the Copernicus Sentinel dataset.
+    """Download the newest DEM data from the Copernicus Sentinel dataset. [1]_
 
     Only the GLO-30 DEM, wich has a 30m resolution, is downloaded as it is freely available.
     If you register as a scientific researcher also the EEA-10, with 10 m resolution, is available.
@@ -136,8 +140,9 @@ def download_dem(out_dir, overwrite, extent, update_user_config, service="prism"
     After downloading the data, the files are merged and saved as a single tif file in the data directory in a subfolder called 'dems'.
     To use the DEM data in the WeatherDB, you will have to define the path to the tif file in the configuration file.
 
-    Source:
-    Copernicus DEM - Global and European Digital Elevation Model. Digital Surface Model (DSM) provided in 3 different resolutions (90m, 30m, 10m) with varying geographical extent (EEA: European and GLO: global) and varying format (INSPIRE, DGED, DTED). DOI:10.5270/ESA-c5d3d65.
+    References:
+    -----------
+    .. [1] Copernicus DEM - Global and European Digital Elevation Model. Digital Surface Model (DSM) provided in 3 different resolutions (90m, 30m, 10m) with varying geographical extent (EEA: European and GLO: global) and varying format (INSPIRE, DGED, DTED). `DOI:10.5270/ESA-c5d3d65 <https://doi.org/10.5270/ESA-c5d3d65>`_
     """
     click.echo("Starting downloading digital elevation model from Copernicus")
     from weatherdb.utils.get_data import download_dem
@@ -221,6 +226,16 @@ def set_db_version():
     else:
         click.echo("aborting setting db version")
 
+@cli.command(short_help="Forcefully set the active broker flag in the database to deactivated. This is useful if the broker got exited before it could deactivate itself. (!!!Only use this if you're sure that the database did all the necessary updates!!!)")
+def force_deactivate_all_broker():
+    click.echo(dedent("""
+        Are you sure that there is no more broker running? This could lead to problems as multiple brokers could run at the same time contradicting themself."""))
+    if click.confirm("Are you sure you want to set the activation flag in the database to deactivated?"):
+        click.echo("deactivating the brokers flag")
+        broker = weatherdb.broker.Broker()
+        broker.force_deactivate_all()
+    else:
+        click.echo("aborting the deactivation of the brokers flag")
 
 # cli
 # ---------------------------------------

{weatherdb-1.1.2 → weatherdb-1.2.1}/weatherdb/config/config_default.ini

@@ -143,17 +143,46 @@ RASTER_BUFFER_CRS = ${weatherdb:HORIZON_CRS}
 ; The default is 1999-01-01
 MIN_DATE = 1999-01-01
 
+[weatherdb:LINEAR_INTERPOLATION_LIMIT]
+; When there are still NAs after filling with neighboring stations, the module can use linear interpolation to fill the gaps
+; The limit defines the maximum interval that is allowed to fill with linear interpolation
+; The value is given with a unit, e.g. 3 days, 30 minutes, 1 hour
+; For further explanation about the format see https://www.postgresql.org/docs/current/datatype-datetime.html#DATATYPE-INTERVAL-INPUT
+; for precipitation, where the timeseries resolution is 10 minutes, 30 minutes means 3 missing values are interpolated, but not more.
+; If there shouldn't be any linear interpolation, set the limit to 0
+; For precipitation, the default is 1 hour
+P = 1 hour
+; For temperature, the default is 2 days
+T = 2 days
+; For Evapotranspiration, the default is 2 days
+ET = 2 days
+
 
 [weatherdb:max_fillup_distance]
 ; The maximum distance in meters to use for the filling of the station data
 ; For each parameter (P, T, ET) the module uses a different distance
 ; Precipitation (P)
-P = 110000
+P = 130000
 ; Temperature (T)
 T = 150000
 ; Evapotranspiration (ET)
 ET = 150000
 
+
+[system:parallel_processes]
+; You can configure how many processes are getting started in parrallel when tghe database is updated.
+; depending on your system, the database can handle more or less processes
+; There  are 2 possible ways to configure this.
+; either define the absolute number of processes:
+; ABSOLUTE =
+
+; or define the number of processes in relation to the number of CPUs,
+; by defining how many CPUs should be used less than the maximum available CPUs
+; if the absolute number is defined, the relative number is ignored
+; This is the default setting of cpu count -1
+CPU_COUNT_MINUS = 1
+
+
 ; for developpers:
 ; ----------------
 ;