weatherdb 1.2.3__tar.gz → 1.2.5__tar.gz

{weatherdb-1.2.3 → weatherdb-1.2.5}/.gitlab-ci.yml

@@ -12,8 +12,9 @@ workflow:
 test_single:
   stage: test_single
   rules:
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_TAG == null
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_COMMIT_TAG =~ /^[Vv]*\d+\.\d+\.\d+$/
   image: "python:$PYTHON_VERSION"
   services:
     - name: timescale/timescaledb-ha:pg17
@@ -144,4 +145,46 @@ release_gitlab:
   release:
     name: '$CI_COMMIT_TAG'
     tag_name: '$CI_COMMIT_TAG'
-    description: '$CI_COMMIT_TAG_MESSAGE'
+    description: '$CI_COMMIT_TAG_MESSAGE'
+
+# deploying to Zenodo
+include:
+  - remote: 'https://gitlab.com/deploy2zenodo/deploy2zenodo/-/releases/permalink/latest/downloads/deploy2zenodo.yaml'
+
+deploy2zenodo:
+  stage: release
+  rules:
+    - !reference [build, rules]
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: never
+    - when: manual
+      allow_failure: true
+  needs:
+    - job: release_pypi
+      optional: true
+  variables:
+    DEPLOY2ZENODO_JSON: zenodo.json
+    DEPLOY2ZENODO_ADD_IsNewVersionOf: yes
+    DEPLOY2ZENODO_GET_METADATA: "result.json"
+  before_script:
+    - apk add --no-cache curl jq git pandoc
+    # pack
+    - TAG=$(echo $CI_COMMIT_TAG | cut -d "v" -f 2)
+    - if [ -z "$TAG" ]; then echo "TAG is empty, exiting"; exit 1; fi
+    - DEPLOY2ZENODO_UPLOAD=v$TAG.zip
+    - git archive --format zip --output "$DEPLOY2ZENODO_UPLOAD" $CI_COMMIT_SHA
+    # prepare zenodo.json
+    - publication_date=$(echo "$CI_COMMIT_TIMESTAMP" | grep -Eo "^[0-9]{4}-[0-9]{2}-[0-9]{2}")
+    - pandoc -o README.html README.md
+    - tmpjson="$(mktemp)"
+    - |
+        jq . $DEPLOY2ZENODO_JSON | \
+        jq ".metadata.version = \"$TAG\"" | \
+        jq ".metadata.publication_date = \"$publication_date\"" | \
+        jq --rawfile README README.html '.metadata.description = $README' | \
+        jq . | tee "$tmpjson"
+    - mv "$tmpjson" "$DEPLOY2ZENODO_JSON"
+  artifacts:
+    paths:
+      - $DEPLOY2ZENODO_JSON
+      - $DEPLOY2ZENODO_GET_METADATA

{weatherdb-1.2.3 → weatherdb-1.2.5}/CHANGES.md

@@ -1,5 +1,19 @@
 # Change-log
 
+## Version 1.2.5
+
+- cli updating methods have now additional parameters for stids, period and para to be able to update only a some specific stations
+- fillup: update also Temperature values if the filled_max and filled_min are NULL.
+
+## Version 1.2.4
+
+- added Zenodo upload
+- fix problem with creating user config
+- add format placeholders for GroupStations.create_ts for file_names parameter
+- translate Parameter name for Precipitation for create_roger_ts back to german names to work with actual RoGeR version
+- fix problem when create_ts is invoked with a zip file. 
+  Previously the precipitation timeseries did not get expanded to the last complete date and stopped at 00:00 o'clock
+
 ## Version 1.2.3
 
 - fix: dropping stations

{weatherdb-1.2.3 → weatherdb-1.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: weatherdb
-Version: 1.2.3
+Version: 1.2.5
 Summary: This is a package to work with and to create the Weather Database which handles, checks, fills and corrects DWD Weather Station data.
 Author-email: Max Schmit <max.schmit@hydrology.uni-freiburg.de>
 License:                     GNU GENERAL PUBLIC LICENSE
@@ -714,6 +714,7 @@ Requires-Dist: setuptools_scm
 Provides-Extra: optionals
 Requires-Dist: coloredlogs; extra == "optionals"
 Requires-Dist: colorama; extra == "optionals"
+Dynamic: license-file
 
 # WeatherDB - module
 

weatherdb-1.2.5/weatherdb/_version.py

@@ -0,0 +1 @@
+__version__ = "1.2.5"

{weatherdb-1.2.3 → weatherdb-1.2.5}/weatherdb/cli.py

@@ -7,6 +7,43 @@ from textwrap import dedent
 sys.path.insert(0, Path(__file__).resolve().parent.parent.as_posix())
 import weatherdb
 
+# Reusable option decorators
+# ---------------------------------------
+
+def period_options(f):
+    """Add period options only"""
+    f = click.option("--period_from", "-f",
+                     default=None, show_default=True,
+                     help="The from timestamp of the period to apply the method on. Timestamp should be in the format YYYY-MM-DD.")(f)
+    f = click.option("--period_until", "-u",
+                     default=None, show_default=True,
+                     help="The until timestamp of the period to apply the method on. Timestamp should be in the format YYYY-MM-DD.")(f)
+    return f
+
+def stid_option(f):
+    """Add station ID options"""
+    f = click.option("--stid", "-i",
+                     default=["all"], show_default=True, multiple=True,
+                     callback=lambda ctx, param, value: "all" if "all" in value else [int(v) for v in value],
+                     help="The station IDs to apply the method to. Options are 'all' or a specific station ID. " +
+                          "You can enter multiple values to add multiple station IDs.")(f)
+    return f
+
+def para_options(f):
+    """Add parameter options"""
+    f = click.option("--para", "-p",
+                     default=["p", "t", "et"], show_default=True, multiple=True,
+                     help="The parameters to work with. Options are 'p', 't' and 'et'. " +
+                          "You can enter multiple values to add multiple parameters.")(f)
+    return f
+
+def common_update_options(f):
+    """Add common options for processing commands (para, stid, period)"""
+    f = para_options(f)
+    f = stid_option(f)
+    f = period_options(f)
+    return f
+
 # main cli group
 # ---------------------------------------
 
@@ -185,31 +222,43 @@ def update_ma_raster():
 
 
 @cli.command(short_help="Update the raw data of the complete database.")
-def update_raw():
+@common_update_options
+def update_raw(para, stid, period_from, period_until):
     click.echo("starting updating the raw data")
     broker = weatherdb.broker.Broker()
-    broker.update_raw()
+    broker.update_raw(paras=para,
+                      stids=stid,
+                      period=(period_from, period_until))
 
 
 @cli.command(short_help="Do the quality check of the complete database.")
-def quality_check():
+@common_update_options
+def quality_check(para, stid, period_from, period_until):
     click.echo("starting quality check")
     broker = weatherdb.broker.Broker()
-    broker.quality_check()
+    broker.quality_check(paras=para,
+                         stids=stid,
+                         period=(period_from, period_until))
 
 
 @cli.command(short_help="Do the filling of the complete database.")
-def fillup():
+@common_update_options
+def fillup(para, stid, period_from, period_until):
     click.echo("starting filling up")
     broker = weatherdb.broker.Broker()
-    broker.fillup()
+    broker.fillup(paras=para,
+                  stids=stid,
+                  period=(period_from, period_until))
 
 
 @cli.command(short_help="Do the richter correction of the complete database.")
-def richter_correct():
+@stid_option
+@period_options
+def richter_correct(stid, period_from, period_until):
     click.echo("starting richter correction")
     broker = weatherdb.broker.Broker()
-    broker.richter_correct()
+    broker.richter_correct(stids=stid,
+                           period=(period_from, period_until))
 
 
 # cli admin stuff

{weatherdb-1.2.3 → weatherdb-1.2.5}/weatherdb/config/ConfigParser.py

@@ -357,7 +357,7 @@ class ConfigParser(configparser.ConfigParser):
                 user_config_file = "ask"
 
         # ask for the user config file
-        if user_config_file == "ask":
+        if user_config_file.lower().strip()[0] == "a":
             try:
                 from tkinter import Tk
                 from tkinter import filedialog

{weatherdb-1.2.3 → weatherdb-1.2.5}/weatherdb/station/GroupStation.py

@@ -344,8 +344,13 @@ class GroupStation(object):
     def get_name(self):
         return self.station_parts[0].get_name()
 
-    def create_roger_ts(self, dir, period=(None, None),
-                        kind="best", r_r0=1, add_t_min=False, add_t_max=False,
+    def create_roger_ts(self,
+                        dir,
+                        period=(None, None),
+                        kind="best",
+                        r_r0=1,
+                        add_t_min=False,
+                        add_t_max=False,
                         do_toolbox_format=False,
                         **kwargs):
         """Create the timeserie files for roger as csv.
@@ -394,10 +399,15 @@ class GroupStation(object):
         """
         if do_toolbox_format:
             return self.create_ts(
-                dir=dir, period=period, kinds=kind,
-                agg_to="10 min", r_r0=r_r0, split_date=True,
+                dir=dir,
+                period=period,
+                kinds=kind,
+                agg_to="10 min",
+                r_r0=r_r0,
+                split_date=True,
                 nas_allowed=False,
-                add_t_min=add_t_min, add_t_max=add_t_max,
+                add_t_min=add_t_min,
+                add_t_max=add_t_max,
                 file_names={"P":"PREC.txt", "T":"TA.txt", "ET":"PET.txt"},
                 col_names={"P":"PREC", "ET":"PET",
                            "T":"TA", "T_min":"TA_min", "T_max":"TA_max",
@@ -408,18 +418,34 @@ class GroupStation(object):
                 **kwargs)
         else:
             return self.create_ts(
-                dir=dir, period=period, kinds=kind,
-                agg_to="10 min", r_r0=r_r0, split_date=True,
+                dir=dir,
+                period=period,
+                kinds=kind,
+                agg_to="10 min",
+                r_r0=r_r0,
+                split_date=True,
                 nas_allowed=False,
-                add_t_min=add_t_min, add_t_max=add_t_max,
+                file_names={"P":"N_{id:0>5}.txt"},
+                col_names={"P":"N"},
+                add_t_min=add_t_min,
+                add_t_max=add_t_max,
                 **kwargs)
 
-    def create_ts(self, dir, period=(None, None),
-                  kinds="best", paras="all",
-                  agg_to="10 min", r_r0=None, split_date=False,
-                  nas_allowed=True, add_na_share=False,
-                  add_t_min=False, add_t_max=False,
-                  add_meta=True, file_names={}, col_names={},
+    def create_ts(self,
+                  dir,
+                  period=(None, None),
+                  kinds="best",
+                  paras="all",
+                  agg_to="10 min",
+                  r_r0=None,
+                  split_date=False,
+                  nas_allowed=True,
+                  add_na_share=False,
+                  add_t_min=False,
+                  add_t_max=False,
+                  add_meta=True,
+                  file_names={},
+                  col_names={},
                   keep_date_parts=False,
                   **kwargs):
         """Create the timeserie files as csv.
@@ -485,6 +511,7 @@ class GroupStation(object):
         file_names : dict, optional
             A dictionary with the file names for the different parameters.
             e.g.{"P":"PREC.txt", "T":"TA.txt", "ET":"ET.txt"}
+            Additionally, the station ID ("{id}" as int) and period ({period} as TimestampPeriod) are available as placeholders.
             If an empty dictionary is given, then the standard names are used.
             The default is {}.
         col_names : dict, optional
@@ -536,19 +563,22 @@ class GroupStation(object):
             del kwargs["_skip_period_check"]
 
         # prepare loop
-        name_suffix = "_{stid:0>5}.txt".format(stid=self.id)
+        name_suffix = "_{id:0>5}.txt".format(id=self.id)
         x, y = self.get_geom().coords.xy
-        name = self.get_name() + " (ID: {stid})".format(stid=self.id)
+        name = self.get_name() + " (ID: {id})".format(id=self.id)
         do_zip = isinstance(dir, zipfile.ZipFile)
 
         for para in paras:
             # get the timeserie
             df = self.get_df(
-                period=period, kinds=kinds,
-                paras=[para], agg_to=agg_to,
+                period=period,
+                kinds=kinds,
+                paras=[para],
+                agg_to=agg_to,
                 nas_allowed=nas_allowed,
                 add_na_share=add_na_share,
-                add_t_min=add_t_min, add_t_max=add_t_max,
+                add_t_min=add_t_min,
+                add_t_max=add_t_max,
                 _skip_period_check=True,
                 **kwargs)
 
@@ -573,8 +603,7 @@ class GroupStation(object):
             # check for NAs
             filled_cols = [col for col in df.columns if "filled_by" in col]
             if not nas_allowed and df.drop(filled_cols, axis=1).isna().sum().sum() > 0:
-                warnings.warn("There were NAs in the timeserie for Station {stid}.".format(
-                    stid=self.id))
+                warnings.warn(f"There were NAs in the timeseries for Station {self.id}.")
 
             # special operations for et
             if para == "et" and r_r0 is not None:
@@ -609,9 +638,11 @@ class GroupStation(object):
 
             # get file name
             if para.upper() in file_names:
-                file_name = file_names[para.upper()]
+                file_name = file_names[para.upper()]\
+                    .format(id=self.id, period=period)
             elif para in file_names:
-                file_name = file_names[para]
+                file_name = file_names[para]\
+                    .format(id=self.id, period=period)
             else:
                 file_name = para.upper() + name_suffix
 

{weatherdb-1.2.3 → weatherdb-1.2.5}/weatherdb/station/StationBases.py

@@ -1447,7 +1447,8 @@ class StationBase:
                     {sql_format_dict["extra_exec_cols"].format(i=i)}
                     filled_by[{i}]=%1$s
                 FROM timeseries.%2$I nb
-                WHERE nf.filled IS NULL AND nf.nb_mean[{i}] IS NULL {prev_check}
+                WHERE (nf.filled IS NULL {sql_format_dict["extra_fillup_update_where"]})
+                    AND nf.nb_mean[{i}] IS NULL {prev_check}
                     AND nf.timestamp = nb.timestamp;"""
                 prev_check += f" AND nf.nb_mean[{i}] IS NOT NULL AND nf.filled_by[{i}] != %1$s"
 
@@ -1689,6 +1690,7 @@ class StationBase:
                 "extra_fillup_where": "",
                 "mul_elev_order": "",
                 "extra_exec_cols": "",
+                "extra_fillup_update_where": "",
                 "extra_after_loop_extra_col": ""}
 
     @db_engine.deco_update_privilege

{weatherdb-1.2.3 → weatherdb-1.2.5}/weatherdb/station/StationT.py

@@ -135,6 +135,7 @@ class StationT(StationTETBase):
                                   ' OR ts."filled_max" IS DISTINCT FROM new."filled_max"',
             "extra_exec_cols": "nb_max[{i}]=round(nb.raw_max + %3$s, 0)::int,"+
                                "nb_min[{i}]=round(nb.raw_min + %3$s, 0)::int,",
+            "extra_fillup_update_where": ' OR nf."filled_min" IS NULL OR nf."filled_max" IS NULL',
             "extra_after_loop_extra_col": """,
                 filled_min=(SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY v)
                             FROM unnest(nb_min) as T(v)),

{weatherdb-1.2.3 → weatherdb-1.2.5}/weatherdb/stations/GroupStations.py

@@ -293,10 +293,17 @@ class GroupStations(object):
 
         return stations
 
-    def create_ts(self, dir, period=(None, None), kinds="best",
-                  stids="all", agg_to="10 min", r_r0=None, split_date=False,
-                  nas_allowed=True, add_na_share=False,
-                  add_t_min=False, add_t_max=False,
+    def create_ts(self, dir,
+                  period=(None, None),
+                  kinds="best",
+                  stids="all",
+                  agg_to="10 min",
+                  r_r0=None,
+                  split_date=False,
+                  nas_allowed=True,
+                  add_na_share=False,
+                  add_t_min=False,
+                  add_t_max=False,
                   **kwargs):
         """Download and create the weather tables as csv files.
 
@@ -393,7 +400,6 @@ class GroupStations(object):
                         add_na_share=add_na_share,
                         add_t_min=add_t_min,
                         add_t_max=add_t_max,
-                        _skip_period_check=True,
                         **kwargs)
                     pbar.variables["last_station"] = stat.id
                     pbar.update(pbar.value + 1)
@@ -498,10 +504,16 @@ class GroupStations(object):
         """
         if do_toolbox_format:
             return self.create_ts(
-                dir=dir, period=period, kinds=kind,
-                agg_to="10 min", r_r0=r_r0, stids=stids,
-                split_date=True, nas_allowed=False,
-                add_t_min=add_t_min, add_t_max=add_t_max,
+                dir=dir,
+                period=period,
+                kinds=kind,
+                agg_to="10 min",
+                r_r0=r_r0,
+                stids=stids,
+                split_date=True,
+                nas_allowed=False,
+                add_t_min=add_t_min,
+                add_t_max=add_t_max,
                 file_names={"P":"PREC.txt", "T":"TA.txt", "ET":"PET.txt"},
                 col_names={"P":"PREC", "ET":"PET",
                            "T":"TA", "T_min":"TA_min", "T_max":"TA_max",
@@ -512,8 +524,16 @@ class GroupStations(object):
                 **kwargs)
         else:
             return self.create_ts(
-                dir=dir, period=period, kinds=kind,
-                agg_to="10 min", r_r0=r_r0, stids=stids,
-                split_date=True, nas_allowed=False,
-                add_t_min=add_t_min, add_t_max=add_t_max,
+                dir=dir,
+                period=period,
+                kinds=kind,
+                agg_to="10 min",
+                r_r0=r_r0,
+                stids=stids,
+                split_date=True,
+                nas_allowed=False,
+                file_names={"P": "N_{id:0>5}.txt"},
+                col_names={"P":"N"},
+                add_t_min=add_t_min,
+                add_t_max=add_t_max,
                 **kwargs)

{weatherdb-1.2.3 → weatherdb-1.2.5}/weatherdb.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: weatherdb
-Version: 1.2.3
+Version: 1.2.5
 Summary: This is a package to work with and to create the Weather Database which handles, checks, fills and corrects DWD Weather Station data.
 Author-email: Max Schmit <max.schmit@hydrology.uni-freiburg.de>
 License:                     GNU GENERAL PUBLIC LICENSE
@@ -714,6 +714,7 @@ Requires-Dist: setuptools_scm
 Provides-Extra: optionals
 Requires-Dist: coloredlogs; extra == "optionals"
 Requires-Dist: colorama; extra == "optionals"
+Dynamic: license-file
 
 # WeatherDB - module
 

{weatherdb-1.2.3 → weatherdb-1.2.5}/weatherdb.egg-info/SOURCES.txt

@@ -7,6 +7,7 @@ LICENSE
 MANIFEST.in
 README.md
 pyproject.toml
+zenodo.json
 .github/workflows/cleanup-cache.yml
 .github/workflows/python-publish.yml
 .github/workflows/python-test.yml

weatherdb-1.2.5/zenodo.json

@@ -0,0 +1,57 @@
+{
+    "metadata":{
+        "creators":[
+            {
+                "name": "Schmit, Max",
+                "affiliation": "University of Freiburg",
+                "orcid": "0000-0002-0712-4116"
+            }
+        ],
+        "contributors":[
+            {
+                "name": "Weiler, Markus",
+                "affiliation": "University of Freiburg",
+                "orcid": "0000-0001-6245-6917",
+                "type": "Supervisor"
+            }
+        ],
+        "license": {"id":"GPL-3.0-or-later"},
+        "title": "WeatherDB",
+        "version": "***",
+        "publication_date": "***",
+        "upload_type":"software",
+        "related_identifiers": [
+            {
+                "relation": "cites",
+                "identifier": "10.5281/zenodo.10066044",
+                "resource_type": "dataset"
+            },
+            {
+                "relation": "requires",
+                "identifier": "https://opendata.dwd.de/climate_environment/CDC/",
+                "resource_type": "dataset"
+            }
+        ],
+        "references": [
+            "DWD Climate Data Center (CDC): Historical daily station observations (temperature, pressure, precipitation, sunshine duration, etc.) for Germany, version v21.3, 2021, online available: https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/daily/kl/historical/",
+            "DWD Climate Data Center (CDC): Recent daily station observations (temperature, pressure, precipitation, sunshine duration, etc.) for Germany, quality control not completed yet, version recent, online available: https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/daily/kl/recent/",
+            "DWD Climate Data Center: Berechnete historische Tageswerte von charakteristischen Elementen aus dem Boden und dem Pflanzenbestand, historic values, Version v2, parameter \"VPGFAO\" online available: https://opendata.dwd.de/climate_environment/CDC/derived_germany/soil/daily/historical/",
+            "DWD Climate Data Center: Berechnete historische Tageswerte von charakteristischen Elementen aus dem Boden und dem Pflanzenbestand, recent values, Version v2, parameter \"VPGFAO\", online available: https://opendata.dwd.de/climate_environment/CDC/derived_germany/soil/daily/recent/",
+            "DWD Climate Data Center (CDC): Historical 10-minute station observations of precipitation for Germany, version V1, 2019, online available: https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/10_minutes/precipitation/historical/",
+            "DWD Climate Data Center (CDC): Recent 10-minute station observations of precipitation for Germany,version recent, 2019, online available: https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/10_minutes/precipitation/recent",
+            "DWD Climate Data Center (CDC): Historical daily station observations (temperature, pressure, precipitation, sunshine duration, etc.) for Germany, version v21.3, 2021, online available: https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/daily/kl/historical/",
+            "DWD Climate Data Center (CDC): Recent daily station observations (temperature, pressure, precipitation, sunshine duration, etc.) for Germany, quality control not completed yet, version recent, online available: https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/daily/kl/recent/",
+            "Schmit, M., & Weiler, M. (2023). German weather services (DWD) multi annual meteorological rasters for the climate period 1991-2020 refined to 25m grid (1.0.0) [Data set]. Zenodo. DOI:10.5281/zenodo.10066045",
+            "Richter, D. 1995. Ergebnisse methodischer Untersuchungen zur Korrektur des systematischen Meßfehlers des Hellmann-Niederschlagsmessers. Offenbach am Main: Selbstverl. des Dt. Wetterdienstes.",
+            "Copernicus. 2016. European Digital Elevation Model (EU-DEM), version 1.1. online available: https://land.copernicus.eu/imagery-in-situ/eu-dem/eu-dem-v1.1",
+            "Stoelzle, Michael & Weiler, Markus & Steinbrich, Andreas. (2016) Starkregengefährdung in Baden-Württemberg – von der Methodenentwicklung zur Starkregenkartierung. Tag der Hydrologie.",
+            "LARSIM Dokumentation, Stand 06.04.2023, online available: https://www.larsim.info/dokumentation/LARSIM-Dokumentation.pdf"
+        ],
+        "subjects": [
+            {"term": "Meteorology", "identifier": "mesh:D055904"},
+            {"term": "Meteorology", "identifier": "euroscivoc:281"},
+            {"term": "Meteorology", "identifier": "gemet:concept/5197"}
+        ],
+        "language": "eng"
+    }
+}

weatherdb-1.2.3/weatherdb/_version.py

@@ -1 +0,0 @@
-__version__ = "1.2.3"