weatherdb 1.2.2__tar.gz → 1.2.4__tar.gz

{weatherdb-1.2.2 → weatherdb-1.2.4}/.gitlab-ci.yml

@@ -16,7 +16,7 @@ test_single:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
   image: "python:$PYTHON_VERSION"
   services:
-    - name: postgis/postgis:16-3.4
+    - name: timescale/timescaledb-ha:pg17
       alias: postgres
       command: ["postgres", "-c", "fsync=off"]
   before_script:
@@ -44,7 +44,7 @@ test_single:
       - $WEATHERDB_TEST_ARTIFACT_DIR/*
   interruptible: true
   variables:
-    PYTHON_VERSION: 3.8
+    PYTHON_VERSION: 3.12
     PIP_CACHE_DIR: $CI_PROJECT_DIR/.cache/pip
     POSTGRES_USER: postgres
     POSTGRES_PASSWORD: postgres
@@ -77,7 +77,7 @@ test_matrix:
   extends: test_single
   parallel:
     matrix:
-      - PY_VERSION: ["3.9","3.10","3.11","3.12","3.13"]
+      - PY_VERSION: ["3.8", "3.9","3.10","3.11","3.13"]
   variables:
     PYTHON_VERSION: $PY_VERSION
 
@@ -92,8 +92,7 @@ test_download_rasters:
   variables:
     PYTHON_VERSION: $PY_VERSION
   parallel:
-    matrix:
-      - PY_VERSION: ["3.9","3.10","3.11","3.12","3.13"]
+    matrix: !reference [test_matrix, parallel, matrix ]
 
 build:
   stage: build
@@ -101,7 +100,7 @@ build:
     - if: $CI_COMMIT_TAG =~ /^[Vv]*\d+\.\d+\.\d+$/
   dependencies:
     - test_matrix
-  image: python:3.11
+  image: python:3.12
   script:
     - python -m pip install --upgrade pip --root-user-action=ignore
     - python -m pip install setuptools wheel build setuptools-scm --root-user-action=ignore
@@ -145,4 +144,40 @@ release_gitlab:
   release:
     name: '$CI_COMMIT_TAG'
     tag_name: '$CI_COMMIT_TAG'
-    description: '$CI_COMMIT_TAG_MESSAGE'
+    description: '$CI_COMMIT_TAG_MESSAGE'
+
+# deploying to Zenodo
+include:
+  - remote: 'https://gitlab.com/deploy2zenodo/deploy2zenodo/-/releases/permalink/latest/downloads/deploy2zenodo.yaml'
+
+deploy2zenodo:
+  stage: release
+  rules:
+    - !reference [build, rules]
+  dependencies:
+    - release_pypi
+  variables:
+    DEPLOY2ZENODO_JSON: zenodo.json
+    DEPLOY2ZENODO_ADD_IsNewVersionOf: yes
+    DEPLOY2ZENODO_GET_METADATA: "result.json"
+  before_script:
+    - apk add --no-cache curl jq git pandoc
+    # pack
+    - TAG=$(echo $CI_COMMIT_TAG | cut -d "v" -f 2)
+    - DEPLOY2ZENODO_UPLOAD=v$TAG.zip
+    - git archive --format zip --output "$DEPLOY2ZENODO_UPLOAD" $CI_COMMIT_SHA
+    # prepare zenodo.json
+    - publication_date=$(echo "$CI_COMMIT_TIMESTAMP" | grep -Eo "^[0-9]{4}-[0-9]{2}-[0-9]{2}")
+    - pandoc -o README.html README.md
+    - tmpjson="$(mktemp)"
+    - |
+        jq . $DEPLOY2ZENODO_JSON | \
+        jq ".metadata.version = \"$TAG\"" | \
+        jq ".metadata.publication_date = \"$publication_date\"" | \
+        jq --rawfile README README.html '.metadata.description = $README' | \
+        jq . | tee "$tmpjson"
+    - mv "$tmpjson" "$DEPLOY2ZENODO_JSON"
+  artifacts:
+    paths:
+      - $DEPLOY2ZENODO_JSON
+      - $DEPLOY2ZENODO_GET_METADATA

{weatherdb-1.2.2 → weatherdb-1.2.4}/CHANGES.md

@@ -1,5 +1,20 @@
 # Change-log
 
+## Version 1.2.4
+
+- added Zenodo upload
+- fix problem with creating user config
+- add format placeholders for GroupStations.create_ts for file_names parameter
+- translate Parameter name for Precipitation for create_roger_ts back to german names to work with actual RoGeR version
+- fix problem when create_ts is invoked with a zip file. 
+  Previously the precipitation timeseries did not get expanded to the last complete date and stopped at 00:00 o'clock
+
+## Version 1.2.3
+
+- fix: dropping stations
+- fix: test environment as DWD kicked station 2388
+- docker: optimize image with timescaledb image instead of postgis
+
 ## Version 1.2.2
 
 - config: define utf-8 as default encoding for config files
@@ -7,7 +22,7 @@
 
 ## Version 1.2.1
 
-- fix: filllup linear regression did not take configuration value and therefor did never work
+- fix: fillup linear regression did not take configuration value and therefor did never work
 - config: change LINEAR_INTERPOLATION_LIMIT to be a category
 - cli: add version parameter
 - multiprocessing: make process count configurable through user configuration

{weatherdb-1.2.2 → weatherdb-1.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: weatherdb
-Version: 1.2.2
+Version: 1.2.4
 Summary: This is a package to work with and to create the Weather Database which handles, checks, fills and corrects DWD Weather Station data.
 Author-email: Max Schmit <max.schmit@hydrology.uni-freiburg.de>
 License:                     GNU GENERAL PUBLIC LICENSE
@@ -693,8 +693,28 @@ Classifier: Intended Audience :: Science/Research
 Classifier: Natural Language :: English
 Requires-Python: <3.14,>=3.8
 Description-Content-Type: text/markdown
-Provides-Extra: optionals
 License-File: LICENSE
+Requires-Dist: geopandas
+Requires-Dist: shapely>=2
+Requires-Dist: pandas
+Requires-Dist: numpy
+Requires-Dist: progressbar2
+Requires-Dist: rasterio
+Requires-Dist: sqlalchemy>2.0
+Requires-Dist: psycopg2-binary
+Requires-Dist: GeoAlchemy2
+Requires-Dist: alembic
+Requires-Dist: requests
+Requires-Dist: keyring
+Requires-Dist: keyrings.alt; sys_platform == "linux"
+Requires-Dist: rasterstats
+Requires-Dist: click
+Requires-Dist: setuptools>=61.0
+Requires-Dist: setuptools_scm
+Provides-Extra: optionals
+Requires-Dist: coloredlogs; extra == "optionals"
+Requires-Dist: colorama; extra == "optionals"
+Dynamic: license-file
 
 # WeatherDB - module
 

{weatherdb-1.2.2 → weatherdb-1.2.4}/docker/docker-compose.yaml

@@ -38,11 +38,13 @@ services:
           target: /src/tests
 
   db:
-    image: postgis/postgis:16-3.4
+    image: timescale/timescaledb-ha:pg17 # about 1 minute faster for tests
+    # image: postgis/postgis:17-3.4
     environment:
       POSTGRES_DB: weatherdb
       POSTGRES_USER: postgres
       POSTGRES_PASSWORD_FILE: /run/secrets/db_password
+    shm_size: '2g'
     ports:
       - 5454:5432
     volumes:

weatherdb-1.2.4/weatherdb/_version.py

@@ -0,0 +1 @@
+__version__ = "1.2.4"

{weatherdb-1.2.2 → weatherdb-1.2.4}/weatherdb/config/ConfigParser.py

@@ -357,7 +357,7 @@ class ConfigParser(configparser.ConfigParser):
                 user_config_file = "ask"
 
         # ask for the user config file
-        if user_config_file == "ask":
+        if user_config_file.lower().strip()[0] == "a":
             try:
                 from tkinter import Tk
                 from tkinter import filedialog

{weatherdb-1.2.2 → weatherdb-1.2.4}/weatherdb/station/GroupStation.py

@@ -344,8 +344,13 @@ class GroupStation(object):
     def get_name(self):
         return self.station_parts[0].get_name()
 
-    def create_roger_ts(self, dir, period=(None, None),
-                        kind="best", r_r0=1, add_t_min=False, add_t_max=False,
+    def create_roger_ts(self,
+                        dir,
+                        period=(None, None),
+                        kind="best",
+                        r_r0=1,
+                        add_t_min=False,
+                        add_t_max=False,
                         do_toolbox_format=False,
                         **kwargs):
         """Create the timeserie files for roger as csv.
@@ -394,10 +399,15 @@ class GroupStation(object):
         """
         if do_toolbox_format:
             return self.create_ts(
-                dir=dir, period=period, kinds=kind,
-                agg_to="10 min", r_r0=r_r0, split_date=True,
+                dir=dir,
+                period=period,
+                kinds=kind,
+                agg_to="10 min",
+                r_r0=r_r0,
+                split_date=True,
                 nas_allowed=False,
-                add_t_min=add_t_min, add_t_max=add_t_max,
+                add_t_min=add_t_min,
+                add_t_max=add_t_max,
                 file_names={"P":"PREC.txt", "T":"TA.txt", "ET":"PET.txt"},
                 col_names={"P":"PREC", "ET":"PET",
                            "T":"TA", "T_min":"TA_min", "T_max":"TA_max",
@@ -408,18 +418,34 @@ class GroupStation(object):
                 **kwargs)
         else:
             return self.create_ts(
-                dir=dir, period=period, kinds=kind,
-                agg_to="10 min", r_r0=r_r0, split_date=True,
+                dir=dir,
+                period=period,
+                kinds=kind,
+                agg_to="10 min",
+                r_r0=r_r0,
+                split_date=True,
                 nas_allowed=False,
-                add_t_min=add_t_min, add_t_max=add_t_max,
+                file_names={"P":"N_{id:0>5}.txt"},
+                col_names={"P":"N"},
+                add_t_min=add_t_min,
+                add_t_max=add_t_max,
                 **kwargs)
 
-    def create_ts(self, dir, period=(None, None),
-                  kinds="best", paras="all",
-                  agg_to="10 min", r_r0=None, split_date=False,
-                  nas_allowed=True, add_na_share=False,
-                  add_t_min=False, add_t_max=False,
-                  add_meta=True, file_names={}, col_names={},
+    def create_ts(self,
+                  dir,
+                  period=(None, None),
+                  kinds="best",
+                  paras="all",
+                  agg_to="10 min",
+                  r_r0=None,
+                  split_date=False,
+                  nas_allowed=True,
+                  add_na_share=False,
+                  add_t_min=False,
+                  add_t_max=False,
+                  add_meta=True,
+                  file_names={},
+                  col_names={},
                   keep_date_parts=False,
                   **kwargs):
         """Create the timeserie files as csv.
@@ -485,6 +511,7 @@ class GroupStation(object):
         file_names : dict, optional
             A dictionary with the file names for the different parameters.
             e.g.{"P":"PREC.txt", "T":"TA.txt", "ET":"ET.txt"}
+            Additionally, the station ID ("{id}" as int) and period ({period} as TimestampPeriod) are available as placeholders.
             If an empty dictionary is given, then the standard names are used.
             The default is {}.
         col_names : dict, optional
@@ -536,19 +563,22 @@ class GroupStation(object):
             del kwargs["_skip_period_check"]
 
         # prepare loop
-        name_suffix = "_{stid:0>5}.txt".format(stid=self.id)
+        name_suffix = "_{id:0>5}.txt".format(id=self.id)
         x, y = self.get_geom().coords.xy
-        name = self.get_name() + " (ID: {stid})".format(stid=self.id)
+        name = self.get_name() + " (ID: {id})".format(id=self.id)
         do_zip = isinstance(dir, zipfile.ZipFile)
 
         for para in paras:
             # get the timeserie
             df = self.get_df(
-                period=period, kinds=kinds,
-                paras=[para], agg_to=agg_to,
+                period=period,
+                kinds=kinds,
+                paras=[para],
+                agg_to=agg_to,
                 nas_allowed=nas_allowed,
                 add_na_share=add_na_share,
-                add_t_min=add_t_min, add_t_max=add_t_max,
+                add_t_min=add_t_min,
+                add_t_max=add_t_max,
                 _skip_period_check=True,
                 **kwargs)
 
@@ -573,8 +603,7 @@ class GroupStation(object):
             # check for NAs
             filled_cols = [col for col in df.columns if "filled_by" in col]
             if not nas_allowed and df.drop(filled_cols, axis=1).isna().sum().sum() > 0:
-                warnings.warn("There were NAs in the timeserie for Station {stid}.".format(
-                    stid=self.id))
+                warnings.warn(f"There were NAs in the timeseries for Station {self.id}.")
 
             # special operations for et
             if para == "et" and r_r0 is not None:
@@ -609,9 +638,11 @@ class GroupStation(object):
 
             # get file name
             if para.upper() in file_names:
-                file_name = file_names[para.upper()]
+                file_name = file_names[para.upper()]\
+                    .format(id=self.id, period=period)
             elif para in file_names:
-                file_name = file_names[para]
+                file_name = file_names[para]\
+                    .format(id=self.id, period=period)
             else:
                 file_name = para.upper() + name_suffix
 

{weatherdb-1.2.2 → weatherdb-1.2.4}/weatherdb/station/StationBases.py

@@ -595,7 +595,7 @@ class StationBase:
             DROP TABLE IF EXISTS timeseries."{self.id}_{self._para}";
             DELETE FROM meta_{self._para} WHERE station_id={self.id};
             DELETE FROM station_ma_raster WHERE station_id={self.id} and parameter='{self._para}';
-            DELETE FROM station_ma_timeseries WHERE station_id={self.id} and parameter='{self._para}';
+            DELETE FROM station_ma_timeserie WHERE station_id={self.id} and parameter='{self._para}';
             INSERT INTO dropped_stations(station_id, parameter, why, timestamp)
             VALUES ('{self.id}', '{self._para}', '{why}', NOW())
             ON CONFLICT (station_id, parameter)

{weatherdb-1.2.2 → weatherdb-1.2.4}/weatherdb/station/StationP.py

@@ -240,8 +240,8 @@ class StationP(StationPBase):
     @db_engine.deco_create_privilege
     def _create_timeseries_table(self):
         """Create the timeseries table in the DB if it is not yet existing."""
-        sql_add_table = '''
-            CREATE TABLE IF NOT EXISTS timeseries."{stid}_{para}"  (
+        sql_add_table = f'''
+            CREATE TABLE IF NOT EXISTS timeseries."{self.id}_{self._para}"  (
                 timestamp timestamp PRIMARY KEY,
                 raw int4,
                 qn smallint,
@@ -250,7 +250,7 @@ class StationP(StationPBase):
                 filled_by int2,
                 corr int4
             );
-        '''.format(stid=self.id, para=self._para)
+        '''
         with db_engine.connect() as con:
             con.execute(sqltxt(sql_add_table))
             con.commit()

{weatherdb-1.2.2 → weatherdb-1.2.4}/weatherdb/stations/GroupStations.py

@@ -293,10 +293,17 @@ class GroupStations(object):
 
         return stations
 
-    def create_ts(self, dir, period=(None, None), kinds="best",
-                  stids="all", agg_to="10 min", r_r0=None, split_date=False,
-                  nas_allowed=True, add_na_share=False,
-                  add_t_min=False, add_t_max=False,
+    def create_ts(self, dir,
+                  period=(None, None),
+                  kinds="best",
+                  stids="all",
+                  agg_to="10 min",
+                  r_r0=None,
+                  split_date=False,
+                  nas_allowed=True,
+                  add_na_share=False,
+                  add_t_min=False,
+                  add_t_max=False,
                   **kwargs):
         """Download and create the weather tables as csv files.
 
@@ -393,7 +400,6 @@ class GroupStations(object):
                         add_na_share=add_na_share,
                         add_t_min=add_t_min,
                         add_t_max=add_t_max,
-                        _skip_period_check=True,
                         **kwargs)
                     pbar.variables["last_station"] = stat.id
                     pbar.update(pbar.value + 1)
@@ -498,10 +504,16 @@ class GroupStations(object):
         """
         if do_toolbox_format:
             return self.create_ts(
-                dir=dir, period=period, kinds=kind,
-                agg_to="10 min", r_r0=r_r0, stids=stids,
-                split_date=True, nas_allowed=False,
-                add_t_min=add_t_min, add_t_max=add_t_max,
+                dir=dir,
+                period=period,
+                kinds=kind,
+                agg_to="10 min",
+                r_r0=r_r0,
+                stids=stids,
+                split_date=True,
+                nas_allowed=False,
+                add_t_min=add_t_min,
+                add_t_max=add_t_max,
                 file_names={"P":"PREC.txt", "T":"TA.txt", "ET":"PET.txt"},
                 col_names={"P":"PREC", "ET":"PET",
                            "T":"TA", "T_min":"TA_min", "T_max":"TA_max",
@@ -512,8 +524,16 @@ class GroupStations(object):
                 **kwargs)
         else:
             return self.create_ts(
-                dir=dir, period=period, kinds=kind,
-                agg_to="10 min", r_r0=r_r0, stids=stids,
-                split_date=True, nas_allowed=False,
-                add_t_min=add_t_min, add_t_max=add_t_max,
+                dir=dir,
+                period=period,
+                kinds=kind,
+                agg_to="10 min",
+                r_r0=r_r0,
+                stids=stids,
+                split_date=True,
+                nas_allowed=False,
+                file_names={"P": "N_{id:0>5}.txt"},
+                col_names={"P":"N"},
+                add_t_min=add_t_min,
+                add_t_max=add_t_max,
                 **kwargs)

{weatherdb-1.2.2 → weatherdb-1.2.4}/weatherdb.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: weatherdb
-Version: 1.2.2
+Version: 1.2.4
 Summary: This is a package to work with and to create the Weather Database which handles, checks, fills and corrects DWD Weather Station data.
 Author-email: Max Schmit <max.schmit@hydrology.uni-freiburg.de>
 License:                     GNU GENERAL PUBLIC LICENSE
@@ -693,8 +693,28 @@ Classifier: Intended Audience :: Science/Research
 Classifier: Natural Language :: English
 Requires-Python: <3.14,>=3.8
 Description-Content-Type: text/markdown
-Provides-Extra: optionals
 License-File: LICENSE
+Requires-Dist: geopandas
+Requires-Dist: shapely>=2
+Requires-Dist: pandas
+Requires-Dist: numpy
+Requires-Dist: progressbar2
+Requires-Dist: rasterio
+Requires-Dist: sqlalchemy>2.0
+Requires-Dist: psycopg2-binary
+Requires-Dist: GeoAlchemy2
+Requires-Dist: alembic
+Requires-Dist: requests
+Requires-Dist: keyring
+Requires-Dist: keyrings.alt; sys_platform == "linux"
+Requires-Dist: rasterstats
+Requires-Dist: click
+Requires-Dist: setuptools>=61.0
+Requires-Dist: setuptools_scm
+Provides-Extra: optionals
+Requires-Dist: coloredlogs; extra == "optionals"
+Requires-Dist: colorama; extra == "optionals"
+Dynamic: license-file
 
 # WeatherDB - module
 

{weatherdb-1.2.2 → weatherdb-1.2.4}/weatherdb.egg-info/SOURCES.txt

@@ -7,6 +7,7 @@ LICENSE
 MANIFEST.in
 README.md
 pyproject.toml
+zenodo.json
 .github/workflows/cleanup-cache.yml
 .github/workflows/python-publish.yml
 .github/workflows/python-test.yml

weatherdb-1.2.4/zenodo.json

@@ -0,0 +1,57 @@
+{
+    "metadata":{
+        "creators":[
+            {
+                "name": "Schmit, Max",
+                "affiliation": "University of Freiburg",
+                "orcid": "0000-0002-0712-4116"
+            }
+        ],
+        "contributor":[
+            {
+                "name": "Weiler, Markus",
+                "affiliation": "University of Freiburg",
+                "orcid": "0000-0001-6245-6917",
+                "type": "Supervisor"
+            }
+        ],
+        "license": {"id":"GPL-3.0-or-later"},
+        "title": "WeatherDB",
+        "version": "***",
+        "publication_date": "***",
+        "upload_type":"software",
+        "related_identifiers": [
+            {
+                "relation": "cites",
+                "identifier": "10.5281/zenodo.10066044",
+                "resource_type": "Dataset"
+            },
+            {
+                "relation": "requires",
+                "identifier": "https://opendata.dwd.de/climate_environment/CDC/",
+                "resource_type": "Dataset"
+            }
+        ],
+        "references": [
+            "DWD Climate Data Center (CDC): Historical daily station observations (temperature, pressure, precipitation, sunshine duration, etc.) for Germany, version v21.3, 2021, online available: https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/daily/kl/historical/",
+            "DWD Climate Data Center (CDC): Recent daily station observations (temperature, pressure, precipitation, sunshine duration, etc.) for Germany, quality control not completed yet, version recent, online available: https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/daily/kl/recent/",
+            "DWD Climate Data Center: Berechnete historische Tageswerte von charakteristischen Elementen aus dem Boden und dem Pflanzenbestand, historic values, Version v2, parameter \"VPGFAO\" online available: https://opendata.dwd.de/climate_environment/CDC/derived_germany/soil/daily/historical/",
+            "DWD Climate Data Center: Berechnete historische Tageswerte von charakteristischen Elementen aus dem Boden und dem Pflanzenbestand, recent values, Version v2, parameter \"VPGFAO\", online available: https://opendata.dwd.de/climate_environment/CDC/derived_germany/soil/daily/recent/",
+            "DWD Climate Data Center (CDC): Historical 10-minute station observations of precipitation for Germany, version V1, 2019, online available: https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/10_minutes/precipitation/historical/",
+            "DWD Climate Data Center (CDC): Recent 10-minute station observations of precipitation for Germany,version recent, 2019, online available: https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/10_minutes/precipitation/recent",
+            "DWD Climate Data Center (CDC): Historical daily station observations (temperature, pressure, precipitation, sunshine duration, etc.) for Germany, version v21.3, 2021, online available: https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/daily/kl/historical/",
+            "DWD Climate Data Center (CDC): Recent daily station observations (temperature, pressure, precipitation, sunshine duration, etc.) for Germany, quality control not completed yet, version recent, online available: https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/daily/kl/recent/",
+            "Schmit, M., & Weiler, M. (2023). German weather services (DWD) multi annual meteorological rasters for the climate period 1991-2020 refined to 25m grid (1.0.0) [Data set]. Zenodo. DOI:10.5281/zenodo.10066045",
+            "Richter, D. 1995. Ergebnisse methodischer Untersuchungen zur Korrektur des systematischen Meßfehlers des Hellmann-Niederschlagsmessers. Offenbach am Main: Selbstverl. des Dt. Wetterdienstes.",
+            "Copernicus. 2016. European Digital Elevation Model (EU-DEM), version 1.1. online available: https://land.copernicus.eu/imagery-in-situ/eu-dem/eu-dem-v1.1",
+            "Stoelzle, Michael & Weiler, Markus & Steinbrich, Andreas. (2016) Starkregengefährdung in Baden-Württemberg – von der Methodenentwicklung zur Starkregenkartierung. Tag der Hydrologie.",
+            "LARSIM Dokumentation, Stand 06.04.2023, online available: https://www.larsim.info/dokumentation/LARSIM-Dokumentation.pdf"
+        ],
+        "subjects": [
+            {"term": "Meteorology", "identifier": "mesh:D055904"},
+            {"term": "Meteorology", "identifier": "euroscivoc:281"},
+            {"term": "Meteorology", "identifier": "gemet:concept/5197"}
+        ],
+        "language": "English"
+    }
+}

weatherdb-1.2.2/weatherdb/_version.py

@@ -1 +0,0 @@
-__version__ = "1.2.2"