rainfallqc 0.2.4__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rainfallqc-0.3.0/PKG-INFO +333 -0
- rainfallqc-0.3.0/README.rst +274 -0
- {rainfallqc-0.2.4 → rainfallqc-0.3.0}/pyproject.toml +1 -1
- {rainfallqc-0.2.4 → rainfallqc-0.3.0}/src/rainfallqc/__init__.py +1 -1
- {rainfallqc-0.2.4 → rainfallqc-0.3.0}/src/rainfallqc/checks/neighbourhood_checks.py +130 -131
- {rainfallqc-0.2.4 → rainfallqc-0.3.0}/src/rainfallqc/checks/timeseries_checks.py +20 -13
- {rainfallqc-0.2.4 → rainfallqc-0.3.0}/src/rainfallqc/core/all_qc_checks.py +2 -2
- {rainfallqc-0.2.4 → rainfallqc-0.3.0}/src/rainfallqc/qc_frameworks/apply_qc_framework.py +19 -11
- {rainfallqc-0.2.4 → rainfallqc-0.3.0}/src/rainfallqc/qc_frameworks/inbuilt_qc_frameworks.py +1 -1
- {rainfallqc-0.2.4 → rainfallqc-0.3.0}/src/rainfallqc/utils/data_readers.py +80 -80
- {rainfallqc-0.2.4 → rainfallqc-0.3.0}/src/rainfallqc/utils/data_utils.py +2 -2
- {rainfallqc-0.2.4 → rainfallqc-0.3.0}/src/rainfallqc/utils/neighbourhood_utils.py +12 -2
- rainfallqc-0.2.4/PKG-INFO +0 -207
- rainfallqc-0.2.4/README.rst +0 -148
- {rainfallqc-0.2.4 → rainfallqc-0.3.0}/LICENSE +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.3.0}/src/rainfallqc/checks/__init__.py +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.3.0}/src/rainfallqc/checks/comparison_checks.py +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.3.0}/src/rainfallqc/checks/gauge_checks.py +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.3.0}/src/rainfallqc/checks/pypwsqc_filters.py +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.3.0}/src/rainfallqc/core/__init__.py +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.3.0}/src/rainfallqc/data/ETCCDI/RawData_HADEX2_CDD_1951-2010_ANN_from-90to90_from-180to180.nc +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.3.0}/src/rainfallqc/data/ETCCDI/RawData_HADEX2_CWD_1951-2010_ANN_from-90to90_from-180to180.nc +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.3.0}/src/rainfallqc/data/ETCCDI/RawData_HADEX2_PRCPTOT_1951-2010_ANN_from-90to90_from-180to180.nc +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.3.0}/src/rainfallqc/data/ETCCDI/RawData_HADEX2_R99p_1951-2010_ANN_from-90to90_from-180to180.nc +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.3.0}/src/rainfallqc/data/ETCCDI/RawData_HADEX2_Rx1day_1951-2010_ANN_from-90to90_from-180to180.nc +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.3.0}/src/rainfallqc/data/ETCCDI/RawData_HADEX2_SDII_1951-2010_ANN_from-90to90_from-180to180.nc +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.3.0}/src/rainfallqc/data/ETCCDI/__init__.py +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.3.0}/src/rainfallqc/data/__init__.py +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.3.0}/src/rainfallqc/qc_frameworks/__init__.py +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.3.0}/src/rainfallqc/utils/__init__.py +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.3.0}/src/rainfallqc/utils/spatial_utils.py +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.3.0}/src/rainfallqc/utils/stats.py +0 -0
|
@@ -0,0 +1,333 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: rainfallqc
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Quality control for rainfall data
|
|
5
|
+
Author-email: Tom Keel <tomkee@ceh.ac.uk>
|
|
6
|
+
Maintainer-email: Tom Keel <tomkee@ceh.ac.uk>
|
|
7
|
+
Requires-Python: >=3.10.0
|
|
8
|
+
Description-Content-Type: text/x-rst
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
|
|
13
|
+
Classifier: Natural Language :: English
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Hydrology
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Requires-Dist: scipy
|
|
23
|
+
Requires-Dist: numpy
|
|
24
|
+
Requires-Dist: polars>=1.30
|
|
25
|
+
Requires-Dist: pandas
|
|
26
|
+
Requires-Dist: pyarrow
|
|
27
|
+
Requires-Dist: xarray
|
|
28
|
+
Requires-Dist: netCDF4
|
|
29
|
+
Requires-Dist: bottleneck
|
|
30
|
+
Requires-Dist: geopy
|
|
31
|
+
Requires-Dist: poligrain
|
|
32
|
+
Requires-Dist: pypwsqc
|
|
33
|
+
Requires-Dist: pip ; extra == "dev"
|
|
34
|
+
Requires-Dist: numpy ; extra == "dev"
|
|
35
|
+
Requires-Dist: bump-my-version ; extra == "dev"
|
|
36
|
+
Requires-Dist: ruff ; extra == "dev"
|
|
37
|
+
Requires-Dist: black ; extra == "dev"
|
|
38
|
+
Requires-Dist: pre_commit ; extra == "dev"
|
|
39
|
+
Requires-Dist: pylint ; extra == "dev"
|
|
40
|
+
Requires-Dist: pytest ; extra == "dev"
|
|
41
|
+
Requires-Dist: pytest-cov ; extra == "dev"
|
|
42
|
+
Requires-Dist: matplotlib ; extra == "dev"
|
|
43
|
+
Requires-Dist: sphinx ; extra == "dev"
|
|
44
|
+
Requires-Dist: sphinx ; extra == "docs"
|
|
45
|
+
Requires-Dist: sphinx-copybutton ; extra == "docs"
|
|
46
|
+
Requires-Dist: sphinx-rtd-theme ; extra == "docs"
|
|
47
|
+
Requires-Dist: mypy ; extra == "lint"
|
|
48
|
+
Requires-Dist: ruff ; extra == "lint"
|
|
49
|
+
Requires-Dist: pytest ; extra == "test"
|
|
50
|
+
Requires-Dist: pytest-cov ; extra == "test"
|
|
51
|
+
Project-URL: bugs, https://github.com/thomasjkeel/RainfallQC/issues
|
|
52
|
+
Project-URL: changelog, https://github.com/thomasjkeel/RainfallQC/blob/master/changelog.md
|
|
53
|
+
Project-URL: homepage, https://github.com/thomasjkeel/RainfallQC
|
|
54
|
+
Provides-Extra: dev
|
|
55
|
+
Provides-Extra: docs
|
|
56
|
+
Provides-Extra: lint
|
|
57
|
+
Provides-Extra: test
|
|
58
|
+
|
|
59
|
+
.. image:: https://github.com/NERC-CEH/RainfallQC/blob/main/docs/logos/rainfallQC_logo.png
|
|
60
|
+
:align: center
|
|
61
|
+
:height: 180px
|
|
62
|
+
:width: 200 px
|
|
63
|
+
:alt: RainfallQC
|
|
64
|
+
|
|
65
|
+
===============================================
|
|
66
|
+
RainfallQC - Quality control for rainfall data
|
|
67
|
+
===============================================
|
|
68
|
+
|
|
69
|
+
.. image:: https://img.shields.io/pypi/v/rainfallqc.svg
|
|
70
|
+
:target: https://pypi.python.org/pypi/rainfallqc
|
|
71
|
+
|
|
72
|
+
..
|
|
73
|
+
image:: https://readthedocs.org/projects/rainfallqc/badge/?version=latest
|
|
74
|
+
:target: https://rainfallqc.readthedocs.io/en/latest/?version=latest
|
|
75
|
+
:alt: Documentation Status
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
Provides methods for running rainfall quality control.
|
|
79
|
+
|
|
80
|
+
Installation
|
|
81
|
+
============
|
|
82
|
+
RainfallQC can be installed from PyPi:
|
|
83
|
+
|
|
84
|
+
.. code-block:: bash
|
|
85
|
+
|
|
86
|
+
pip install rainfallqc
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
Example use
|
|
90
|
+
===========
|
|
91
|
+
|
|
92
|
+
Example 1. - Running individual checks on a single rain gauge
|
|
93
|
+
-------------------------------------------------------------
|
|
94
|
+
Let's say you have data for a single rain gauge stored in "hourly_rain_gauge_data.csv" which looks like this:
|
|
95
|
+
|
|
96
|
+
.. table:: Example data 1. Single rain gauge
|
|
97
|
+
:widths: auto
|
|
98
|
+
:align: center
|
|
99
|
+
|
|
100
|
+
+---------------------+---------+
|
|
101
|
+
| time | rain_mm |
|
|
102
|
+
+=====================+=========+
|
|
103
|
+
| 2020-01-01 00:00 | 0.0 |
|
|
104
|
+
+---------------------+---------+
|
|
105
|
+
| 2020-01-01 01:00 | 0.1 |
|
|
106
|
+
+---------------------+---------+
|
|
107
|
+
| 2020-01-01 02:00 | 0.0 |
|
|
108
|
+
+---------------------+---------+
|
|
109
|
+
| 2020-01-01 03:00 | 105.0 |
|
|
110
|
+
+---------------------+---------+
|
|
111
|
+
| 2020-01-01 04:00 | 0.6 |
|
|
112
|
+
+---------------------+---------+
|
|
113
|
+
| ... | ... |
|
|
114
|
+
+---------------------+---------+
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
For the majority of the checks in RainfallQC, you can load in your data using `polars <https://pola-rs.github.io/polars-book/>`_ and run the checks directly.
|
|
118
|
+
Below, we run 2 example QC checks:
|
|
119
|
+
|
|
120
|
+
- 1) ``check_intermittency`` - to flag years where there are periods of non-zero bounded by 0 (see Figure 1.),
|
|
121
|
+
- 2) ``daily_accumulations`` - to flag accumulations of hourly values into daily.
|
|
122
|
+
|
|
123
|
+
.. figure:: https://thomasjkeel.github.io/UK-Rain-Gauge-Network/example_images/intermittency.png
|
|
124
|
+
:align: center
|
|
125
|
+
:height: 250px
|
|
126
|
+
:width: 300px
|
|
127
|
+
|
|
128
|
+
**Figure 1.** Example of an intermittency issue within the rainfall record
|
|
129
|
+
|
|
130
|
+
.. code-block:: python
|
|
131
|
+
|
|
132
|
+
import polars as pl
|
|
133
|
+
from rainfallqc import gauge_checks, timeseries_checks
|
|
134
|
+
|
|
135
|
+
data = pl.read_csv("hourly_rain_gauge_data.csv")
|
|
136
|
+
|
|
137
|
+
intermittent_years = gauge_checks.check_intermittency(data, target_gauge_col="rain_mm")
|
|
138
|
+
|
|
139
|
+
daily_accumulation_flags = timeseries_checks.check_daily_accumulations(
|
|
140
|
+
data,
|
|
141
|
+
target_gauge_col="rain_mm",
|
|
142
|
+
gauge_lat=52.0,
|
|
143
|
+
gauge_lon=2.0,
|
|
144
|
+
smallest_measurable_rainfall_amount=0.1,
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
Please note that some checks may require additional metadata, such as gauge location (latitude and longitude) or smallest measurable rainfall amount (e.g. 0.1 mm).
|
|
149
|
+
This could look like:
|
|
150
|
+
|
|
151
|
+
.. table:: Example metadata 1. Rain gauge metadata
|
|
152
|
+
:widths: auto
|
|
153
|
+
:align: center
|
|
154
|
+
|
|
155
|
+
+--------------------+----------+-----------+------------------+------------------+---------------------+
|
|
156
|
+
| station_id | latitude | longitude | start_datetime | end_datetime | path |
|
|
157
|
+
+====================+==========+===========+==================+==================+=====================+
|
|
158
|
+
| rain_mm_gauge_1 | 53.0 | 2.0 | 2020-01-01 00:00 | 2024-01-01 00:00 | path/to/gauge_1.csv |
|
|
159
|
+
+--------------------+----------+-----------+------------------+------------------+---------------------+
|
|
160
|
+
| rain_mm_gauge_2 | 54.1 | -0.5 | 2018-01-01 00:00 | 2023-01-01 00:00 | path/to/gauge_2.csv |
|
|
161
|
+
+--------------------+----------+-----------+------------------+------------------+---------------------+
|
|
162
|
+
| rain_mm_gauge_3 | 56.9 | 1.9 | 2015-01-01 00:00 | 2025-01-01 00:00 | path/to/gauge_3.csv |
|
|
163
|
+
+--------------------+----------+-----------+------------------+------------------+---------------------+
|
|
164
|
+
| ... | ... | ... | | | ... |
|
|
165
|
+
+--------------------+----------+-----------+------------------+------------------+---------------------+
|
|
166
|
+
|
|
167
|
+
You could then run checks that require metadata i.e. the ``check_hourly_exceedance_etccdi_rx1day`` QC check which flags rainfall values exceeding
|
|
168
|
+
the hourly day rainfall 1-day record at a given location (see Figure 2):
|
|
169
|
+
|
|
170
|
+
.. figure:: https://thomasjkeel.github.io/UK-Rain-Gauge-Network/example_images/rx1day_check.png
|
|
171
|
+
:align: center
|
|
172
|
+
:height: 250px
|
|
173
|
+
:width: 300px
|
|
174
|
+
|
|
175
|
+
**Figure 2.** Example of an Rx1day check from the IntenseQC framework
|
|
176
|
+
|
|
177
|
+
The code for that check looks like:
|
|
178
|
+
|
|
179
|
+
.. code-block:: python
|
|
180
|
+
|
|
181
|
+
import polars as pl
|
|
182
|
+
from rainfallqc import comparison_checks
|
|
183
|
+
|
|
184
|
+
data = pl.read_csv("hourly_rain_gauge_data_gauge_1.csv")
|
|
185
|
+
metadata = pl.read_csv("rain_gauge_metadata.csv")
|
|
186
|
+
|
|
187
|
+
target_gauge_id = "rain_mm_gauge_1"
|
|
188
|
+
target_metadata = metadata.filter(pl.col("station_id") == target_gauge_id)
|
|
189
|
+
|
|
190
|
+
rx1day_check = comparison_checks.check_hourly_exceedance_etccdi_rx1day(
|
|
191
|
+
data,
|
|
192
|
+
target_gauge_col=target_gauge_col,
|
|
193
|
+
gauge_lat=target_metadata["latitude"],
|
|
194
|
+
gauge_lon=target_metadata["longitude"]
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
Output flags will then look like:
|
|
198
|
+
|
|
199
|
+
.. table:: Example flag outputs for the Rx1day QC check
|
|
200
|
+
:widths: auto
|
|
201
|
+
:align: center
|
|
202
|
+
|
|
203
|
+
+---------------------+--------------+
|
|
204
|
+
| time | rx1day_check |
|
|
205
|
+
+=====================+==============+
|
|
206
|
+
| 2020-01-01 00:00 | 0 |
|
|
207
|
+
+---------------------+--------------+
|
|
208
|
+
| 2020-01-01 01:00 | 0 |
|
|
209
|
+
+---------------------+--------------+
|
|
210
|
+
| 2020-01-01 02:00 | 0 |
|
|
211
|
+
+---------------------+--------------+
|
|
212
|
+
| 2020-01-01 03:00 | 1 |
|
|
213
|
+
+---------------------+--------------+
|
|
214
|
+
| 2020-01-01 04:00 | 0 |
|
|
215
|
+
+---------------------+--------------+
|
|
216
|
+
| ... | ... |
|
|
217
|
+
+---------------------+--------------+
|
|
218
|
+
|
|
219
|
+
Example 2. - Running multiple QC checks on a single target gauge
|
|
220
|
+
----------------------------------------------------------------
|
|
221
|
+
To run multiple QC checks, you can use the `apply_qc_framework() <rainfallqc.checks.html#rainfallqc.qc_frameworks.html#module-rainfallqc.qc_frameworks.apply_qc_framework>`_
|
|
222
|
+
method to run QC methods from a given framework (e.g. IntenseQC).
|
|
223
|
+
|
|
224
|
+
Let's say you have hourly rainfall values from a rain gauge network data like:
|
|
225
|
+
|
|
226
|
+
.. table:: Example data 2. Rain gauge network
|
|
227
|
+
:widths: auto
|
|
228
|
+
:align: center
|
|
229
|
+
|
|
230
|
+
+---------------------+-----------------+-----------------+-----------------+
|
|
231
|
+
| time | rain_mm_gauge_1 | rain_mm_gauge_2 | rain_mm_gauge_3 |
|
|
232
|
+
+=====================+=================+=================+=================+
|
|
233
|
+
| 2020-01-01 00:00 | 0.0 | 0.5 | 0.0 |
|
|
234
|
+
+---------------------+-----------------+-----------------+-----------------+
|
|
235
|
+
| 2020-01-01 01:00 | 0.5 | 0.0 | 1.0 |
|
|
236
|
+
+---------------------+-----------------+-----------------+-----------------+
|
|
237
|
+
| 2020-01-01 02:00 | 0.0 | 1.0 | 0.0 |
|
|
238
|
+
+---------------------+-----------------+-----------------+-----------------+
|
|
239
|
+
| 2020-01-01 03:00 | 105.0 | 0.0 | 0.5 |
|
|
240
|
+
+---------------------+-----------------+-----------------+-----------------+
|
|
241
|
+
| 2020-01-01 04:00 | 0.0 | 0.5 | 0.0 |
|
|
242
|
+
+---------------------+-----------------+-----------------+-----------------+
|
|
243
|
+
| ... | ... | ... | ... |
|
|
244
|
+
+---------------------+-----------------+-----------------+-----------------+
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
... and metadata like example metdata 1.
|
|
248
|
+
You can then run multiple QC checks at once by defining a QC framework, the methods to run and parameters for those methods.
|
|
249
|
+
|
|
250
|
+
As of RainfallQC v0.3.0, there are three QC frameworks:
|
|
251
|
+
|
|
252
|
+
1. "intenseqc" - All 25 checks from IntenseQC/GSDR-QC with names like: "QC1", "QC2" ... "QC25",
|
|
253
|
+
2. "pypwsqc" - 2 checks from pyPWSQC with the names: "FZ" and "SO",
|
|
254
|
+
3. "custom" - Allows the user to select a custom set of checks (see Example 8 in `Tutorials <https://rainfallqc.readthedocs.io/en/latest/tutorials.html>`_).
|
|
255
|
+
|
|
256
|
+
Let's run some QC checks from intenseqc framework below:
|
|
257
|
+
|
|
258
|
+
.. code-block:: python
|
|
259
|
+
|
|
260
|
+
import polars as pl
|
|
261
|
+
from rainfallqc.qc_frameworks import apply_qc_framework
|
|
262
|
+
|
|
263
|
+
network_data = pl.read_csv("hourly_rain_gauge_network.csv")
|
|
264
|
+
metadata = pl.read_csv("rain_gauge_metadata.csv")
|
|
265
|
+
|
|
266
|
+
# 1. Decide which QC methods of IntenseQC will be run
|
|
267
|
+
qc_framework = "IntenseQC"
|
|
268
|
+
qc_methods_to_run = ["QC1", "QC8", "QC9", "QC10", "QC11", "QC12", "QC14", "QC15", "QC16"]
|
|
269
|
+
|
|
270
|
+
# 2. Determine nearest neighbouring gauges for neighbourhood checks
|
|
271
|
+
gauge_lat = gpcc_metadata["latitude"]
|
|
272
|
+
gauge_lon = gpcc_metadata["longitude"]
|
|
273
|
+
nearest_neighbourhours = ["rain_mm_gauge_2", "rain_mm_gauge_3", ...] # or see Example 3 if not determined
|
|
274
|
+
|
|
275
|
+
# 2 Decide which parameters for QC
|
|
276
|
+
qc_kwargs = {
|
|
277
|
+
"QC1": {"quantile": 5},
|
|
278
|
+
"QC14": {"wet_day_threshold": 1.0, "accumulation_multiplying_factor": 2.0},
|
|
279
|
+
"QC16": {
|
|
280
|
+
"list_of_nearest_stations": nearest_neighbourhours,
|
|
281
|
+
"wet_threshold": 1.0,
|
|
282
|
+
"min_n_neighbours": 5,
|
|
283
|
+
"n_neighbours_ignored": 0,
|
|
284
|
+
},
|
|
285
|
+
"shared": {
|
|
286
|
+
"target_gauge_col": "rain_mm_gauge_1",
|
|
287
|
+
"gauge_lat": gauge_lat,
|
|
288
|
+
"gauge_lon": gauge_lon,
|
|
289
|
+
"time_res": "daily",
|
|
290
|
+
"smallest_measurable_rainfall_amount": 0.1,
|
|
291
|
+
},
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
# 3. Run QC methods on network data
|
|
295
|
+
qc_result = apply_qc_framework.run_qc_framework(
|
|
296
|
+
daily_rain_gauge_network, qc_framework=qc_framework, qc_methods_to_run=qc_methods_to_run, qc_kwargs=qc_kwargs
|
|
297
|
+
)
|
|
298
|
+
|
|
299
|
+
Because lots of the checks share the same parameters with a standard vocabulary, you can use the "shared" part of the ``qc_kwargs`` dictionary to set those.
|
|
300
|
+
|
|
301
|
+
Other examples
|
|
302
|
+
--------------
|
|
303
|
+
Of course, your data may not be tabular, or may not be stored in a single file. Therefore, please see our other `Tutorials <https://rainfallqc.readthedocs.io/en/latest/tutorials.html>`_.
|
|
304
|
+
|
|
305
|
+
There is also a `demo notebook <https://github.com/Thomasjkeel/RainfallQC-notebooks/blob/main/notebooks/demo/rainfallQC_demo.ipynb>`_.
|
|
306
|
+
|
|
307
|
+
Finally, different QC methods are suitable for different temporal resolutions, see our `Which checks are suitable for my data's temporal resolution? <https://rainfallqc.readthedocs.io/en/latest/quickstart.html>`_ for more information.
|
|
308
|
+
|
|
309
|
+
Documentation and License
|
|
310
|
+
=========================
|
|
311
|
+
* RainfallQC is developed and maintained by UKCEH.
|
|
312
|
+
* Free software: GNU General Public License v3
|
|
313
|
+
* Documentation: https://rainfallqc.readthedocs.io.
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
Features
|
|
317
|
+
========
|
|
318
|
+
|
|
319
|
+
- 27 rainfall QC methods (25 from IntenseQC, 2 from pyPWSQC)
|
|
320
|
+
- polars DataFrame support for fast data processing
|
|
321
|
+
- modular structure so you can pick and choose which checks to run
|
|
322
|
+
- support for single gauges or networks of gauges
|
|
323
|
+
- editable parameters so you can tweak thresholds, streak or accumulation lengths, and distances to neighbouring gauges
|
|
324
|
+
|
|
325
|
+
Credits
|
|
326
|
+
=======
|
|
327
|
+
* Builds upon `IntenseQC <https://github.com/nclwater/intense-qc/tree/master>`_, and (is compatible with) `pyPWSQC <https://github.com/OpenSenseAction/pypwsqc>`_:
|
|
328
|
+
* Please email tomkee@ceh.ac.uk if you have any questions.
|
|
329
|
+
* This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypackage`_ project template.
|
|
330
|
+
|
|
331
|
+
.. _Cookiecutter: https://github.com/audreyr/cookiecutter
|
|
332
|
+
.. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage
|
|
333
|
+
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
.. image:: https://github.com/NERC-CEH/RainfallQC/blob/main/docs/logos/rainfallQC_logo.png
|
|
2
|
+
:align: center
|
|
3
|
+
:height: 180px
|
|
4
|
+
:width: 200 px
|
|
5
|
+
:alt: RainfallQC
|
|
6
|
+
|
|
7
|
+
===============================================
|
|
8
|
+
RainfallQC - Quality control for rainfall data
|
|
9
|
+
===============================================
|
|
10
|
+
|
|
11
|
+
.. image:: https://img.shields.io/pypi/v/rainfallqc.svg
|
|
12
|
+
:target: https://pypi.python.org/pypi/rainfallqc
|
|
13
|
+
|
|
14
|
+
..
|
|
15
|
+
image:: https://readthedocs.org/projects/rainfallqc/badge/?version=latest
|
|
16
|
+
:target: https://rainfallqc.readthedocs.io/en/latest/?version=latest
|
|
17
|
+
:alt: Documentation Status
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
Provides methods for running rainfall quality control.
|
|
21
|
+
|
|
22
|
+
Installation
|
|
23
|
+
============
|
|
24
|
+
RainfallQC can be installed from PyPi:
|
|
25
|
+
|
|
26
|
+
.. code-block:: bash
|
|
27
|
+
|
|
28
|
+
pip install rainfallqc
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
Example use
|
|
32
|
+
===========
|
|
33
|
+
|
|
34
|
+
Example 1. - Running individual checks on a single rain gauge
|
|
35
|
+
-------------------------------------------------------------
|
|
36
|
+
Let's say you have data for a single rain gauge stored in "hourly_rain_gauge_data.csv" which looks like this:
|
|
37
|
+
|
|
38
|
+
.. table:: Example data 1. Single rain gauge
|
|
39
|
+
:widths: auto
|
|
40
|
+
:align: center
|
|
41
|
+
|
|
42
|
+
+---------------------+---------+
|
|
43
|
+
| time | rain_mm |
|
|
44
|
+
+=====================+=========+
|
|
45
|
+
| 2020-01-01 00:00 | 0.0 |
|
|
46
|
+
+---------------------+---------+
|
|
47
|
+
| 2020-01-01 01:00 | 0.1 |
|
|
48
|
+
+---------------------+---------+
|
|
49
|
+
| 2020-01-01 02:00 | 0.0 |
|
|
50
|
+
+---------------------+---------+
|
|
51
|
+
| 2020-01-01 03:00 | 105.0 |
|
|
52
|
+
+---------------------+---------+
|
|
53
|
+
| 2020-01-01 04:00 | 0.6 |
|
|
54
|
+
+---------------------+---------+
|
|
55
|
+
| ... | ... |
|
|
56
|
+
+---------------------+---------+
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
For the majority of the checks in RainfallQC, you can load in your data using `polars <https://pola-rs.github.io/polars-book/>`_ and run the checks directly.
|
|
60
|
+
Below, we run 2 example QC checks:
|
|
61
|
+
|
|
62
|
+
- 1) ``check_intermittency`` - to flag years where there are periods of non-zero bounded by 0 (see Figure 1.),
|
|
63
|
+
- 2) ``daily_accumulations`` - to flag accumulations of hourly values into daily.
|
|
64
|
+
|
|
65
|
+
.. figure:: https://thomasjkeel.github.io/UK-Rain-Gauge-Network/example_images/intermittency.png
|
|
66
|
+
:align: center
|
|
67
|
+
:height: 250px
|
|
68
|
+
:width: 300px
|
|
69
|
+
|
|
70
|
+
**Figure 1.** Example of an intermittency issue within the rainfall record
|
|
71
|
+
|
|
72
|
+
.. code-block:: python
|
|
73
|
+
|
|
74
|
+
import polars as pl
|
|
75
|
+
from rainfallqc import gauge_checks, timeseries_checks
|
|
76
|
+
|
|
77
|
+
data = pl.read_csv("hourly_rain_gauge_data.csv")
|
|
78
|
+
|
|
79
|
+
intermittent_years = gauge_checks.check_intermittency(data, target_gauge_col="rain_mm")
|
|
80
|
+
|
|
81
|
+
daily_accumulation_flags = timeseries_checks.check_daily_accumulations(
|
|
82
|
+
data,
|
|
83
|
+
target_gauge_col="rain_mm",
|
|
84
|
+
gauge_lat=52.0,
|
|
85
|
+
gauge_lon=2.0,
|
|
86
|
+
smallest_measurable_rainfall_amount=0.1,
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
Please note that some checks may require additional metadata, such as gauge location (latitude and longitude) or smallest measurable rainfall amount (e.g. 0.1 mm).
|
|
91
|
+
This could look like:
|
|
92
|
+
|
|
93
|
+
.. table:: Example metadata 1. Rain gauge metadata
|
|
94
|
+
:widths: auto
|
|
95
|
+
:align: center
|
|
96
|
+
|
|
97
|
+
+--------------------+----------+-----------+------------------+------------------+---------------------+
|
|
98
|
+
| station_id | latitude | longitude | start_datetime | end_datetime | path |
|
|
99
|
+
+====================+==========+===========+==================+==================+=====================+
|
|
100
|
+
| rain_mm_gauge_1 | 53.0 | 2.0 | 2020-01-01 00:00 | 2024-01-01 00:00 | path/to/gauge_1.csv |
|
|
101
|
+
+--------------------+----------+-----------+------------------+------------------+---------------------+
|
|
102
|
+
| rain_mm_gauge_2 | 54.1 | -0.5 | 2018-01-01 00:00 | 2023-01-01 00:00 | path/to/gauge_2.csv |
|
|
103
|
+
+--------------------+----------+-----------+------------------+------------------+---------------------+
|
|
104
|
+
| rain_mm_gauge_3 | 56.9 | 1.9 | 2015-01-01 00:00 | 2025-01-01 00:00 | path/to/gauge_3.csv |
|
|
105
|
+
+--------------------+----------+-----------+------------------+------------------+---------------------+
|
|
106
|
+
| ... | ... | ... | | | ... |
|
|
107
|
+
+--------------------+----------+-----------+------------------+------------------+---------------------+
|
|
108
|
+
|
|
109
|
+
You could then run checks that require metadata i.e. the ``check_hourly_exceedance_etccdi_rx1day`` QC check which flags rainfall values exceeding
|
|
110
|
+
the hourly day rainfall 1-day record at a given location (see Figure 2):
|
|
111
|
+
|
|
112
|
+
.. figure:: https://thomasjkeel.github.io/UK-Rain-Gauge-Network/example_images/rx1day_check.png
|
|
113
|
+
:align: center
|
|
114
|
+
:height: 250px
|
|
115
|
+
:width: 300px
|
|
116
|
+
|
|
117
|
+
**Figure 2.** Example of an Rx1day check from the IntenseQC framework
|
|
118
|
+
|
|
119
|
+
The code for that check looks like:
|
|
120
|
+
|
|
121
|
+
.. code-block:: python
|
|
122
|
+
|
|
123
|
+
import polars as pl
|
|
124
|
+
from rainfallqc import comparison_checks
|
|
125
|
+
|
|
126
|
+
data = pl.read_csv("hourly_rain_gauge_data_gauge_1.csv")
|
|
127
|
+
metadata = pl.read_csv("rain_gauge_metadata.csv")
|
|
128
|
+
|
|
129
|
+
target_gauge_id = "rain_mm_gauge_1"
|
|
130
|
+
target_metadata = metadata.filter(pl.col("station_id") == target_gauge_id)
|
|
131
|
+
|
|
132
|
+
rx1day_check = comparison_checks.check_hourly_exceedance_etccdi_rx1day(
|
|
133
|
+
data,
|
|
134
|
+
target_gauge_col=target_gauge_col,
|
|
135
|
+
gauge_lat=target_metadata["latitude"],
|
|
136
|
+
gauge_lon=target_metadata["longitude"]
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
Output flags will then look like:
|
|
140
|
+
|
|
141
|
+
.. table:: Example flag outputs for the Rx1day QC check
|
|
142
|
+
:widths: auto
|
|
143
|
+
:align: center
|
|
144
|
+
|
|
145
|
+
+---------------------+--------------+
|
|
146
|
+
| time | rx1day_check |
|
|
147
|
+
+=====================+==============+
|
|
148
|
+
| 2020-01-01 00:00 | 0 |
|
|
149
|
+
+---------------------+--------------+
|
|
150
|
+
| 2020-01-01 01:00 | 0 |
|
|
151
|
+
+---------------------+--------------+
|
|
152
|
+
| 2020-01-01 02:00 | 0 |
|
|
153
|
+
+---------------------+--------------+
|
|
154
|
+
| 2020-01-01 03:00 | 1 |
|
|
155
|
+
+---------------------+--------------+
|
|
156
|
+
| 2020-01-01 04:00 | 0 |
|
|
157
|
+
+---------------------+--------------+
|
|
158
|
+
| ... | ... |
|
|
159
|
+
+---------------------+--------------+
|
|
160
|
+
|
|
161
|
+
Example 2. - Running multiple QC checks on a single target gauge
|
|
162
|
+
----------------------------------------------------------------
|
|
163
|
+
To run multiple QC checks, you can use the `apply_qc_framework() <rainfallqc.checks.html#rainfallqc.qc_frameworks.html#module-rainfallqc.qc_frameworks.apply_qc_framework>`_
|
|
164
|
+
method to run QC methods from a given framework (e.g. IntenseQC).
|
|
165
|
+
|
|
166
|
+
Let's say you have hourly rainfall values from a rain gauge network data like:
|
|
167
|
+
|
|
168
|
+
.. table:: Example data 2. Rain gauge network
|
|
169
|
+
:widths: auto
|
|
170
|
+
:align: center
|
|
171
|
+
|
|
172
|
+
+---------------------+-----------------+-----------------+-----------------+
|
|
173
|
+
| time | rain_mm_gauge_1 | rain_mm_gauge_2 | rain_mm_gauge_3 |
|
|
174
|
+
+=====================+=================+=================+=================+
|
|
175
|
+
| 2020-01-01 00:00 | 0.0 | 0.5 | 0.0 |
|
|
176
|
+
+---------------------+-----------------+-----------------+-----------------+
|
|
177
|
+
| 2020-01-01 01:00 | 0.5 | 0.0 | 1.0 |
|
|
178
|
+
+---------------------+-----------------+-----------------+-----------------+
|
|
179
|
+
| 2020-01-01 02:00 | 0.0 | 1.0 | 0.0 |
|
|
180
|
+
+---------------------+-----------------+-----------------+-----------------+
|
|
181
|
+
| 2020-01-01 03:00 | 105.0 | 0.0 | 0.5 |
|
|
182
|
+
+---------------------+-----------------+-----------------+-----------------+
|
|
183
|
+
| 2020-01-01 04:00 | 0.0 | 0.5 | 0.0 |
|
|
184
|
+
+---------------------+-----------------+-----------------+-----------------+
|
|
185
|
+
| ... | ... | ... | ... |
|
|
186
|
+
+---------------------+-----------------+-----------------+-----------------+
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
... and metadata like example metdata 1.
|
|
190
|
+
You can then run multiple QC checks at once by defining a QC framework, the methods to run and parameters for those methods.
|
|
191
|
+
|
|
192
|
+
As of RainfallQC v0.3.0, there are three QC frameworks:
|
|
193
|
+
|
|
194
|
+
1. "intenseqc" - All 25 checks from IntenseQC/GSDR-QC with names like: "QC1", "QC2" ... "QC25",
|
|
195
|
+
2. "pypwsqc" - 2 checks from pyPWSQC with the names: "FZ" and "SO",
|
|
196
|
+
3. "custom" - Allows the user to select a custom set of checks (see Example 8 in `Tutorials <https://rainfallqc.readthedocs.io/en/latest/tutorials.html>`_).
|
|
197
|
+
|
|
198
|
+
Let's run some QC checks from intenseqc framework below:
|
|
199
|
+
|
|
200
|
+
.. code-block:: python
|
|
201
|
+
|
|
202
|
+
import polars as pl
|
|
203
|
+
from rainfallqc.qc_frameworks import apply_qc_framework
|
|
204
|
+
|
|
205
|
+
network_data = pl.read_csv("hourly_rain_gauge_network.csv")
|
|
206
|
+
metadata = pl.read_csv("rain_gauge_metadata.csv")
|
|
207
|
+
|
|
208
|
+
# 1. Decide which QC methods of IntenseQC will be run
|
|
209
|
+
qc_framework = "IntenseQC"
|
|
210
|
+
qc_methods_to_run = ["QC1", "QC8", "QC9", "QC10", "QC11", "QC12", "QC14", "QC15", "QC16"]
|
|
211
|
+
|
|
212
|
+
# 2. Determine nearest neighbouring gauges for neighbourhood checks
|
|
213
|
+
gauge_lat = gpcc_metadata["latitude"]
|
|
214
|
+
gauge_lon = gpcc_metadata["longitude"]
|
|
215
|
+
nearest_neighbourhours = ["rain_mm_gauge_2", "rain_mm_gauge_3", ...] # or see Example 3 if not determined
|
|
216
|
+
|
|
217
|
+
# 2 Decide which parameters for QC
|
|
218
|
+
qc_kwargs = {
|
|
219
|
+
"QC1": {"quantile": 5},
|
|
220
|
+
"QC14": {"wet_day_threshold": 1.0, "accumulation_multiplying_factor": 2.0},
|
|
221
|
+
"QC16": {
|
|
222
|
+
"list_of_nearest_stations": nearest_neighbourhours,
|
|
223
|
+
"wet_threshold": 1.0,
|
|
224
|
+
"min_n_neighbours": 5,
|
|
225
|
+
"n_neighbours_ignored": 0,
|
|
226
|
+
},
|
|
227
|
+
"shared": {
|
|
228
|
+
"target_gauge_col": "rain_mm_gauge_1",
|
|
229
|
+
"gauge_lat": gauge_lat,
|
|
230
|
+
"gauge_lon": gauge_lon,
|
|
231
|
+
"time_res": "daily",
|
|
232
|
+
"smallest_measurable_rainfall_amount": 0.1,
|
|
233
|
+
},
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
# 3. Run QC methods on network data
|
|
237
|
+
qc_result = apply_qc_framework.run_qc_framework(
|
|
238
|
+
daily_rain_gauge_network, qc_framework=qc_framework, qc_methods_to_run=qc_methods_to_run, qc_kwargs=qc_kwargs
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
Because lots of the checks share the same parameters with a standard vocabulary, you can use the "shared" part of the ``qc_kwargs`` dictionary to set those.
|
|
242
|
+
|
|
243
|
+
Other examples
|
|
244
|
+
--------------
|
|
245
|
+
Of course, your data may not be tabular, or may not be stored in a single file. Therefore, please see our other `Tutorials <https://rainfallqc.readthedocs.io/en/latest/tutorials.html>`_.
|
|
246
|
+
|
|
247
|
+
There is also a `demo notebook <https://github.com/Thomasjkeel/RainfallQC-notebooks/blob/main/notebooks/demo/rainfallQC_demo.ipynb>`_.
|
|
248
|
+
|
|
249
|
+
Finally, different QC methods are suitable for different temporal resolutions, see our `Which checks are suitable for my data's temporal resolution? <https://rainfallqc.readthedocs.io/en/latest/quickstart.html>`_ for more information.
|
|
250
|
+
|
|
251
|
+
Documentation and License
|
|
252
|
+
=========================
|
|
253
|
+
* RainfallQC is developed and maintained by UKCEH.
|
|
254
|
+
* Free software: GNU General Public License v3
|
|
255
|
+
* Documentation: https://rainfallqc.readthedocs.io.
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
Features
|
|
259
|
+
========
|
|
260
|
+
|
|
261
|
+
- 27 rainfall QC methods (25 from IntenseQC, 2 from pyPWSQC)
|
|
262
|
+
- polars DataFrame support for fast data processing
|
|
263
|
+
- modular structure so you can pick and choose which checks to run
|
|
264
|
+
- support for single gauges or networks of gauges
|
|
265
|
+
- editable parameters so you can tweak thresholds, streak or accumulation lengths, and distances to neighbouring gauges
|
|
266
|
+
|
|
267
|
+
Credits
|
|
268
|
+
=======
|
|
269
|
+
* Builds upon `IntenseQC <https://github.com/nclwater/intense-qc/tree/master>`_, and (is compatible with) `pyPWSQC <https://github.com/OpenSenseAction/pypwsqc>`_:
|
|
270
|
+
* Please email tomkee@ceh.ac.uk if you have any questions.
|
|
271
|
+
* This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypackage`_ project template.
|
|
272
|
+
|
|
273
|
+
.. _Cookiecutter: https://github.com/audreyr/cookiecutter
|
|
274
|
+
.. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage
|
|
@@ -68,7 +68,7 @@ homepage = "https://github.com/thomasjkeel/RainfallQC"
|
|
|
68
68
|
target-version = ["py310", "py311", "py312", "py313"]
|
|
69
69
|
|
|
70
70
|
[tool.bumpversion]
|
|
71
|
-
current_version = "0.
|
|
71
|
+
current_version = "0.3.0"
|
|
72
72
|
commit = true
|
|
73
73
|
message = "Bump version: {current_version} → {new_version}"
|
|
74
74
|
commit_args = ""
|